Dump DB and artifacts for third-party certification

It conforms with Kubernetes Conformance [1] model and could be reused
by OVP or any third-party CNTT-based certification.

It publishes the final archive in the S3 repository.

XtestingCI will generate the final job calling this feature.

[1] https://github.com/cncf/k8s-conformance

Change-Id: I7394c9d52d2cc9370eae125ca5398ff734e5b8d6
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
(cherry picked from commit 65b489bbc42c7b24c7f9a435de6e5094d5208cf3)

4 files changed, 220 insertions, 3 deletions

diff --git a/setup.cfg b/setup.cfg
index 279e46a3..7eb7b83b 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -23,6 +23,7 @@ packages = xtesting
 [entry_points]
 console_scripts =
     run_tests = xtesting.ci.run_tests:main
+    zip_campaign = xtesting.core.campaign:main
 xtesting.testcase =
     bashfeature = xtesting.core.feature:BashFeature
     robotframework = xtesting.core.robotframework:RobotFramework
diff --git a/tox.ini b/tox.ini
index 1f71cbe9..bfe2b727 100644
--- a/tox.ini
+++ b/tox.ini
@@ -32,7 +32,8 @@ commands = flake8
 basepython = python2.7
 whitelist_externals = bash
 commands =
-  pylint --disable=locally-disabled --ignore-imports=y --reports=n xtesting
+  pylint --min-similarity-lines=10 \
+    --disable=locally-disabled --ignore-imports=y --reports=n xtesting
 
 [testenv:yamllint]
 basepython = python2.7
diff --git a/xtesting/core/campaign.py b/xtesting/core/campaign.py
new file mode 100644
index 00000000..edf109b5
--- /dev/null
+++ b/xtesting/core/campaign.py
@@ -0,0 +1,215 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2019 Orange and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+"""Dump DB and artifacts for third-party certifications."""
+
+import json
+import logging
+import logging.config
+import mimetypes
+import os
+import re
+import zipfile
+
+import boto3
+from boto3.s3.transfer import TransferConfig
+import botocore
+import pkg_resources
+import requests
+from six.moves import urllib
+
+from xtesting.core import testcase
+from xtesting.utils import env
+
+__author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
+
+
+class Campaign(object):
+    "Dump, archive and publish all results and artifacts from a campaign."
+
+    EX_OK = os.EX_OK
+    """everything is OK"""
+
+    EX_DUMP_FROM_DB_ERROR = os.EX_SOFTWARE - 5
+    """dump_db() failed"""
+
+    EX_DUMP_ARTIFACTS_ERROR = os.EX_SOFTWARE - 6
+    """dump_artifacts() failed"""
+
+    EX_ZIP_CAMPAIGN_FILES_ERROR = os.EX_SOFTWARE - 7
+    """dump_artifacts() failed"""
+
+    __logger = logging.getLogger(__name__)
+
+    @staticmethod
+    def dump_db():
+        """Dump all test campaign results from the DB.
+
+        It allows collecting all the results from the DB.
+
+        It could be overriden if the common implementation is not
+        suitable.
+
+        The next vars must be set in env:
+
+            * TEST_DB_URL,
+            * BUILD_TAG.
+
+        Returns:
+            Campaign.EX_OK if results were collected from DB.
+            Campaign.EX_DUMP_FROM_DB_ERROR otherwise.
+        """
+        try:
+            url = env.get('TEST_DB_URL')
+            req = requests.get(
+                "{}?build_tag={}".format(url, env.get('BUILD_TAG')),
+                headers=testcase.TestCase.headers)
+            req.raise_for_status()
+            Campaign.__logger.debug("data from DB: \n%s", req.json())
+            with open("{}.json".format(env.get('BUILD_TAG')), "w") as dfile:
+                json.dump(req.json(), dfile)
+        except Exception:  # pylint: disable=broad-except
+            Campaign.__logger.exception(
+                "The results cannot be collected from DB")
+            return Campaign.EX_DUMP_FROM_DB_ERROR
+        return Campaign.EX_OK
+
+    @staticmethod
+    def dump_artifacts():
+        """Dump all test campaign artifacts from the S3 repository.
+
+        It allows collecting all the artifacts from the S3 repository.
+
+        It could be overriden if the common implementation is not
+        suitable.
+
+        The credentials must be configured before publishing the artifacts:
+
+            * fill ~/.aws/credentials or ~/.boto,
+            * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
+
+        The next vars must be set in env:
+
+            * S3_ENDPOINT_URL (http://127.0.0.1:9000),
+            * S3_DST_URL (s3://xtesting/prefix),
+
+        Returns:
+            Campaign.EX_OK if artifacts were published to repository.
+            Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
+        """
+        try:
+            build_tag = env.get('BUILD_TAG')
+            b3resource = boto3.resource(
+                's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
+            dst_s3_url = os.environ["S3_DST_URL"]
+            multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
+                "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
+            config = TransferConfig(multipart_threshold=multipart_threshold)
+            bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
+            s3path = re.search(
+                '^/*(.*)/*$', urllib.parse.urlparse(dst_s3_url).path).group(1)
+            prefix = os.path.join(s3path, build_tag)
+            # pylint: disable=no-member
+            for s3_object in b3resource.Bucket(bucket_name).objects.filter(
+                    Prefix="{}/".format(prefix)):
+                path, _ = os.path.split(s3_object.key)
+                lpath = re.sub('^{}/*'.format(s3path), '', path)
+                if lpath and not os.path.exists(lpath):
+                    os.makedirs(lpath)
+                # pylint: disable=no-member
+                b3resource.Bucket(bucket_name).download_file(
+                    s3_object.key,
+                    re.sub('^{}/*'.format(s3path), '', s3_object.key),
+                    Config=config)
+                Campaign.__logger.info(
+                    "Downloading %s",
+                    re.sub('^{}/*'.format(s3path), '', s3_object.key))
+            return Campaign.EX_OK
+        except Exception:  # pylint: disable=broad-except
+            Campaign.__logger.exception("Cannot publish the artifacts")
+            return Campaign.EX_DUMP_ARTIFACTS_ERROR
+
+    @staticmethod
+    def zip_campaign_files():  # pylint: disable=too-many-locals
+        """Archive and publish all test campaign data to the S3 repository.
+
+        It allows collecting all the artifacts from the S3 repository.
+
+        It could be overriden if the common implementation is not
+        suitable.
+
+        The credentials must be configured before publishing the artifacts:
+
+            * fill ~/.aws/credentials or ~/.boto,
+            * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
+
+        The next vars must be set in env:
+
+            * S3_ENDPOINT_URL (http://127.0.0.1:9000),
+            * S3_DST_URL (s3://xtesting/prefix),
+
+        Returns:
+            Campaign.EX_OK if artifacts were published to repository.
+            Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
+        """
+        try:
+            build_tag = env.get('BUILD_TAG')
+            assert Campaign.dump_db() == Campaign.EX_OK
+            assert Campaign.dump_artifacts() == Campaign.EX_OK
+            with zipfile.ZipFile('{}.zip'.format(build_tag), 'w') as zfile:
+                zfile.write("{}.json".format(build_tag))
+                for root, _, files in os.walk(build_tag):
+                    for filename in files:
+                        zfile.write(os.path.join(root, filename))
+            b3resource = boto3.resource(
+                's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
+            dst_s3_url = os.environ["S3_DST_URL"]
+            multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
+                "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
+            config = TransferConfig(multipart_threshold=multipart_threshold)
+            bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
+            mime_type = mimetypes.guess_type('{}.zip'.format(build_tag))
+            path = urllib.parse.urlparse(dst_s3_url).path.strip("/")
+            # pylint: disable=no-member
+            b3resource.Bucket(bucket_name).upload_file(
+                '{}.zip'.format(build_tag),
+                os.path.join(path, '{}.zip'.format(build_tag)),
+                Config=config,
+                ExtraArgs={'ContentType': mime_type[
+                    0] or 'application/octet-stream'})
+            dst_http_url = os.environ["HTTP_DST_URL"]
+            link = os.path.join(dst_http_url, '{}.zip'.format(build_tag))
+            Campaign.__logger.info(
+                "All data were successfully published:\n\n%s", link)
+            return Campaign.EX_OK
+        except KeyError as ex:
+            Campaign.__logger.error("Please check env var: %s", str(ex))
+            return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
+        except botocore.exceptions.NoCredentialsError:
+            Campaign.__logger.error(
+                "Please fill ~/.aws/credentials, ~/.boto or set "
+                "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
+            return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
+        except Exception:  # pylint: disable=broad-except
+            Campaign.__logger.exception("Cannot publish the artifacts")
+            return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
+
+
+def main():
+    """Entry point for Campaign.zip_campaign_files()."""
+    if not os.path.exists(testcase.TestCase.dir_results):
+        os.makedirs(testcase.TestCase.dir_results)
+    if env.get('DEBUG').lower() == 'true':
+        logging.config.fileConfig(pkg_resources.resource_filename(
+            'xtesting', 'ci/logging.debug.ini'))
+    else:
+        logging.config.fileConfig(pkg_resources.resource_filename(
+            'xtesting', 'ci/logging.ini'))
+    logging.captureWarnings(True)
+    Campaign.zip_campaign_files()
diff --git a/xtesting/core/testcase.py b/xtesting/core/testcase.py
index ab82ecfc..f72cd377 100644
--- a/xtesting/core/testcase.py
+++ b/xtesting/core/testcase.py
@@ -57,7 +57,7 @@ class TestCase(object):
 
     dir_results = "/var/lib/xtesting/results"
     _job_name_rule = "(dai|week)ly-(.+?)-[0-9]*"
-    _headers = {'Content-Type': 'application/json'}
+    headers = {'Content-Type': 'application/json'}
     __logger = logging.getLogger(__name__)
 
     def __init__(self, **kwargs):
@@ -234,7 +234,7 @@ class TestCase(object):
                 data["version"] = "unknown"
             req = requests.post(
                 url, data=json.dumps(data, sort_keys=True),
-                headers=self._headers)
+                headers=self.headers)
             req.raise_for_status()
             if urllib.parse.urlparse(url).scheme != "file":
                 # href must be postprocessed as OPNFV testapi is misconfigured