aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorCédric Ollivier <cedric.ollivier@orange.com>2019-12-19 07:13:12 +0100
committerCédric Ollivier <cedric.ollivier@orange.com>2019-12-21 12:08:12 +0100
commit65b489bbc42c7b24c7f9a435de6e5094d5208cf3 (patch)
treee832140b979e41a6787644d71b33058ec689e2a8
parent4ac26ff408f928518c7a35f1178fcb590b32cd7f (diff)
Dump DB and artifacts for third-party certification
It conforms with Kubernetes Conformance [1] model and could be reused by OVP or any third-party CNTT-based certification. It publishes the final archive in the S3 repository. XtestingCI will generate the final job calling this feature. [1] https://github.com/cncf/k8s-conformance Change-Id: I7394c9d52d2cc9370eae125ca5398ff734e5b8d6 Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
-rw-r--r--setup.cfg1
-rw-r--r--xtesting/core/campaign.py215
-rw-r--r--xtesting/core/testcase.py4
3 files changed, 218 insertions, 2 deletions
diff --git a/setup.cfg b/setup.cfg
index 76d0b885..9b8dc92e 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -23,6 +23,7 @@ packages = xtesting
[entry_points]
console_scripts =
run_tests = xtesting.ci.run_tests:main
+ zip_campaign = xtesting.core.campaign:main
xtesting.testcase =
bashfeature = xtesting.core.feature:BashFeature
robotframework = xtesting.core.robotframework:RobotFramework
diff --git a/xtesting/core/campaign.py b/xtesting/core/campaign.py
new file mode 100644
index 00000000..8419cf3a
--- /dev/null
+++ b/xtesting/core/campaign.py
@@ -0,0 +1,215 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2019 Orange and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+"""Dump DB and artifacts for third-party certifications."""
+
+import json
+import logging
+import logging.config
+import mimetypes
+import os
+import re
+import zipfile
+
+import boto3
+from boto3.s3.transfer import TransferConfig
+import botocore
+import pkg_resources
+import requests
+from six.moves import urllib
+
+from xtesting.core import testcase
+from xtesting.utils import env
+
+__author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
+
+
+class Campaign():
+ "Dump, archive and publish all results and artifacts from a campaign."
+
+ EX_OK = os.EX_OK
+ """everything is OK"""
+
+ EX_DUMP_FROM_DB_ERROR = os.EX_SOFTWARE - 5
+ """dump_db() failed"""
+
+ EX_DUMP_ARTIFACTS_ERROR = os.EX_SOFTWARE - 6
+ """dump_artifacts() failed"""
+
+ EX_ZIP_CAMPAIGN_FILES_ERROR = os.EX_SOFTWARE - 7
+ """dump_artifacts() failed"""
+
+ __logger = logging.getLogger(__name__)
+
+ @staticmethod
+ def dump_db():
+ """Dump all test campaign results from the DB.
+
+ It allows collecting all the results from the DB.
+
+ It could be overriden if the common implementation is not
+ suitable.
+
+ The next vars must be set in env:
+
+ * TEST_DB_URL,
+ * BUILD_TAG.
+
+ Returns:
+ Campaign.EX_OK if results were collected from DB.
+ Campaign.EX_DUMP_FROM_DB_ERROR otherwise.
+ """
+ try:
+ url = env.get('TEST_DB_URL')
+ req = requests.get(
+ "{}?build_tag={}".format(url, env.get('BUILD_TAG')),
+ headers=testcase.TestCase.headers)
+ req.raise_for_status()
+ Campaign.__logger.debug("data from DB: \n%s", req.json())
+ with open("{}.json".format(env.get('BUILD_TAG')), "w") as dfile:
+ json.dump(req.json(), dfile)
+ except Exception: # pylint: disable=broad-except
+ Campaign.__logger.exception(
+ "The results cannot be collected from DB")
+ return Campaign.EX_DUMP_FROM_DB_ERROR
+ return Campaign.EX_OK
+
+ @staticmethod
+ def dump_artifacts():
+ """Dump all test campaign artifacts from the S3 repository.
+
+ It allows collecting all the artifacts from the S3 repository.
+
+ It could be overriden if the common implementation is not
+ suitable.
+
+ The credentials must be configured before publishing the artifacts:
+
+ * fill ~/.aws/credentials or ~/.boto,
+ * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
+
+ The next vars must be set in env:
+
+ * S3_ENDPOINT_URL (http://127.0.0.1:9000),
+ * S3_DST_URL (s3://xtesting/prefix),
+
+ Returns:
+ Campaign.EX_OK if artifacts were published to repository.
+ Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
+ """
+ try:
+ build_tag = env.get('BUILD_TAG')
+ b3resource = boto3.resource(
+ 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
+ dst_s3_url = os.environ["S3_DST_URL"]
+ multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
+ "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
+ config = TransferConfig(multipart_threshold=multipart_threshold)
+ bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
+ s3path = re.search(
+ '^/*(.*)/*$', urllib.parse.urlparse(dst_s3_url).path).group(1)
+ prefix = os.path.join(s3path, build_tag)
+ # pylint: disable=no-member
+ for s3_object in b3resource.Bucket(bucket_name).objects.filter(
+ Prefix="{}/".format(prefix)):
+ path, _ = os.path.split(s3_object.key)
+ lpath = re.sub('^{}/*'.format(s3path), '', path)
+ if lpath and not os.path.exists(lpath):
+ os.makedirs(lpath)
+ # pylint: disable=no-member
+ b3resource.Bucket(bucket_name).download_file(
+ s3_object.key,
+ re.sub('^{}/*'.format(s3path), '', s3_object.key),
+ Config=config)
+ Campaign.__logger.info(
+ "Downloading %s",
+ re.sub('^{}/*'.format(s3path), '', s3_object.key))
+ return Campaign.EX_OK
+ except Exception: # pylint: disable=broad-except
+ Campaign.__logger.exception("Cannot publish the artifacts")
+ return Campaign.EX_DUMP_ARTIFACTS_ERROR
+
+ @staticmethod
+ def zip_campaign_files(): # pylint: disable=too-many-locals
+ """Archive and publish all test campaign data to the S3 repository.
+
+ It allows collecting all the artifacts from the S3 repository.
+
+ It could be overriden if the common implementation is not
+ suitable.
+
+ The credentials must be configured before publishing the artifacts:
+
+ * fill ~/.aws/credentials or ~/.boto,
+ * set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env.
+
+ The next vars must be set in env:
+
+ * S3_ENDPOINT_URL (http://127.0.0.1:9000),
+ * S3_DST_URL (s3://xtesting/prefix),
+
+ Returns:
+ Campaign.EX_OK if artifacts were published to repository.
+ Campaign.EX_DUMP_ARTIFACTS_ERROR otherwise.
+ """
+ try:
+ build_tag = env.get('BUILD_TAG')
+ assert Campaign.dump_db() == Campaign.EX_OK
+ assert Campaign.dump_artifacts() == Campaign.EX_OK
+ with zipfile.ZipFile('{}.zip'.format(build_tag), 'w') as zfile:
+ zfile.write("{}.json".format(build_tag))
+ for root, _, files in os.walk(build_tag):
+ for filename in files:
+ zfile.write(os.path.join(root, filename))
+ b3resource = boto3.resource(
+ 's3', endpoint_url=os.environ["S3_ENDPOINT_URL"])
+ dst_s3_url = os.environ["S3_DST_URL"]
+ multipart_threshold = 5 * 1024 ** 5 if "google" in os.environ[
+ "S3_ENDPOINT_URL"] else 8 * 1024 * 1024
+ config = TransferConfig(multipart_threshold=multipart_threshold)
+ bucket_name = urllib.parse.urlparse(dst_s3_url).netloc
+ mime_type = mimetypes.guess_type('{}.zip'.format(build_tag))
+ path = urllib.parse.urlparse(dst_s3_url).path.strip("/")
+ # pylint: disable=no-member
+ b3resource.Bucket(bucket_name).upload_file(
+ '{}.zip'.format(build_tag),
+ os.path.join(path, '{}.zip'.format(build_tag)),
+ Config=config,
+ ExtraArgs={'ContentType': mime_type[
+ 0] or 'application/octet-stream'})
+ dst_http_url = os.environ["HTTP_DST_URL"]
+ link = os.path.join(dst_http_url, '{}.zip'.format(build_tag))
+ Campaign.__logger.info(
+ "All data were successfully published:\n\n%s", link)
+ return Campaign.EX_OK
+ except KeyError as ex:
+ Campaign.__logger.error("Please check env var: %s", str(ex))
+ return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
+ except botocore.exceptions.NoCredentialsError:
+ Campaign.__logger.error(
+ "Please fill ~/.aws/credentials, ~/.boto or set "
+ "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in env")
+ return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
+ except Exception: # pylint: disable=broad-except
+ Campaign.__logger.exception("Cannot publish the artifacts")
+ return Campaign.EX_ZIP_CAMPAIGN_FILES_ERROR
+
+
+def main():
+ """Entry point for Campaign.zip_campaign_files()."""
+ if not os.path.exists(testcase.TestCase.dir_results):
+ os.makedirs(testcase.TestCase.dir_results)
+ if env.get('DEBUG').lower() == 'true':
+ logging.config.fileConfig(pkg_resources.resource_filename(
+ 'xtesting', 'ci/logging.debug.ini'))
+ else:
+ logging.config.fileConfig(pkg_resources.resource_filename(
+ 'xtesting', 'ci/logging.ini'))
+ logging.captureWarnings(True)
+ Campaign.zip_campaign_files()
diff --git a/xtesting/core/testcase.py b/xtesting/core/testcase.py
index 9ccd4376..c4dec22f 100644
--- a/xtesting/core/testcase.py
+++ b/xtesting/core/testcase.py
@@ -57,7 +57,7 @@ class TestCase():
dir_results = "/var/lib/xtesting/results"
_job_name_rule = "(dai|week)ly-(.+?)-[0-9]*"
- _headers = {'Content-Type': 'application/json'}
+ headers = {'Content-Type': 'application/json'}
__logger = logging.getLogger(__name__)
def __init__(self, **kwargs):
@@ -234,7 +234,7 @@ class TestCase():
data["version"] = "unknown"
req = requests.post(
url, data=json.dumps(data, sort_keys=True),
- headers=self._headers)
+ headers=self.headers)
req.raise_for_status()
if urllib.parse.urlparse(url).scheme != "file":
# href must be postprocessed as OPNFV testapi is misconfigured