From 53162c3c767acca082b1471294ff54e98169abe9 Mon Sep 17 00:00:00 2001
From: opensource-tnbt <sridhar.rao@spirent.com>
Date: Thu, 10 Sep 2020 17:43:24 +0530
Subject: CIRV_SDV: URLS Checker in container.
This patch implements a Installer Manifest URLS checker as a container
application.
Signed-off-by: Sridhar K. N. Rao <sridhar.rao@spirent.com>
Change-Id: Iff143034a0f4a883e6a06db289bf784dc5cc2b72
---
sdv/docker/sdvurls/Dockerfile | 10 ++
sdv/docker/sdvurls/requirements.txt | 13 ++
sdv/docker/sdvurls/server | 281 ++++++++++++++++++++++++++++++++++++
sdv/docs/docker/urls/userguide.rst | 24 +++
4 files changed, 328 insertions(+)
create mode 100644 sdv/docker/sdvurls/Dockerfile
create mode 100644 sdv/docker/sdvurls/requirements.txt
create mode 100644 sdv/docker/sdvurls/server
create mode 100644 sdv/docs/docker/urls/userguide.rst
diff --git a/sdv/docker/sdvurls/Dockerfile b/sdv/docker/sdvurls/Dockerfile
new file mode 100644
index 0000000..e6d447a
--- /dev/null
+++ b/sdv/docker/sdvurls/Dockerfile
@@ -0,0 +1,10 @@
+FROM python:3.8-slim-buster
+
+WORKDIR /sdvurls/
+
+COPY requirements.txt /state/requirements.txt
+RUN pip install -r requirements.txt
+
+COPY server /sdvurls/
+
+CMD [ "python", "/sdvurls/server" ]
diff --git a/sdv/docker/sdvurls/requirements.txt b/sdv/docker/sdvurls/requirements.txt
new file mode 100644
index 0000000..c38d2e9
--- /dev/null
+++ b/sdv/docker/sdvurls/requirements.txt
@@ -0,0 +1,13 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+# Copyright (c) 2020 Spirent Communications
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+tornado == 6.0.4
+urllib3 # MIT
+GitPython;python_version>='3.0' # BSD License (3 clause)
diff --git a/sdv/docker/sdvurls/server b/sdv/docker/sdvurls/server
new file mode 100644
index 0000000..8d3ec7a
--- /dev/null
+++ b/sdv/docker/sdvurls/server
@@ -0,0 +1,281 @@
+# Copyright 2020 Spirent Communications.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Airship implementation of Software Predeployment Validation
+"""
+
+import os
+import shutil
+from pathlib import Path
+import logging
+import json
+import git
+import urllib3
+
+from tornado.web import Application
+from tornado.ioloop import IOLoop
+import tornado.concurrent
+import tornado.httpserver
+import tornado.ioloop
+import tornado.options
+import tornado.web
+import tornado.log
+
+
+def check_link(link):
+ """
+ Function the check the availability of Hyperlinks
+ """
+ timeout = urllib3.util.Timeout(connect=5.0, read=7.0)
+ http = urllib3.PoolManager(timeout=timeout)
+ try:
+ http.request('HEAD', link)
+ except urllib3.exceptions.LocationValueError as err:
+ print(err.args)
+ return False
+ except urllib3.exceptions.MaxRetryError as err:
+ print(err.args)
+ return False
+ except urllib3.exceptions.RequestError as err:
+ print(err.args)
+ return False
+ except urllib3.exceptions.ConnectTimeoutError as err:
+ print(err.args)
+ return False
+ except urllib3.exceptions.PoolError as err:
+ print(err.args)
+ return False
+ except urllib3.exceptions.HTTPError as err:
+ print(err.args)
+ return False
+ return True
+
+
+class Airship():
+ """
+ Ariship URLS Validation
+ """
+ def __init__(self, params):
+ """ Airship class constructor """
+ self.url = params['AIRSHIP_MANIFEST_URL']
+ self.branch = params['AIRSHIP_MANIFEST_BRANCH']
+ self.dl_path = '/tmp'
+ self.site_name = params['AIRSHIP_MANIFEST_SITE_NAME']
+ self.tmversion = params['AIRSHIP_TREASUREMAP_VERSION']
+ self.manifest = None
+ self.dirpath = Path(self.dl_path, 'airship')
+ self.tmdirpath = Path(self.dl_path, 'treasuremap')
+ self.locations = []
+ self.validcount = 0
+ self.invalidcount = 0
+ self.respath = os.path.join(self.dl_path, ('urls-' +
+ self.site_name +
+ '-check.txt'))
+
+ def clone_repo(self):
+ """
+ Cloning the repos
+ """
+ git.Repo.clone_from(self.url,
+ self.dirpath,
+ branch=self.branch)
+ git.Repo.clone_from('https://github.com/airshipit/treasuremap',
+ self.tmdirpath,
+ branch=self.tmversion)
+
+ def cleanup_manifest(self):
+ """
+ Remove existing manifests
+ """
+ # Next Remove any manifest files, if it exists
+ if self.dirpath.exists() and self.dirpath.is_dir():
+ shutil.rmtree(self.dirpath)
+ if self.tmdirpath.exists() and self.tmdirpath.is_dir():
+ shutil.rmtree(self.tmdirpath)
+
+ def manifest_exists_locally(self):
+ """
+ Check if manifests exists locally
+ """
+ if self.dirpath.exists() and self.dirpath.is_dir():
+ return True
+ return False
+
+ def validate(self):
+ """
+ Hyperlink Validation
+ """
+ self.cleanup_manifest()
+ # Next, clone the repo to the provided path.
+ self.clone_repo()
+
+ if self.dirpath.exists() and self.dirpath.is_dir():
+ # Get the file(s) where links are defined.
+ self.find_locations(
+ os.path.join(self.dirpath, 'type',
+ 'cntt', 'software',
+ 'config', 'versions.yaml'))
+ self.find_locations(
+ os.path.join(self.tmdirpath, 'global',
+ 'software', 'config', 'versions.yaml'))
+
+ with open(self.respath, "w+") as report:
+ for location in self.locations:
+ if check_link(location):
+ report.write("The Link: %s is VALID" % (location))
+ self.validcount += 1
+ else:
+ self.invalidcount += 1
+ report.write("The Link: %s is INVALID" % (location))
+ self.cleanup_manifest()
+
+ def getresults(self):
+ """
+ Return Valid and Invalid Counts
+ """
+ return(self.validcount, self.invalidcount)
+
+ # pylint: disable=consider-using-enumerate
+ def find_locations(self, yamlfile):
+ """
+ Find all the hyperlinks in the manifests
+ """
+ with open(yamlfile, 'r') as filep:
+ lines = filep.readlines()
+ for index in range(len(lines)):
+ line = lines[index].strip()
+ if line.startswith('location:'):
+ link = line.split(":", 1)[1]
+ if "opendev" in link:
+ if ((len(lines) > index+1) and
+ (lines[index+1].strip().startswith(
+ 'reference:'))):
+ ref = lines[index+1].split(":", 1)[1]
+ link = link + '/commit/' + ref.strip()
+ if link.strip() not in self.locations:
+ print(link)
+ self.locations.append(link.strip())
+ if 'docker.' in line:
+ link = line.split(":", 1)[1]
+ link = link.replace('"', '')
+ parts = link.split('/')
+ if len(parts) == 3:
+ link = ('https://index.' +
+ parts[0].strip() +
+ '/v1/repositories/' +
+ parts[1] + '/' + parts[2].split(':')[0] +
+ '/tags/' + parts[2].split(':')[-1])
+ if link.strip() not in self.locations:
+ print(link)
+ self.locations.append(link.strip())
+ # quay.io/coreos/etcd:v3.4.2
+ # https://quay.io/api/v1/repository/coreos/etcd/tag/v3.4.2
+ if 'quay.' in line:
+ link = line.split(":", 1)[1]
+ link = link.replace('"', '')
+ parts = link.split('/')
+ if len(parts) == 3:
+ link = ('https://' +
+ parts[0].strip() +
+ '/api/v1/repository/' +
+ parts[1] + '/' + parts[2].split(':')[0] +
+ '/tag/' + parts[2].split(':')[-1])
+ if link.strip() not in self.locations:
+ print(link)
+ self.locations.append(link.strip())
+
+
+# pylint: disable=W0223
+class AirshipUrlsValidator(tornado.web.RequestHandler):
+ """ Validate URLS """
+ def set_default_headers(self):
+ """ set default headers"""
+ self.set_header('Content-Type', 'application/json')
+
+ def post(self):
+ """
+ POST request
+ usage:
+ /airship/?name='' installer='' link='' version=''
+ :return: logs from test results
+ """
+ # decode the body
+ data = json.loads(self.request.body.decode())
+ params = {}
+ branch = 'master'
+ installer = data['installer']
+ name = data['name']
+ link = data['link']
+ version = data['version']
+ if installer and 'airship' in installer.lower():
+ if name and link and branch and version:
+ params['AIRSHIP_MANIFEST_URL'] = link
+ params['AIRSHIP_MANIFEST_BRANCH'] = branch
+ params['AIRSHIP_MANIFEST_SITE_NAME'] = name
+ params['AIRSHIP_TREASUREMAP_VERSION'] = version
+ airship = Airship(params)
+ airship.validate()
+ valid, invalid = airship.getresults()
+ self.write("Valid Links: " +
+ str(valid) +
+ " Invalid Links: " +
+ str(invalid))
+
+
+# pylint: disable=W0223
+class TripleoUrlsValidator(tornado.web.RequestHandler):
+ """ Validate URLS """
+
+ def post(self):
+ """
+ POST request
+ """
+ self.write('error: Not Implemented')
+
+
+def main():
+ """ The Main Control """
+ app = Application([('/airship', AirshipUrlsValidator),
+ ('/tripleo', TripleoUrlsValidator)])
+
+ # Cli Config
+ tornado.options.define("port", default=8989,
+ help="running on the given port", type=int)
+ tornado.options.parse_command_line()
+
+ # Server Config
+ http_server = tornado.httpserver.HTTPServer(app)
+ http_server.listen(tornado.options.options.port)
+
+ # Tornado's event loop handles it from here
+ print("# Server Listening.... \n [Ctrl + C] to quit")
+
+ # Logging
+ log_file_filename = "/var/log/tornado.log"
+ handler = logging.FileHandler(log_file_filename)
+ app_log = logging.getLogger("tornado.general")
+ tornado.log.enable_pretty_logging()
+ app_log.addHandler(handler)
+
+ try:
+ tornado.ioloop.IOLoop.instance().start()
+ except KeyboardInterrupt:
+ tornado.ioloop.IOLoop.instance().stop()
+
+ # start
+ IOLoop.instance().start()
+
+if __name__ == "__main__":
+ main()
diff --git a/sdv/docs/docker/urls/userguide.rst b/sdv/docs/docker/urls/userguide.rst
new file mode 100644
index 0000000..15d0724
--- /dev/null
+++ b/sdv/docs/docker/urls/userguide.rst
@@ -0,0 +1,24 @@
+********************************************************
+CIRV-SDV: Validating the URLs in the Installer Manifests
+********************************************************
+
+Supported Installer Manifest: Airship.
+
+Building and starting the container:
+* Go to folder sdv/docker/sdvurls
+* Build the container with 'docker build' command. Consider naming/tagging the container properly.
+* Run the container using docker run. The container creates a report under /tmp folder. Hence, consider mapping a volume to '/tmp' folder to get the report.
+
+
+Interacting with the container
+##############################
+Inputs:
+
+* Installer Used. Keyword: "installer". Example Value: "airship". This is mandatory
+* Link to the installer manifests. Keyword: "link". Example Value: "https://gerrit.opnfv.org/gerrit/airship". This is mandatory
+* Version (For Airship, this refers to Treasuremap Version). Keyword: "version". Example Value: "v1.7". This is mandatory only for Airship.
+* Name of the site. Keyword: "name". Example Value: "intel-pod10". This is mandatory only for Airship
+
+Assuming the container is running locally, the example command would be::
+
+ curl --header "Content-Type: application/json" --redata '{"installer":"airship", "link":"https://gerrit.opnfv.org/gerrit/airship", "version":"v1.7", "name":"intel-pod10"}' http://localhost:8989/airship
--
cgit