diff options
Diffstat (limited to 'utils')
-rw-r--r-- | utils/slave-monitor-0.1.sh | 98 | ||||
-rw-r--r-- | utils/test/testapi/3rd_party/static/testapi-ui/components/pods/pods.html | 10 | ||||
-rw-r--r-- | utils/test/testapi/3rd_party/static/testapi-ui/components/pods/podsController.js | 5 | ||||
-rw-r--r-- | utils/test/testapi/MANIFEST.in | 1 | ||||
-rwxr-xr-x | utils/test/testapi/docker/prepare-env.sh | 2 | ||||
-rw-r--r-- | utils/test/testapi/etc/config.ini | 5 | ||||
-rw-r--r-- | utils/test/testapi/opnfv_testapi/cmd/server.py | 2 | ||||
-rw-r--r-- | utils/test/testapi/opnfv_testapi/common/check.py | 35 | ||||
-rw-r--r-- | utils/test/testapi/opnfv_testapi/common/config.py | 19 | ||||
-rw-r--r-- | utils/test/testapi/opnfv_testapi/common/constants.py | 1 | ||||
-rw-r--r-- | utils/test/testapi/opnfv_testapi/router/url_mappings.py | 3 | ||||
-rw-r--r-- | utils/test/testapi/opnfv_testapi/ui/auth/sign.py | 55 | ||||
-rw-r--r-- | utils/test/testapi/opnfv_testapi/ui/root.py | 4 | ||||
-rw-r--r-- | utils/test/testapi/setup.cfg | 15 | ||||
-rw-r--r-- | utils/test/testapi/setup.py | 9 | ||||
-rw-r--r-- | utils/test/testapi/tools/watchdog/docker_watch.sh | 63 |
16 files changed, 194 insertions, 133 deletions
diff --git a/utils/slave-monitor-0.1.sh b/utils/slave-monitor-0.1.sh new file mode 100644 index 000000000..161aaef21 --- /dev/null +++ b/utils/slave-monitor-0.1.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +#This will put a bunch of files in the pwd. you have been warned. +#Counts how long slaves have been online or offline + + +#Yes I know about jq +curlcommand() { +curl -s "https://build.opnfv.org/ci/computer/api/json?tree=computer\[displayName,offline\]" \ + | awk -v k=":" '{n=split($0,a,","); for (i=1; i<=n; i++) print a[i]}' \ + | grep -v "_class" \ + | awk 'NR%2{printf "%s ",$0;next;}1' \ + | awk -F":" '{print $2,$3}' \ + | awk '{print $1,$3}' \ + | sed s,\},,g \ + | sed s,],,g \ + | sed s,\",,g +} + +if [ -f podoutput-current ]; then + cp podoutput-current podoutput-lastiteration +fi + +curlcommand > podoutput-current + +declare -A slavescurrent slaveslastiteration + +while read -r name status ; do + slavescurrent["$name"]="$status" +done < <(cat podoutput-current) + +while read -r name status ; do + slaveslastiteration["$name"]=$status +done < <(cat podoutput-lastiteration) + +main () { +for slavename in "${!slavescurrent[@]}"; do + #Slave is online. Mark it down. + if [ "${slavescurrent[$slavename]}" == "false" ]; then + + if [ -f "$slavename"-offline ]; then + echo "removing offline status from $slavename slave was offline for $(cat "$slavename"-offline ) iterations" + rm "$slavename"-offline + fi + + if ! [ -f "$slavename"-online ]; then + echo "1" > "$slavename"-online + elif [ -f "$slavename"-online ]; then + #read and increment slavename + read -r -d $'\x04' var < "$slavename"-online + ((var++)) + echo -n "ONLINE $slavename " + echo "for $var iterations" + echo "$var" > "$slavename"-online + fi + fi + + #went offline since last iteration. + if [ "${slavescurrent[$slavename]}" == "false" ] && [ "${slaveslastiteration[$slavename]}" == "true" ]; then + echo "JUST WENT OFFLINE $slavename " + if [ -f "$slavename"-online ]; then + echo "removing online status from $slavename. slave was online for $(cat "$slavename"-online ) iterations" + rm "$slavename"-online + fi + + fi + + #slave is offline + if [ "${slavescurrent[$slavename]}" == "true" ]; then + if ! [ -f "$slavename"-offline ]; then + echo "1" > "$slavename"-offline + fi + + if [ -f "$slavename"-offline ]; then + #read and increment slavename + read -r -d $'\x04' var < "$slavename"-offline + ((var++)) + echo "$var" > "$slavename"-offline + if [ "$var" -gt "30" ]; then + echo "OFFLINE FOR $var ITERATIONS REMOVE $slavename " + else + echo "OFFLINE $slavename FOR $var ITERATIONS " + fi + fi + fi + +done +} + +main diff --git a/utils/test/testapi/3rd_party/static/testapi-ui/components/pods/pods.html b/utils/test/testapi/3rd_party/static/testapi-ui/components/pods/pods.html index 7ce36ca7c..e366670a9 100644 --- a/utils/test/testapi/3rd_party/static/testapi-ui/components/pods/pods.html +++ b/utils/test/testapi/3rd_party/static/testapi-ui/components/pods/pods.html @@ -1,8 +1,12 @@ -<h3>{{ctrl.pageHeader}}</h3> -<p>{{ctrl.pageParagraph}}</p> +<h3>Pods</h3> +<p>This page is used to create or query pods.<br> + Querying pods is open to everybody.<br> + But only login users are granted the privilege to create the new pod. +</p> + <div class="row" style="margin-bottom:24px;"></div> -<div class="pod-create"> +<div class="pod-create" ng-class="{ 'hidden': ! auth.isAuthenticated }"> <h4>Create</h4> <div class="row"> <div ng-repeat="require in ctrl.createRequirements"> diff --git a/utils/test/testapi/3rd_party/static/testapi-ui/components/pods/podsController.js b/utils/test/testapi/3rd_party/static/testapi-ui/components/pods/podsController.js index 201258619..894fcc152 100644 --- a/utils/test/testapi/3rd_party/static/testapi-ui/components/pods/podsController.js +++ b/utils/test/testapi/3rd_party/static/testapi-ui/components/pods/podsController.js @@ -20,7 +20,7 @@ .controller('PodsController', PodsController); PodsController.$inject = [ - '$scope', '$http', '$filter', '$state', 'testapiApiUrl','raiseAlert' + '$rootScope', '$scope', '$http', '$filter', '$state', 'testapiApiUrl','raiseAlert' ]; /** @@ -52,9 +52,6 @@ ctrl.mode = 'metal'; ctrl.details = ''; - ctrl.pageHeader = 'Pods'; - ctrl.pageParagraph = 'This page is used to create or query pods.'; - /** * This is called when the date filter calendar is opened. It * does some event handling, and sets a scope variable so the UI diff --git a/utils/test/testapi/MANIFEST.in b/utils/test/testapi/MANIFEST.in new file mode 100644 index 000000000..0ba1808ba --- /dev/null +++ b/utils/test/testapi/MANIFEST.in @@ -0,0 +1 @@ +recursive-include 3rd_party
\ No newline at end of file diff --git a/utils/test/testapi/docker/prepare-env.sh b/utils/test/testapi/docker/prepare-env.sh index b14bc2448..92a0c9fd7 100755 --- a/utils/test/testapi/docker/prepare-env.sh +++ b/utils/test/testapi/docker/prepare-env.sh @@ -10,5 +10,5 @@ if [ "$base_url" != "" ]; then sudo crudini --set --existing $FILE api url $base_url/api/v1 sudo crudini --set --existing $FILE ui url $base_url sudo echo "{\"testapiApiUrl\": \"$base_url/api/v1\"}" > \ - /usr/local/lib/python2.7/dist-packages/opnfv_testapi/static/testapi-ui/config.json + /usr/local/share/opnfv_testapi/testapi-ui/config.json fi diff --git a/utils/test/testapi/etc/config.ini b/utils/test/testapi/etc/config.ini index db0e191d1..8d0bde20b 100644 --- a/utils/test/testapi/etc/config.ini +++ b/utils/test/testapi/etc/config.ini @@ -21,6 +21,11 @@ authenticate = False [ui] url = http://localhost:8000 +# this path should be the seem with data_files installation path +static_path = /usr/local/share/opnfv_testapi + [lfid] # Linux Foundation cas URL cas_url = https://identity.linuxfoundation.org/cas/ +#service url used to authenticate to cas +signin_return = api/v1/auth/signin_return diff --git a/utils/test/testapi/opnfv_testapi/cmd/server.py b/utils/test/testapi/opnfv_testapi/cmd/server.py index 50ac049a0..b7d3caa20 100644 --- a/utils/test/testapi/opnfv_testapi/cmd/server.py +++ b/utils/test/testapi/opnfv_testapi/cmd/server.py @@ -38,7 +38,7 @@ from opnfv_testapi.tornado_swagger import swagger def make_app(): swagger.docs(base_url=CONF.ui_url, - static_path=CONF.static_path) + static_path=CONF.ui_static_path) return swagger.Application( url_mappings.mappings, debug=CONF.api_debug, diff --git a/utils/test/testapi/opnfv_testapi/common/check.py b/utils/test/testapi/opnfv_testapi/common/check.py index 009d3d46c..24ba876a9 100644 --- a/utils/test/testapi/opnfv_testapi/common/check.py +++ b/utils/test/testapi/opnfv_testapi/common/check.py @@ -8,49 +8,14 @@ ############################################################################## import functools -import cas from tornado import gen from tornado import web -from opnfv_testapi.common import constants from opnfv_testapi.common import message from opnfv_testapi.common import raises -from opnfv_testapi.common.config import CONF from opnfv_testapi.db import api as dbapi -def login(method): - @web.asynchronous - @gen.coroutine - @functools.wraps(method) - def wrapper(self, *args, **kwargs): - ticket = self.get_query_argument('ticket', default=None) - if ticket: - client = cas.CASClient(version='2', - server_url=CONF.lfid_cas_url, - service_url=CONF.ui_url) - (user, attrs, _) = client.verify_ticket(ticket=ticket) - print 'login user: {}'.format(user) - login_user = { - 'user': user, - 'email': attrs.get('mail'), - 'fullname': attrs.get('field_lf_full_name'), - 'groups': constants.TESTAPI_USERS + attrs.get('group', []) - } - q_user = {'user': user} - db_user = yield dbapi.db_find_one(constants.USER_TABLE, q_user) - if not db_user: - dbapi.db_save(constants.USER_TABLE, login_user) - else: - dbapi.db_update(constants.USER_TABLE, q_user, login_user) - - self.clear_cookie(constants.TESTAPI_ID) - self.set_secure_cookie(constants.TESTAPI_ID, user) - ret = yield gen.coroutine(method)(self, *args, **kwargs) - raise gen.Return(ret) - return wrapper - - def authenticate(method): @web.asynchronous @gen.coroutine diff --git a/utils/test/testapi/opnfv_testapi/common/config.py b/utils/test/testapi/opnfv_testapi/common/config.py index 4cd53c619..140e49283 100644 --- a/utils/test/testapi/opnfv_testapi/common/config.py +++ b/utils/test/testapi/opnfv_testapi/common/config.py @@ -16,14 +16,10 @@ import sys class Config(object): def __init__(self): - self.config_file = None + self.config_file = '/etc/opnfv_testapi/config.ini' self._set_config_file() self._parse() self._parse_per_page() - self.static_path = os.path.join( - os.path.dirname(os.path.normpath(__file__)), - os.pardir, - 'static') def _parse(self): if not os.path.exists(self.config_file): @@ -56,23 +52,12 @@ class Config(object): return value def _set_config_file(self): - if not self._set_sys_config_file(): - self._set_default_config_file() - - def _set_sys_config_file(self): parser = argparse.ArgumentParser() parser.add_argument("-c", "--config-file", dest='config_file', help="Config file location", metavar="FILE") args, _ = parser.parse_known_args(sys.argv) - try: + if hasattr(args, 'config_file') and args.config_file: self.config_file = args.config_file - finally: - return self.config_file is not None - - def _set_default_config_file(self): - is_venv = os.getenv('VIRTUAL_ENV') - self.config_file = os.path.join('/' if not is_venv else is_venv, - 'etc/opnfv_testapi/config.ini') CONF = Config() diff --git a/utils/test/testapi/opnfv_testapi/common/constants.py b/utils/test/testapi/opnfv_testapi/common/constants.py index b37ebb3d6..70c922383 100644 --- a/utils/test/testapi/opnfv_testapi/common/constants.py +++ b/utils/test/testapi/opnfv_testapi/common/constants.py @@ -2,4 +2,3 @@ TESTAPI_ID = 'testapi_id' CSRF_TOKEN = 'csrf_token' ROLE = 'role' TESTAPI_USERS = ['opnfv-testapi-users'] -USER_TABLE = 'users' diff --git a/utils/test/testapi/opnfv_testapi/router/url_mappings.py b/utils/test/testapi/opnfv_testapi/router/url_mappings.py index be6240e70..ce0a3eeb3 100644 --- a/utils/test/testapi/opnfv_testapi/router/url_mappings.py +++ b/utils/test/testapi/opnfv_testapi/router/url_mappings.py @@ -72,10 +72,11 @@ mappings = [ # static path (r'/(.*\.(css|png|gif|js|html|json|map|woff2|woff|ttf))', tornado.web.StaticFileHandler, - {'path': CONF.static_path}), + {'path': CONF.ui_static_path}), (r'/', root.RootHandler), (r'/api/v1/auth/signin', sign.SigninHandler), + (r'/{}'.format(CONF.lfid_signin_return), sign.SigninReturnHandler), (r'/api/v1/auth/signout', sign.SignoutHandler), (r'/api/v1/profile', user.UserHandler), diff --git a/utils/test/testapi/opnfv_testapi/ui/auth/sign.py b/utils/test/testapi/opnfv_testapi/ui/auth/sign.py index 01cd0f7c3..318473ea2 100644 --- a/utils/test/testapi/opnfv_testapi/ui/auth/sign.py +++ b/utils/test/testapi/opnfv_testapi/ui/auth/sign.py @@ -1,22 +1,59 @@ from cas import CASClient +from tornado import gen +from tornado import web from opnfv_testapi.common import constants from opnfv_testapi.common.config import CONF +from opnfv_testapi.db import api as dbapi from opnfv_testapi.resources import handlers -class SigninHandler(handlers.GenericApiHandler): +class SignBaseHandler(handlers.GenericApiHandler): + def __init__(self, application, request, **kwargs): + super(SignBaseHandler, self).__init__(application, request, **kwargs) + self.table = 'users' + self.cas_client = CASClient(version='2', + server_url=CONF.lfid_cas_url, + service_url='{}/{}'.format( + CONF.ui_url, + CONF.lfid_signin_return)) + + +class SigninHandler(SignBaseHandler): + def get(self): + self.redirect(url=(self.cas_client.get_login_url())) + + +class SigninReturnHandler(SignBaseHandler): + + @web.asynchronous + @gen.coroutine def get(self): - client = CASClient(version='2', - server_url=CONF.lfid_cas_url, - service_url=CONF.ui_url) - self.redirect(url=(client.get_login_url())) + ticket = self.get_query_argument('ticket', default=None) + if ticket: + (user, attrs, _) = self.cas_client.verify_ticket(ticket=ticket) + login_user = { + 'user': user, + 'email': attrs.get('mail'), + 'fullname': attrs.get('field_lf_full_name'), + 'groups': constants.TESTAPI_USERS + attrs.get('group', []) + } + q_user = {'user': user} + db_user = yield dbapi.db_find_one(self.table, q_user) + if not db_user: + dbapi.db_save(self.table, login_user) + else: + dbapi.db_update(self.table, q_user, login_user) + + self.clear_cookie(constants.TESTAPI_ID) + self.set_secure_cookie(constants.TESTAPI_ID, user) + + self.redirect(url=CONF.ui_url) -class SignoutHandler(handlers.GenericApiHandler): +class SignoutHandler(SignBaseHandler): def get(self): """Handle signout request.""" self.clear_cookie(constants.TESTAPI_ID) - client = CASClient(version='2', - server_url=CONF.lfid_cas_url) - self.redirect(url=(client.get_logout_url(redirect_url=CONF.ui_url))) + logout_url = self.cas_client.get_logout_url(redirect_url=CONF.ui_url) + self.redirect(url=logout_url) diff --git a/utils/test/testapi/opnfv_testapi/ui/root.py b/utils/test/testapi/opnfv_testapi/ui/root.py index 069ad5e93..286a6b097 100644 --- a/utils/test/testapi/opnfv_testapi/ui/root.py +++ b/utils/test/testapi/opnfv_testapi/ui/root.py @@ -1,12 +1,10 @@ -from opnfv_testapi.common import check from opnfv_testapi.common.config import CONF from opnfv_testapi.resources import handlers class RootHandler(handlers.GenericApiHandler): def get_template_path(self): - return CONF.static_path + return CONF.ui_static_path - @check.login def get(self): self.render('testapi-ui/index.html') diff --git a/utils/test/testapi/setup.cfg b/utils/test/testapi/setup.cfg index ab1ef553e..d9aa6762e 100644 --- a/utils/test/testapi/setup.cfg +++ b/utils/test/testapi/setup.cfg @@ -23,18 +23,10 @@ setup-hooks = [files] packages = opnfv_testapi -package_data = - opnfv_testapi = - static/*.* - static/*/*.* - static/*/*/*.* - static/*/*/*/*.* - static/*/*/*/*/*.* - static/*/*/*/*/*/*.* - static/*/*/*/*/*/*/*.* + data_files = - /etc/opnfv_testapi = - etc/config.ini + /etc/opnfv_testapi = etc/config.ini + /usr/local/share/opnfv_testapi = 3rd_party/static/* [entry_points] console_scripts = @@ -44,4 +36,3 @@ console_scripts = tag_build = tag_date = 0 tag_svn_revision = 0 - diff --git a/utils/test/testapi/setup.py b/utils/test/testapi/setup.py index dd52373fd..f9d95a32d 100644 --- a/utils/test/testapi/setup.py +++ b/utils/test/testapi/setup.py @@ -1,6 +1,3 @@ -import os -import subprocess - import setuptools __author__ = 'serena' @@ -10,11 +7,7 @@ try: except ImportError: pass -dirpath = os.path.dirname(os.path.abspath(__file__)) -subprocess.call(['ln', '-s', - '{}/3rd_party/static'.format(dirpath), - '{}/opnfv_testapi/static'.format(dirpath)]) setuptools.setup( - setup_requires=['pbr==2.0.0'], + setup_requires=['pbr>=2.0.0'], pbr=True) diff --git a/utils/test/testapi/tools/watchdog/docker_watch.sh b/utils/test/testapi/tools/watchdog/docker_watch.sh index d67e4b380..786fc10b9 100644 --- a/utils/test/testapi/tools/watchdog/docker_watch.sh +++ b/utils/test/testapi/tools/watchdog/docker_watch.sh @@ -21,33 +21,19 @@ modules=(testapi reporting) declare -A ports=( ["testapi"]="8082" ["reporting"]="8084") ## Urls to check if the modules are deployed or not ? -#declare -A urls=( ["testapi"]="http://testresults.opnfv.org/test/" \ -# ["reporting"]="http://testresults.opnfv.org/reporting2/reporting/index.html") - -declare -A urls=( ["testapi"]="http://localhost:8082/" \ +declare -A urls=( ["testapi"]="http://testresults.opnfv.org/test/" \ ["reporting"]="http://testresults.opnfv.org/reporting2/reporting/index.html") - ### Functions related to checking. function is_deploying() { - echo -e "Checking job statuses" - for module in "${modules[@]}" - do - if get_status $module; then - exit 0 - fi - done -} - -function get_status() { xml=$(curl -m10 "https://build.opnfv.org/ci/job/${1}-automate-master/lastBuild/api/xml?depth=1") building=$(grep -oPm1 "(?<=<building>)[^<]+" <<< "$xml") if [[ $building == "false" ]] then - return 1 - else return 0 + else + return 1 fi } @@ -78,6 +64,9 @@ function check_modules() { failed_modules=() for module in "${modules[@]}" do + if is_deploying $module; then + continue + fi if ! check_connectivity $module "${urls[$module]}"; then echo -e "$module failed" failed_modules+=($module) @@ -110,27 +99,30 @@ function docker_proxy_fix() { echo $pid if [ ! -z "$pid" ]; then kill $pid - start_containers_fix $module + start_container_fix $module fi done } function start_containers_fix() { - echo "Runnning start_containers_fix" start_modules=("${@}") for module in "${start_modules[@]}" do - echo -e "Starting a container $module" + start_container_fix $module + done +} + +function start_container_fix() { + echo -e "Starting a container $module" + sudo docker stop $module + sudo docker start $module + sleep 5 + if ! check_connectivity $module "${urls[$module]}"; then + echo -e "Starting an old container $module_old" sudo docker stop $module - sudo docker start $module + sudo docker start $module"_old" sleep 5 - if ! check_connectivity $module "${urls[$module]}"; then - echo -e "Starting an old container $module_old" - sudo docker stop $module - sudo docker start $module"_old" - sleep 5 - fi - done + fi } ### Main Flow @@ -141,11 +133,6 @@ echo -e echo -e `date "+%Y-%m-%d %H:%M:%S.%N"` echo -e -if ! is_deploying; then - echo -e "Jenkins Jobs running" - exit -fi - ## If the problem is related to docker daemon if get_docker_status; then @@ -156,16 +143,16 @@ if get_docker_status; then exit fi -## If the problem is related to docker containers +## If the problem is related to docker proxy if ! check_modules; then - start_containers_fix "${failed_modules[@]}" + docker_proxy_fix "${failed_modules[@]}" fi -## If the problem is related to docker proxy +## If any other problem : restart docker if ! check_modules; then - docker_proxy_fix "${failed_modules[@]}" + restart_docker_fix fi ## If nothing works out @@ -175,4 +162,4 @@ if ! check_modules; then fi sudo docker ps -sudo docker images
\ No newline at end of file +sudo docker images |