diff options
author | Thomas Duval <thomas.duval@orange.com> | 2020-06-03 10:06:52 +0200 |
---|---|---|
committer | Thomas Duval <thomas.duval@orange.com> | 2020-06-03 10:06:52 +0200 |
commit | 7bb53c64da2dcf88894bfd31503accdd81498f3d (patch) | |
tree | 4310e12366818af27947b5e2c80cb162da93a4b5 /tools/bin | |
parent | cbea4e360e9bfaa9698cf7c61c83c96a1ba89b8c (diff) |
Update to new version 5.4HEADstable/jermamaster
Signed-off-by: Thomas Duval <thomas.duval@orange.com>
Change-Id: Idcd868133d75928a1ffd74d749ce98503e0555ea
Diffstat (limited to 'tools/bin')
-rw-r--r-- | tools/bin/README.md | 8 | ||||
-rw-r--r-- | tools/bin/api2rst.py | 145 | ||||
-rw-r--r-- | tools/bin/bootstrap.py | 235 | ||||
-rw-r--r-- | tools/bin/build_all.sh | 36 | ||||
-rw-r--r-- | tools/bin/build_all_pip.sh | 16 | ||||
-rw-r--r-- | tools/bin/delete_orchestrator.sh | 61 | ||||
-rw-r--r-- | tools/bin/get_keystone_token.py | 71 | ||||
-rw-r--r-- | tools/bin/moon_lib_upload.sh | 27 | ||||
-rw-r--r-- | tools/bin/set_auth.src | 7 | ||||
-rwxr-xr-x | tools/bin/start.sh | 39 |
10 files changed, 0 insertions, 645 deletions
diff --git a/tools/bin/README.md b/tools/bin/README.md deleted file mode 100644 index 71ff4a44..00000000 --- a/tools/bin/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Automated Tools/Scripts - -## api2pdf -```bash -python3 $MOON_HOME/tools/bin/api2rst.py -pandoc api.rst --toc -o api.pdf -evince api.pdf -``` diff --git a/tools/bin/api2rst.py b/tools/bin/api2rst.py deleted file mode 100644 index 6d407bdf..00000000 --- a/tools/bin/api2rst.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors -# This software is distributed under the terms and conditions of the 'Apache-2.0' -# license which can be found in the file 'LICENSE' in this package distribution -# or at 'http://www.apache.org/licenses/LICENSE-2.0'. - -import os -import sys -import requests -import logging -import time -import json - -os.unsetenv("http_proxy") -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -HOST = "172.18.0.11" -PORT = 38001 -COMPONENT = sys.argv[2] if len(sys.argv) > 1 else "Interface" -FILENAME = sys.argv[2] if len(sys.argv) > 2 else "api.rst" -CURRENT_TIME = time.strftime("%Y/%m/%d %H:%M:%S %Z") -REVISION = time.strftime("%Y%m%d_%H%M%S_%Z") -AUTHOR = "Thomas Duval <thomas.duval@orange.com>" - -logger.info("Writing to {}".format(FILENAME)) - -toc = ( - "generic", - "models", - "policies", - "pdp", - "meta_rules", - "meta_data", - "perimeter", - "data", - "assignments", - "rules", - "authz", -) - - -def get_api_list(): - url = "http://{}:{}/api".format(HOST, PORT) - cnx = requests.get(url) - try: - return cnx.json() - except json.decoder.JSONDecodeError: - logger.error("Error decoding JSON on {}\n{}".format(url, cnx.content)) - sys.exit(1) - - -def analyse_description(desc): - result = "" - if not desc: - return "No description" - for line in desc.splitlines(): - if line.strip().startswith(":"): - if ":request body:" in line: - result += ":request body:\n\n.. code-block:: json\n\n" - result += line.replace(":request body: ", "") + "\n\n" - elif ":return:" in line: - result += ":return:\n\n.. code-block:: json\n\n" - result += line.replace(":return: ", "") + "\n" - else: - result += line.strip() + "\n\n" - else: - result += line + "\n" - return result - - -def filter_and_sort(list_group_api): - results = list() - keys = list_group_api.keys() - for element in toc: - if element in keys: - results.append(element) - for element in keys: - if element not in results: - results.append(element) - return results - - -def main(): - list_group_api = get_api_list() - - _toc = filter_and_sort(list_group_api) - - file_desc = open(FILENAME, "w") - length_of_title = len("Moon {component} API".format(component=COMPONENT)) - file_desc.write(HEADERS.format( - component=COMPONENT, - date=CURRENT_TIME, - revision=REVISION, - title_headers="="*length_of_title, - author=AUTHOR - )) - - for key in _toc: - logger.info(key) - file_desc.write("{}\n".format(key)) - file_desc.write("{}\n\n".format("="*len(key))) - if "description" in list_group_api[key]: - file_desc.write("{}\n\n".format(list_group_api[key]["description"])) - version = "unknown" - logger.debug(list_group_api.keys()) - if "version" in list_group_api[key]: - version = list_group_api[key]["version"] - file_desc.write("Version: {}\n\n".format(version)) - for api in list_group_api[key]: - logger.info("\t{}".format(api)) - if api in ("description", "version"): - continue - file_desc.write("{}\n".format(api)) - file_desc.write("{}\n\n".format("-" * len(api))) - - file_desc.write("{}\n\n".format(list_group_api[key][api]["description"])) - - file_desc.write("URLs are:\n\n") - for _url in list_group_api[key][api]["urls"]: - file_desc.write("* {}\n".format(_url)) - - file_desc.write("\nMethods are:\n\n") - for _method in list_group_api[key][api]["methods"]: - file_desc.write("→ {}\n".format(_method)) - file_desc.write("{}\n\n".format("~"*(len(_method) + 2))) - file_desc.write("{}\n\n".format(analyse_description(list_group_api[key][api]["methods"][_method]))) - -HEADERS = """{title_headers} -Moon {component} API -{title_headers} - -:Info: See <https://git.opnfv.org/cgit/moon/> for code. -:Author: {author} -:Date: {date} -:Revision: $Revision: {revision} $ -:Description: List of the API served by the Moon {component} component - -This document list all of the API connectors served by the Moon {component} component -Here are Moon API with some examples of posted data and returned data. -All requests must be prefixed with the host and port, for example: http://localhost:38001/authz/123456789/123456789/servers/list - -""" - -if __name__ == "__main__": - main() diff --git a/tools/bin/bootstrap.py b/tools/bin/bootstrap.py deleted file mode 100644 index 6f2a5e03..00000000 --- a/tools/bin/bootstrap.py +++ /dev/null @@ -1,235 +0,0 @@ -import os -import sys -import time -import requests -import yaml -import logging -import json -import base64 -import mysql.connector -import re -import subprocess - -logging.basicConfig(level=logging.INFO) -log = logging.getLogger("moon.bootstrap") -requests_log = logging.getLogger("requests.packages.urllib3") -requests_log.setLevel(logging.WARNING) -requests_log.propagate = True - -if len(sys.argv) == 2: - if os.path.isfile(sys.argv[1]): - CONF_FILENAME = sys.argv[1] - CONSUL_HOST = "consul" - else: - CONF_FILENAME = "moon.conf" - CONSUL_HOST = sys.argv[1] - CONSUL_PORT = 8500 -else: - CONSUL_HOST = sys.argv[1] if len(sys.argv) > 1 else "consul" - CONSUL_PORT = sys.argv[2] if len(sys.argv) > 2 else 8500 - CONF_FILENAME = sys.argv[3] if len(sys.argv) > 3 else "moon.conf" -HEADERS = {"content-type": "application/json"} - - -def search_config_file(): - data_config = None - for _file in ( - CONF_FILENAME, - "conf/moon.conf", - "../moon.conf", - "../conf/moon.conf", - "/etc/moon/moon.conf", - ): - try: - data_config = yaml.safe_load(open(_file)) - except FileNotFoundError: - data_config = None - continue - else: - break - if not data_config: - raise Exception("Configuration file not found...") - return data_config - - -def put(key, value): - url = "http://{host}:{port}/v1/kv/{key}".format(host=CONSUL_HOST, port=CONSUL_PORT, key=key) - log.info(url) - req = requests.put( - url, - headers=HEADERS, - json=value - ) - if req.status_code != 200: - raise Exception("Error connecting to Consul ({}, {})".format(req.status_code, req.text)) - - -def get(key): - url = "http://{host}:{port}/v1/kv/{key}".format(host=CONSUL_HOST, port=CONSUL_PORT, key=key) - req = requests.get(url) - data = req.json() - for item in data: - log.info("{} {} -> {}".format( - req.status_code, - item["Key"], - json.loads(base64.b64decode(item["Value"]).decode("utf-8")) - )) - yield json.loads(base64.b64decode(item["Value"]).decode("utf-8")) - - -def start_consul(data_config): - cmd = ["docker", "run", "-d", "--net=moon", "--name=consul", "--hostname=consul", "-p", "8500:8500", "consul"] - output = subprocess.run(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - if output.returncode != 0: - log.info(" ".join(cmd)) - log.info(output.returncode) - log.error(output.stderr) - log.error(output.stdout) - raise Exception("Error starting Consul container!") - while True: - try: - req = requests.get("http://{}:{}/ui".format(CONSUL_HOST, CONSUL_PORT)) - except requests.exceptions.ConnectionError: - log.info("Waiting for Consul ({}:{})".format(CONSUL_HOST, CONSUL_PORT)) - time.sleep(1) - continue - else: - break - # if req.status_code in (302, 200): - # break - # log.info("Waiting for Consul ({}:{})".format(CONSUL_HOST, CONSUL_PORT)) - # time.sleep(1) - log.info("Consul is up") - - req = requests.get("http://{}:{}/v1/kv/database".format(CONSUL_HOST, CONSUL_PORT)) - if req.status_code == 200: - log.info("Consul is already populated") - return - - put("database", data_config["database"]) - put("messenger", data_config["messenger"]) - put("slave", data_config["slave"]) - put("docker", data_config["docker"]) - put("logging", data_config["logging"]) - put("components_port_start", data_config["components"]["port_start"]) - - for _key, _value in data_config["components"].items(): - if type(_value) is dict: - put("components/{}".format(_key), data_config["components"][_key]) - - for _key, _value in data_config["plugins"].items(): - put("plugins/{}".format(_key), data_config["plugins"][_key]) - - for _key, _value in data_config["openstack"].items(): - put("openstack/{}".format(_key), data_config["openstack"][_key]) - - -def start_database(): - cmd = ["docker", "run", "-dti", "--net=moon", "--hostname=db", "--name=db", - "-e", "MYSQL_ROOT_PASSWORD=p4sswOrd1", "-e", "MYSQL_DATABASE=moon", "-e", "MYSQL_USER=moon", - "-e", "MYSQL_PASSWORD=p4sswOrd1", "-p", "3306:3306", "mysql:latest"] - output = subprocess.run(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - if output.returncode != 0: - log.info(cmd) - log.error(output.stderr) - log.error(output.stdout) - raise Exception("Error starting DB container!") - for database in get("database"): - database_url = database['url'] - match = re.search("(?P<proto>^[\\w+]+):\/\/(?P<user>\\w+):(?P<password>.+)@(?P<host>\\w+):*(?P<port>\\d*)", - database_url) - config = match.groupdict() - while True: - try: - conn = mysql.connector.connect( - host=config["host"], - user=config["user"], - password=config["password"], - database="moon" - ) - conn.close() - except mysql.connector.errors.InterfaceError: - log.info("Waiting for Database ({})".format(config["host"])) - time.sleep(1) - continue - else: - log.info("Database is up, populating it...") - output = subprocess.run(["moon_db_manager", "upgrade"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - if output.returncode != 0: - raise Exception("Error populating the database!") - break - - -def start_keystone(): - output = subprocess.run(["docker", "run", "-dti", "--net=moon", "--hostname=keystone", "--name=keystone", - "-e", "DB_HOST=db", "-e", "DB_PASSWORD_ROOT=p4sswOrd1", "-p", "35357:35357", - "-p", "5000:5000", "keystone:mitaka"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - if output.returncode != 0: - raise Exception("Error starting Keystone container!") - # TODO: Keystone answers request too quickly - # even if it is not fully loaded - # we must test if a token retrieval is possible or not - # to see if Keystone is truly up and running - for config in get("openstack/keystone"): - while True: - try: - time.sleep(1) - req = requests.get(config["url"]) - except requests.exceptions.ConnectionError: - log.info("Waiting for Keystone ({})".format(config["url"])) - time.sleep(1) - continue - else: - log.info("Keystone is up") - break - - -def start_moon(data_config): - cmds = [ - # ["docker", "run", "-dti", "--net=moon", "--name=wrapper", "--hostname=wrapper", "-p", - # "{0}:{0}".format(data_config['components']['wrapper']['port']), - # data_config['components']['wrapper']['container']], - ["docker", "run", "-dti", "--net=moon", "--name=manager", - "--hostname=manager", "-p", - "{0}:{0}".format(data_config['components']['manager']['port']), - data_config['components']['manager']['container']], - ["docker", "run", "-dti", "--net=moon", "--name=interface", - "--hostname=interface", "-p", - "{0}:{0}".format(data_config['components']['interface']['port']), - data_config['components']['interface']['container']], - ] - for cmd in cmds: - log.warning("Start {}".format(cmd[-1])) - # answer = input() - # if answer.lower() in ("y", "yes", "o", "oui"): - output = subprocess.run(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - time.sleep(3) - if output.returncode != 0: - log.info(" ".join(cmd)) - log.info(output.returncode) - log.error(output.stderr) - log.error(output.stdout) - raise Exception("Error starting {} container!".format(cmd[-1])) - subprocess.run(["docker", "ps"]) - - -def main(): - data_config = search_config_file() - subprocess.run(["docker", "rm", "-f", "consul", "db", "manager", "wrapper", "interface", "authz*", "keystone"]) - start_consul(data_config) - start_database() - start_keystone() - start_moon(data_config) - -main() - diff --git a/tools/bin/build_all.sh b/tools/bin/build_all.sh deleted file mode 100644 index 5bbf6a19..00000000 --- a/tools/bin/build_all.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -VERSION=v4.1 -export DOCKER_HOST=tcp://172.88.88.1:2376 - - -mkdir $MOON_HOME/moon_orchestrator/dist 2>/dev/null - -echo Building Moon_Orchestrator -cd $MOON_HOME/moon_orchestrator -docker build -t wukongsun/moon_orchestrator:${VERSION} . - -echo Building Moon_Interface -cd $MOON_HOME/moon_interface -docker build -t wukongsun/moon_interface:${VERSION} . - -echo Building Moon_Security_Router -cd $MOON_HOME/moon_secrouter -docker build -t wukongsun/moon_router:${VERSION} . - -echo Building Moon_Manager -cd $MOON_HOME/moon_manager -docker build -t wukongsun/moon_manager:${VERSION} . - -echo Building Moon_Authz -cd $MOON_HOME/moon_authz -docker build -t wukongsun/moon_authz:${VERSION} . - - -echo Building Moon_DB -cd $MOON_HOME/moon_db -python3 setup.py sdist bdist_wheel > /tmp/moon_db.log - -echo Building Moon_Utilities -cd $MOON_HOME/moon_utilities -python3 setup.py sdist bdist_wheel > /tmp/moon_utilities.log diff --git a/tools/bin/build_all_pip.sh b/tools/bin/build_all_pip.sh deleted file mode 100644 index 2b415bf0..00000000 --- a/tools/bin/build_all_pip.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - - -echo Building Moon_DB -cd $MOON_HOME/moon_db -python3 setup.py sdist bdist_wheel> /tmp/moon_db.log - - -echo Building Moon_Utilities -cd $MOON_HOME/moon_utilities -python3 setup.py sdist bdist_wheel> /tmp/moon_utilities.log - - -echo Building Moon_Orchestrator -cd $MOON_HOME/moon_orchestrator -python3 setup.py sdist bdist_wheel> /tmp/moon_orchestrator.log
\ No newline at end of file diff --git a/tools/bin/delete_orchestrator.sh b/tools/bin/delete_orchestrator.sh deleted file mode 100644 index 4d9d7c98..00000000 --- a/tools/bin/delete_orchestrator.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -set +x - -kubectl delete -n moon -f tools/moon_kubernetes/templates/moon_orchestrator.yaml -for i in $(kubectl get deployments -n moon | grep wrapper | cut -d " " -f 1 | xargs); do - echo deleting $i - kubectl delete deployments/$i -n moon; -done -for i in $(kubectl get deployments -n moon | grep pipeline | cut -d " " -f 1 | xargs); do - echo deleting $i - kubectl delete deployments/$i -n moon; -done -for i in $(kubectl get services -n moon | grep wrapper | cut -d " " -f 1 | xargs); do - echo deleting $i - kubectl delete services/$i -n moon; -done -for i in $(kubectl get services -n moon | grep pipeline | cut -d " " -f 1 | xargs); do - echo deleting $i - kubectl delete services/$i -n moon; -done - -if [ "$1" = "build" ]; then - - DOCKER_ARGS="" - - cd moon_manager - docker build -t wukongsun/moon_manager:v4.3.1 . ${DOCKER_ARGS} - if [ "$2" = "push" ]; then - docker push wukongsun/moon_manager:v4.3.1 - fi - cd - - - cd moon_orchestrator - docker build -t wukongsun/moon_orchestrator:v4.3 . ${DOCKER_ARGS} - if [ "$2" = "push" ]; then - docker push wukongsun/moon_orchestrator:v4.3 - fi - cd - - - cd moon_interface - docker build -t wukongsun/moon_interface:v4.3 . ${DOCKER_ARGS} - if [ "$2" = "push" ]; then - docker push wukongsun/moon_interface:v4.3 - fi - cd - - - cd moon_authz - docker build -t wukongsun/moon_authz:v4.3 . ${DOCKER_ARGS} - if [ "$2" = "push" ]; then - docker push wukongsun/moon_authz:v4.3 - fi - cd - - - cd moon_wrapper - docker build -t wukongsun/moon_wrapper:v4.3 . ${DOCKER_ARGS} - if [ "$2" = "push" ]; then - docker push wukongsun/moon_wrapper:v4.3 - fi - cd - -fi diff --git a/tools/bin/get_keystone_token.py b/tools/bin/get_keystone_token.py deleted file mode 100644 index 1856aab8..00000000 --- a/tools/bin/get_keystone_token.py +++ /dev/null @@ -1,71 +0,0 @@ -import requests -from oslo_config import cfg -from oslo_log import log as logging -from python_moonutilities import exceptions - -CONF = cfg.CONF -LOG = logging.getLogger(__name__) - - -def login(user=None, password=None, domain=None, project=None, url=None): - print("""Configuration: - user: {user} - domain: {domain} - project: {project} - url: {url}""".format( - user=CONF.keystone.user, - domain=CONF.keystone.domain, - project=CONF.keystone.project, - url=CONF.keystone.url, - )) - if not user: - user = CONF.keystone.user - if not password: - password = CONF.keystone.password - if not domain: - domain = CONF.keystone.domain - if not project: - project = CONF.keystone.project - if not url: - url = CONF.keystone.url - headers = { - "Content-Type": "application/json" - } - data_auth = { - "auth": { - "identity": { - "methods": [ - "password" - ], - "password": { - "user": { - "domain": { - "id": domain - }, - "name": user, - "password": password - } - } - }, - "scope": { - "project": { - "domain": { - "id": domain - }, - "name": project - } - } - } - } - - req = requests.post("{}/auth/tokens".format(url), - json=data_auth, headers=headers, - verify=False) - - if req.status_code not in (200, 201): - LOG.error(req.text) - raise exceptions.KeystoneError - headers['X-Auth-Token'] = req.headers['X-Subject-Token'] - return headers - -print(login()['X-Auth-Token']) diff --git a/tools/bin/moon_lib_upload.sh b/tools/bin/moon_lib_upload.sh deleted file mode 100644 index d2dc2a3f..00000000 --- a/tools/bin/moon_lib_upload.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -# usage: moon_update.sh <GPG_ID> - -COMPONENT=$(basename $(pwd)) -GPG_ID=$1 - -if [ -f setup.py ]; then - echo -else - echo "Not a python package" - exit 1 -fi - -VERSION=${COMPONENT}-$(grep __version__ ${COMPONENT}/__init__.py | cut -d "\"" -f 2) - -python3 setup.py sdist bdist_wheel - -echo $COMPONENT -echo $VERSION - -# Instead of "A0A96E75", use your own GPG ID -rm dist/*.asc 2>/dev/null -gpg --detach-sign -u "${GPG_ID}" -a dist/${VERSION}-py3-none-any.whl -gpg --detach-sign -u "${GPG_ID}" -a dist/${VERSION/_/-}.tar.gz -twine upload dist/${VERSION}-py3-none-any.whl dist/${VERSION}-py3-none-any.whl.asc -twine upload dist/${VERSION/_/-}.tar.gz dist/${VERSION/_/-}.tar.gz.asc diff --git a/tools/bin/set_auth.src b/tools/bin/set_auth.src deleted file mode 100644 index d955e30b..00000000 --- a/tools/bin/set_auth.src +++ /dev/null @@ -1,7 +0,0 @@ -export OS_USERNAME=admin -export OS_PASSWORD=p4ssw0rd -export OS_REGION_NAME=Orange -export OS_TENANT_NAME=admin -export OS_AUTH_URL=http://keystone:5000/v3 -export OS_DOMAIN_NAME=Default -export MOON_URL=http://172.18.0.11:38001 diff --git a/tools/bin/start.sh b/tools/bin/start.sh deleted file mode 100755 index e95ac393..00000000 --- a/tools/bin/start.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -VERSION=4.1 -export DOCKER_HOST=tcp://172.88.88.1:2376 - -echo -e "\033[31mDeleting previous dockers\033[m" -docker rm -f $(docker ps -a | grep moon | cut -d " " -f 1) 2>/dev/null -docker rm -f messenger db keystone consul 2>/dev/null - -echo -e "\033[32mStarting Messenger\033[m" -docker run -dti --net=moon --hostname messenger --name messenger -e RABBITMQ_DEFAULT_USER=moon -e RABBITMQ_DEFAULT_PASS=p4sswOrd1 -e RABBITMQ_NODENAME=rabbit@messenger -e RABBITMQ_DEFAULT_VHOST=moon -e RABBITMQ_HIPE_COMPILE=1 -p 5671:5671 -p 5672:5672 -p 8080:15672 rabbitmq:3-management - -echo -e "\033[32mStarting DB manager\033[m" -docker run -dti --net=moon --hostname db --name db -e MYSQL_ROOT_PASSWORD=p4sswOrd1 -e MYSQL_DATABASE=moon -e MYSQL_USER=moon -e MYSQL_PASSWORD=p4sswOrd1 -p 3306:3306 mysql:latest - -docker run -d --net=moon --name=consul --hostname=consul -p 8500:8500 consul - -echo "waiting for Database (it may takes time)..." -echo -e "\033[35m" -sed '/ready for connections/q' <(docker logs db -f) -echo -e "\033[m" - -echo "waiting for Messenger (it may takes time)..." -echo -e "\033[35m" -sed '/Server startup complete;/q' <(docker logs messenger -f) -echo -e "\033[m" - -docker run -dti --net moon --hostname keystone --name keystone -e DB_HOST=db -e DB_PASSWORD_ROOT=p4sswOrd1 -p 35357:35357 -p 5000:5000 keystone:mitaka - -echo -e "\033[32mConfiguring Moon platform\033[m" -sudo pip install moon_db -moon_db_manager upgrade - -cd ${MOON_HOME}/moon_orchestrator -python3 populate_consul.py - -echo -e "\033[32mStarting Moon platform\033[m" - -docker container run -dti --net moon --hostname orchestrator --name orchestrator wukongsun/moon_orchestrator:${VERSION} |