diff options
Diffstat (limited to 'old/tools/bin')
-rw-r--r-- | old/tools/bin/README.md | 8 | ||||
-rw-r--r-- | old/tools/bin/api2rst.py | 145 | ||||
-rw-r--r-- | old/tools/bin/bootstrap.py | 235 | ||||
-rw-r--r-- | old/tools/bin/build_all.sh | 36 | ||||
-rw-r--r-- | old/tools/bin/build_all_pip.sh | 16 | ||||
-rw-r--r-- | old/tools/bin/delete_orchestrator.sh | 61 | ||||
-rw-r--r-- | old/tools/bin/get_keystone_token.py | 71 | ||||
-rw-r--r-- | old/tools/bin/moon_lib_upload.sh | 27 | ||||
-rw-r--r-- | old/tools/bin/set_auth.src | 7 | ||||
-rwxr-xr-x | old/tools/bin/start.sh | 39 |
10 files changed, 645 insertions, 0 deletions
diff --git a/old/tools/bin/README.md b/old/tools/bin/README.md new file mode 100644 index 00000000..71ff4a44 --- /dev/null +++ b/old/tools/bin/README.md @@ -0,0 +1,8 @@ +# Automated Tools/Scripts + +## api2pdf +```bash +python3 $MOON_HOME/tools/bin/api2rst.py +pandoc api.rst --toc -o api.pdf +evince api.pdf +``` diff --git a/old/tools/bin/api2rst.py b/old/tools/bin/api2rst.py new file mode 100644 index 00000000..6d407bdf --- /dev/null +++ b/old/tools/bin/api2rst.py @@ -0,0 +1,145 @@ +# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors +# This software is distributed under the terms and conditions of the 'Apache-2.0' +# license which can be found in the file 'LICENSE' in this package distribution +# or at 'http://www.apache.org/licenses/LICENSE-2.0'. + +import os +import sys +import requests +import logging +import time +import json + +os.unsetenv("http_proxy") +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +HOST = "172.18.0.11" +PORT = 38001 +COMPONENT = sys.argv[2] if len(sys.argv) > 1 else "Interface" +FILENAME = sys.argv[2] if len(sys.argv) > 2 else "api.rst" +CURRENT_TIME = time.strftime("%Y/%m/%d %H:%M:%S %Z") +REVISION = time.strftime("%Y%m%d_%H%M%S_%Z") +AUTHOR = "Thomas Duval <thomas.duval@orange.com>" + +logger.info("Writing to {}".format(FILENAME)) + +toc = ( + "generic", + "models", + "policies", + "pdp", + "meta_rules", + "meta_data", + "perimeter", + "data", + "assignments", + "rules", + "authz", +) + + +def get_api_list(): + url = "http://{}:{}/api".format(HOST, PORT) + cnx = requests.get(url) + try: + return cnx.json() + except json.decoder.JSONDecodeError: + logger.error("Error decoding JSON on {}\n{}".format(url, cnx.content)) + sys.exit(1) + + +def analyse_description(desc): + result = "" + if not desc: + return "No description" + for line in desc.splitlines(): + if line.strip().startswith(":"): + if ":request body:" in line: + result += ":request body:\n\n.. code-block:: json\n\n" + result += line.replace(":request body: ", "") + "\n\n" + elif ":return:" in line: + result += ":return:\n\n.. code-block:: json\n\n" + result += line.replace(":return: ", "") + "\n" + else: + result += line.strip() + "\n\n" + else: + result += line + "\n" + return result + + +def filter_and_sort(list_group_api): + results = list() + keys = list_group_api.keys() + for element in toc: + if element in keys: + results.append(element) + for element in keys: + if element not in results: + results.append(element) + return results + + +def main(): + list_group_api = get_api_list() + + _toc = filter_and_sort(list_group_api) + + file_desc = open(FILENAME, "w") + length_of_title = len("Moon {component} API".format(component=COMPONENT)) + file_desc.write(HEADERS.format( + component=COMPONENT, + date=CURRENT_TIME, + revision=REVISION, + title_headers="="*length_of_title, + author=AUTHOR + )) + + for key in _toc: + logger.info(key) + file_desc.write("{}\n".format(key)) + file_desc.write("{}\n\n".format("="*len(key))) + if "description" in list_group_api[key]: + file_desc.write("{}\n\n".format(list_group_api[key]["description"])) + version = "unknown" + logger.debug(list_group_api.keys()) + if "version" in list_group_api[key]: + version = list_group_api[key]["version"] + file_desc.write("Version: {}\n\n".format(version)) + for api in list_group_api[key]: + logger.info("\t{}".format(api)) + if api in ("description", "version"): + continue + file_desc.write("{}\n".format(api)) + file_desc.write("{}\n\n".format("-" * len(api))) + + file_desc.write("{}\n\n".format(list_group_api[key][api]["description"])) + + file_desc.write("URLs are:\n\n") + for _url in list_group_api[key][api]["urls"]: + file_desc.write("* {}\n".format(_url)) + + file_desc.write("\nMethods are:\n\n") + for _method in list_group_api[key][api]["methods"]: + file_desc.write("→ {}\n".format(_method)) + file_desc.write("{}\n\n".format("~"*(len(_method) + 2))) + file_desc.write("{}\n\n".format(analyse_description(list_group_api[key][api]["methods"][_method]))) + +HEADERS = """{title_headers} +Moon {component} API +{title_headers} + +:Info: See <https://git.opnfv.org/cgit/moon/> for code. +:Author: {author} +:Date: {date} +:Revision: $Revision: {revision} $ +:Description: List of the API served by the Moon {component} component + +This document list all of the API connectors served by the Moon {component} component +Here are Moon API with some examples of posted data and returned data. +All requests must be prefixed with the host and port, for example: http://localhost:38001/authz/123456789/123456789/servers/list + +""" + +if __name__ == "__main__": + main() diff --git a/old/tools/bin/bootstrap.py b/old/tools/bin/bootstrap.py new file mode 100644 index 00000000..6f2a5e03 --- /dev/null +++ b/old/tools/bin/bootstrap.py @@ -0,0 +1,235 @@ +import os +import sys +import time +import requests +import yaml +import logging +import json +import base64 +import mysql.connector +import re +import subprocess + +logging.basicConfig(level=logging.INFO) +log = logging.getLogger("moon.bootstrap") +requests_log = logging.getLogger("requests.packages.urllib3") +requests_log.setLevel(logging.WARNING) +requests_log.propagate = True + +if len(sys.argv) == 2: + if os.path.isfile(sys.argv[1]): + CONF_FILENAME = sys.argv[1] + CONSUL_HOST = "consul" + else: + CONF_FILENAME = "moon.conf" + CONSUL_HOST = sys.argv[1] + CONSUL_PORT = 8500 +else: + CONSUL_HOST = sys.argv[1] if len(sys.argv) > 1 else "consul" + CONSUL_PORT = sys.argv[2] if len(sys.argv) > 2 else 8500 + CONF_FILENAME = sys.argv[3] if len(sys.argv) > 3 else "moon.conf" +HEADERS = {"content-type": "application/json"} + + +def search_config_file(): + data_config = None + for _file in ( + CONF_FILENAME, + "conf/moon.conf", + "../moon.conf", + "../conf/moon.conf", + "/etc/moon/moon.conf", + ): + try: + data_config = yaml.safe_load(open(_file)) + except FileNotFoundError: + data_config = None + continue + else: + break + if not data_config: + raise Exception("Configuration file not found...") + return data_config + + +def put(key, value): + url = "http://{host}:{port}/v1/kv/{key}".format(host=CONSUL_HOST, port=CONSUL_PORT, key=key) + log.info(url) + req = requests.put( + url, + headers=HEADERS, + json=value + ) + if req.status_code != 200: + raise Exception("Error connecting to Consul ({}, {})".format(req.status_code, req.text)) + + +def get(key): + url = "http://{host}:{port}/v1/kv/{key}".format(host=CONSUL_HOST, port=CONSUL_PORT, key=key) + req = requests.get(url) + data = req.json() + for item in data: + log.info("{} {} -> {}".format( + req.status_code, + item["Key"], + json.loads(base64.b64decode(item["Value"]).decode("utf-8")) + )) + yield json.loads(base64.b64decode(item["Value"]).decode("utf-8")) + + +def start_consul(data_config): + cmd = ["docker", "run", "-d", "--net=moon", "--name=consul", "--hostname=consul", "-p", "8500:8500", "consul"] + output = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if output.returncode != 0: + log.info(" ".join(cmd)) + log.info(output.returncode) + log.error(output.stderr) + log.error(output.stdout) + raise Exception("Error starting Consul container!") + while True: + try: + req = requests.get("http://{}:{}/ui".format(CONSUL_HOST, CONSUL_PORT)) + except requests.exceptions.ConnectionError: + log.info("Waiting for Consul ({}:{})".format(CONSUL_HOST, CONSUL_PORT)) + time.sleep(1) + continue + else: + break + # if req.status_code in (302, 200): + # break + # log.info("Waiting for Consul ({}:{})".format(CONSUL_HOST, CONSUL_PORT)) + # time.sleep(1) + log.info("Consul is up") + + req = requests.get("http://{}:{}/v1/kv/database".format(CONSUL_HOST, CONSUL_PORT)) + if req.status_code == 200: + log.info("Consul is already populated") + return + + put("database", data_config["database"]) + put("messenger", data_config["messenger"]) + put("slave", data_config["slave"]) + put("docker", data_config["docker"]) + put("logging", data_config["logging"]) + put("components_port_start", data_config["components"]["port_start"]) + + for _key, _value in data_config["components"].items(): + if type(_value) is dict: + put("components/{}".format(_key), data_config["components"][_key]) + + for _key, _value in data_config["plugins"].items(): + put("plugins/{}".format(_key), data_config["plugins"][_key]) + + for _key, _value in data_config["openstack"].items(): + put("openstack/{}".format(_key), data_config["openstack"][_key]) + + +def start_database(): + cmd = ["docker", "run", "-dti", "--net=moon", "--hostname=db", "--name=db", + "-e", "MYSQL_ROOT_PASSWORD=p4sswOrd1", "-e", "MYSQL_DATABASE=moon", "-e", "MYSQL_USER=moon", + "-e", "MYSQL_PASSWORD=p4sswOrd1", "-p", "3306:3306", "mysql:latest"] + output = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if output.returncode != 0: + log.info(cmd) + log.error(output.stderr) + log.error(output.stdout) + raise Exception("Error starting DB container!") + for database in get("database"): + database_url = database['url'] + match = re.search("(?P<proto>^[\\w+]+):\/\/(?P<user>\\w+):(?P<password>.+)@(?P<host>\\w+):*(?P<port>\\d*)", + database_url) + config = match.groupdict() + while True: + try: + conn = mysql.connector.connect( + host=config["host"], + user=config["user"], + password=config["password"], + database="moon" + ) + conn.close() + except mysql.connector.errors.InterfaceError: + log.info("Waiting for Database ({})".format(config["host"])) + time.sleep(1) + continue + else: + log.info("Database is up, populating it...") + output = subprocess.run(["moon_db_manager", "upgrade"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if output.returncode != 0: + raise Exception("Error populating the database!") + break + + +def start_keystone(): + output = subprocess.run(["docker", "run", "-dti", "--net=moon", "--hostname=keystone", "--name=keystone", + "-e", "DB_HOST=db", "-e", "DB_PASSWORD_ROOT=p4sswOrd1", "-p", "35357:35357", + "-p", "5000:5000", "keystone:mitaka"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if output.returncode != 0: + raise Exception("Error starting Keystone container!") + # TODO: Keystone answers request too quickly + # even if it is not fully loaded + # we must test if a token retrieval is possible or not + # to see if Keystone is truly up and running + for config in get("openstack/keystone"): + while True: + try: + time.sleep(1) + req = requests.get(config["url"]) + except requests.exceptions.ConnectionError: + log.info("Waiting for Keystone ({})".format(config["url"])) + time.sleep(1) + continue + else: + log.info("Keystone is up") + break + + +def start_moon(data_config): + cmds = [ + # ["docker", "run", "-dti", "--net=moon", "--name=wrapper", "--hostname=wrapper", "-p", + # "{0}:{0}".format(data_config['components']['wrapper']['port']), + # data_config['components']['wrapper']['container']], + ["docker", "run", "-dti", "--net=moon", "--name=manager", + "--hostname=manager", "-p", + "{0}:{0}".format(data_config['components']['manager']['port']), + data_config['components']['manager']['container']], + ["docker", "run", "-dti", "--net=moon", "--name=interface", + "--hostname=interface", "-p", + "{0}:{0}".format(data_config['components']['interface']['port']), + data_config['components']['interface']['container']], + ] + for cmd in cmds: + log.warning("Start {}".format(cmd[-1])) + # answer = input() + # if answer.lower() in ("y", "yes", "o", "oui"): + output = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + time.sleep(3) + if output.returncode != 0: + log.info(" ".join(cmd)) + log.info(output.returncode) + log.error(output.stderr) + log.error(output.stdout) + raise Exception("Error starting {} container!".format(cmd[-1])) + subprocess.run(["docker", "ps"]) + + +def main(): + data_config = search_config_file() + subprocess.run(["docker", "rm", "-f", "consul", "db", "manager", "wrapper", "interface", "authz*", "keystone"]) + start_consul(data_config) + start_database() + start_keystone() + start_moon(data_config) + +main() + diff --git a/old/tools/bin/build_all.sh b/old/tools/bin/build_all.sh new file mode 100644 index 00000000..5bbf6a19 --- /dev/null +++ b/old/tools/bin/build_all.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +VERSION=v4.1 +export DOCKER_HOST=tcp://172.88.88.1:2376 + + +mkdir $MOON_HOME/moon_orchestrator/dist 2>/dev/null + +echo Building Moon_Orchestrator +cd $MOON_HOME/moon_orchestrator +docker build -t wukongsun/moon_orchestrator:${VERSION} . + +echo Building Moon_Interface +cd $MOON_HOME/moon_interface +docker build -t wukongsun/moon_interface:${VERSION} . + +echo Building Moon_Security_Router +cd $MOON_HOME/moon_secrouter +docker build -t wukongsun/moon_router:${VERSION} . + +echo Building Moon_Manager +cd $MOON_HOME/moon_manager +docker build -t wukongsun/moon_manager:${VERSION} . + +echo Building Moon_Authz +cd $MOON_HOME/moon_authz +docker build -t wukongsun/moon_authz:${VERSION} . + + +echo Building Moon_DB +cd $MOON_HOME/moon_db +python3 setup.py sdist bdist_wheel > /tmp/moon_db.log + +echo Building Moon_Utilities +cd $MOON_HOME/moon_utilities +python3 setup.py sdist bdist_wheel > /tmp/moon_utilities.log diff --git a/old/tools/bin/build_all_pip.sh b/old/tools/bin/build_all_pip.sh new file mode 100644 index 00000000..2b415bf0 --- /dev/null +++ b/old/tools/bin/build_all_pip.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + + +echo Building Moon_DB +cd $MOON_HOME/moon_db +python3 setup.py sdist bdist_wheel> /tmp/moon_db.log + + +echo Building Moon_Utilities +cd $MOON_HOME/moon_utilities +python3 setup.py sdist bdist_wheel> /tmp/moon_utilities.log + + +echo Building Moon_Orchestrator +cd $MOON_HOME/moon_orchestrator +python3 setup.py sdist bdist_wheel> /tmp/moon_orchestrator.log
\ No newline at end of file diff --git a/old/tools/bin/delete_orchestrator.sh b/old/tools/bin/delete_orchestrator.sh new file mode 100644 index 00000000..4d9d7c98 --- /dev/null +++ b/old/tools/bin/delete_orchestrator.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +set +x + +kubectl delete -n moon -f tools/moon_kubernetes/templates/moon_orchestrator.yaml +for i in $(kubectl get deployments -n moon | grep wrapper | cut -d " " -f 1 | xargs); do + echo deleting $i + kubectl delete deployments/$i -n moon; +done +for i in $(kubectl get deployments -n moon | grep pipeline | cut -d " " -f 1 | xargs); do + echo deleting $i + kubectl delete deployments/$i -n moon; +done +for i in $(kubectl get services -n moon | grep wrapper | cut -d " " -f 1 | xargs); do + echo deleting $i + kubectl delete services/$i -n moon; +done +for i in $(kubectl get services -n moon | grep pipeline | cut -d " " -f 1 | xargs); do + echo deleting $i + kubectl delete services/$i -n moon; +done + +if [ "$1" = "build" ]; then + + DOCKER_ARGS="" + + cd moon_manager + docker build -t wukongsun/moon_manager:v4.3.1 . ${DOCKER_ARGS} + if [ "$2" = "push" ]; then + docker push wukongsun/moon_manager:v4.3.1 + fi + cd - + + cd moon_orchestrator + docker build -t wukongsun/moon_orchestrator:v4.3 . ${DOCKER_ARGS} + if [ "$2" = "push" ]; then + docker push wukongsun/moon_orchestrator:v4.3 + fi + cd - + + cd moon_interface + docker build -t wukongsun/moon_interface:v4.3 . ${DOCKER_ARGS} + if [ "$2" = "push" ]; then + docker push wukongsun/moon_interface:v4.3 + fi + cd - + + cd moon_authz + docker build -t wukongsun/moon_authz:v4.3 . ${DOCKER_ARGS} + if [ "$2" = "push" ]; then + docker push wukongsun/moon_authz:v4.3 + fi + cd - + + cd moon_wrapper + docker build -t wukongsun/moon_wrapper:v4.3 . ${DOCKER_ARGS} + if [ "$2" = "push" ]; then + docker push wukongsun/moon_wrapper:v4.3 + fi + cd - +fi diff --git a/old/tools/bin/get_keystone_token.py b/old/tools/bin/get_keystone_token.py new file mode 100644 index 00000000..1856aab8 --- /dev/null +++ b/old/tools/bin/get_keystone_token.py @@ -0,0 +1,71 @@ +import requests +from oslo_config import cfg +from oslo_log import log as logging +from python_moonutilities import exceptions + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + + +def login(user=None, password=None, domain=None, project=None, url=None): + print("""Configuration: + user: {user} + domain: {domain} + project: {project} + url: {url}""".format( + user=CONF.keystone.user, + domain=CONF.keystone.domain, + project=CONF.keystone.project, + url=CONF.keystone.url, + )) + if not user: + user = CONF.keystone.user + if not password: + password = CONF.keystone.password + if not domain: + domain = CONF.keystone.domain + if not project: + project = CONF.keystone.project + if not url: + url = CONF.keystone.url + headers = { + "Content-Type": "application/json" + } + data_auth = { + "auth": { + "identity": { + "methods": [ + "password" + ], + "password": { + "user": { + "domain": { + "id": domain + }, + "name": user, + "password": password + } + } + }, + "scope": { + "project": { + "domain": { + "id": domain + }, + "name": project + } + } + } + } + + req = requests.post("{}/auth/tokens".format(url), + json=data_auth, headers=headers, + verify=False) + + if req.status_code not in (200, 201): + LOG.error(req.text) + raise exceptions.KeystoneError + headers['X-Auth-Token'] = req.headers['X-Subject-Token'] + return headers + +print(login()['X-Auth-Token']) diff --git a/old/tools/bin/moon_lib_upload.sh b/old/tools/bin/moon_lib_upload.sh new file mode 100644 index 00000000..d2dc2a3f --- /dev/null +++ b/old/tools/bin/moon_lib_upload.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# usage: moon_update.sh <GPG_ID> + +COMPONENT=$(basename $(pwd)) +GPG_ID=$1 + +if [ -f setup.py ]; then + echo +else + echo "Not a python package" + exit 1 +fi + +VERSION=${COMPONENT}-$(grep __version__ ${COMPONENT}/__init__.py | cut -d "\"" -f 2) + +python3 setup.py sdist bdist_wheel + +echo $COMPONENT +echo $VERSION + +# Instead of "A0A96E75", use your own GPG ID +rm dist/*.asc 2>/dev/null +gpg --detach-sign -u "${GPG_ID}" -a dist/${VERSION}-py3-none-any.whl +gpg --detach-sign -u "${GPG_ID}" -a dist/${VERSION/_/-}.tar.gz +twine upload dist/${VERSION}-py3-none-any.whl dist/${VERSION}-py3-none-any.whl.asc +twine upload dist/${VERSION/_/-}.tar.gz dist/${VERSION/_/-}.tar.gz.asc diff --git a/old/tools/bin/set_auth.src b/old/tools/bin/set_auth.src new file mode 100644 index 00000000..d955e30b --- /dev/null +++ b/old/tools/bin/set_auth.src @@ -0,0 +1,7 @@ +export OS_USERNAME=admin +export OS_PASSWORD=p4ssw0rd +export OS_REGION_NAME=Orange +export OS_TENANT_NAME=admin +export OS_AUTH_URL=http://keystone:5000/v3 +export OS_DOMAIN_NAME=Default +export MOON_URL=http://172.18.0.11:38001 diff --git a/old/tools/bin/start.sh b/old/tools/bin/start.sh new file mode 100755 index 00000000..e95ac393 --- /dev/null +++ b/old/tools/bin/start.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +VERSION=4.1 +export DOCKER_HOST=tcp://172.88.88.1:2376 + +echo -e "\033[31mDeleting previous dockers\033[m" +docker rm -f $(docker ps -a | grep moon | cut -d " " -f 1) 2>/dev/null +docker rm -f messenger db keystone consul 2>/dev/null + +echo -e "\033[32mStarting Messenger\033[m" +docker run -dti --net=moon --hostname messenger --name messenger -e RABBITMQ_DEFAULT_USER=moon -e RABBITMQ_DEFAULT_PASS=p4sswOrd1 -e RABBITMQ_NODENAME=rabbit@messenger -e RABBITMQ_DEFAULT_VHOST=moon -e RABBITMQ_HIPE_COMPILE=1 -p 5671:5671 -p 5672:5672 -p 8080:15672 rabbitmq:3-management + +echo -e "\033[32mStarting DB manager\033[m" +docker run -dti --net=moon --hostname db --name db -e MYSQL_ROOT_PASSWORD=p4sswOrd1 -e MYSQL_DATABASE=moon -e MYSQL_USER=moon -e MYSQL_PASSWORD=p4sswOrd1 -p 3306:3306 mysql:latest + +docker run -d --net=moon --name=consul --hostname=consul -p 8500:8500 consul + +echo "waiting for Database (it may takes time)..." +echo -e "\033[35m" +sed '/ready for connections/q' <(docker logs db -f) +echo -e "\033[m" + +echo "waiting for Messenger (it may takes time)..." +echo -e "\033[35m" +sed '/Server startup complete;/q' <(docker logs messenger -f) +echo -e "\033[m" + +docker run -dti --net moon --hostname keystone --name keystone -e DB_HOST=db -e DB_PASSWORD_ROOT=p4sswOrd1 -p 35357:35357 -p 5000:5000 keystone:mitaka + +echo -e "\033[32mConfiguring Moon platform\033[m" +sudo pip install moon_db +moon_db_manager upgrade + +cd ${MOON_HOME}/moon_orchestrator +python3 populate_consul.py + +echo -e "\033[32mStarting Moon platform\033[m" + +docker container run -dti --net moon --hostname orchestrator --name orchestrator wukongsun/moon_orchestrator:${VERSION} |