From 454e9c5f8664ea99ccea2417b6cc3ffb238cf834 Mon Sep 17 00:00:00 2001
From: RHE <rebirthmonkey@gmail.com>
Date: Tue, 26 Dec 2017 13:35:54 +0100
Subject: moon v4 re-organization
Change-Id: I73665f739f35ae18175f98d0739567e403c1fa80
Signed-off-by: RHE <rebirthmonkey@gmail.com>
---
tools/bin/README.md | 5 +
tools/bin/bootstrap.py | 235 +++++++++++++++++++++++++++++++++++++++
tools/bin/build_all.sh | 36 ++++++
tools/bin/build_all_pip.sh | 16 +++
tools/bin/delete_orchestrator.sh | 63 +++++++++++
tools/bin/moon_lib_update.sh | 43 +++++++
tools/bin/set_auth.src | 7 ++
tools/bin/start.sh | 39 +++++++
8 files changed, 444 insertions(+)
create mode 100644 tools/bin/README.md
create mode 100644 tools/bin/bootstrap.py
create mode 100644 tools/bin/build_all.sh
create mode 100644 tools/bin/build_all_pip.sh
create mode 100644 tools/bin/delete_orchestrator.sh
create mode 100644 tools/bin/moon_lib_update.sh
create mode 100644 tools/bin/set_auth.src
create mode 100755 tools/bin/start.sh
(limited to 'tools/bin')
diff --git a/tools/bin/README.md b/tools/bin/README.md
new file mode 100644
index 00000000..3125c468
--- /dev/null
+++ b/tools/bin/README.md
@@ -0,0 +1,5 @@
+# Automated Tools/Scripts
+
+## moon_utilities_update
+- update moon_utilities to PIP: `./moon_utilities_update.sh upload`
+- locally update moon_utilities for each moon Python package: `./moon_utilities_update.sh copy`
\ No newline at end of file
diff --git a/tools/bin/bootstrap.py b/tools/bin/bootstrap.py
new file mode 100644
index 00000000..6f2a5e03
--- /dev/null
+++ b/tools/bin/bootstrap.py
@@ -0,0 +1,235 @@
+import os
+import sys
+import time
+import requests
+import yaml
+import logging
+import json
+import base64
+import mysql.connector
+import re
+import subprocess
+
+logging.basicConfig(level=logging.INFO)
+log = logging.getLogger("moon.bootstrap")
+requests_log = logging.getLogger("requests.packages.urllib3")
+requests_log.setLevel(logging.WARNING)
+requests_log.propagate = True
+
+if len(sys.argv) == 2:
+ if os.path.isfile(sys.argv[1]):
+ CONF_FILENAME = sys.argv[1]
+ CONSUL_HOST = "consul"
+ else:
+ CONF_FILENAME = "moon.conf"
+ CONSUL_HOST = sys.argv[1]
+ CONSUL_PORT = 8500
+else:
+ CONSUL_HOST = sys.argv[1] if len(sys.argv) > 1 else "consul"
+ CONSUL_PORT = sys.argv[2] if len(sys.argv) > 2 else 8500
+ CONF_FILENAME = sys.argv[3] if len(sys.argv) > 3 else "moon.conf"
+HEADERS = {"content-type": "application/json"}
+
+
+def search_config_file():
+ data_config = None
+ for _file in (
+ CONF_FILENAME,
+ "conf/moon.conf",
+ "../moon.conf",
+ "../conf/moon.conf",
+ "/etc/moon/moon.conf",
+ ):
+ try:
+ data_config = yaml.safe_load(open(_file))
+ except FileNotFoundError:
+ data_config = None
+ continue
+ else:
+ break
+ if not data_config:
+ raise Exception("Configuration file not found...")
+ return data_config
+
+
+def put(key, value):
+ url = "http://{host}:{port}/v1/kv/{key}".format(host=CONSUL_HOST, port=CONSUL_PORT, key=key)
+ log.info(url)
+ req = requests.put(
+ url,
+ headers=HEADERS,
+ json=value
+ )
+ if req.status_code != 200:
+ raise Exception("Error connecting to Consul ({}, {})".format(req.status_code, req.text))
+
+
+def get(key):
+ url = "http://{host}:{port}/v1/kv/{key}".format(host=CONSUL_HOST, port=CONSUL_PORT, key=key)
+ req = requests.get(url)
+ data = req.json()
+ for item in data:
+ log.info("{} {} -> {}".format(
+ req.status_code,
+ item["Key"],
+ json.loads(base64.b64decode(item["Value"]).decode("utf-8"))
+ ))
+ yield json.loads(base64.b64decode(item["Value"]).decode("utf-8"))
+
+
+def start_consul(data_config):
+ cmd = ["docker", "run", "-d", "--net=moon", "--name=consul", "--hostname=consul", "-p", "8500:8500", "consul"]
+ output = subprocess.run(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ if output.returncode != 0:
+ log.info(" ".join(cmd))
+ log.info(output.returncode)
+ log.error(output.stderr)
+ log.error(output.stdout)
+ raise Exception("Error starting Consul container!")
+ while True:
+ try:
+ req = requests.get("http://{}:{}/ui".format(CONSUL_HOST, CONSUL_PORT))
+ except requests.exceptions.ConnectionError:
+ log.info("Waiting for Consul ({}:{})".format(CONSUL_HOST, CONSUL_PORT))
+ time.sleep(1)
+ continue
+ else:
+ break
+ # if req.status_code in (302, 200):
+ # break
+ # log.info("Waiting for Consul ({}:{})".format(CONSUL_HOST, CONSUL_PORT))
+ # time.sleep(1)
+ log.info("Consul is up")
+
+ req = requests.get("http://{}:{}/v1/kv/database".format(CONSUL_HOST, CONSUL_PORT))
+ if req.status_code == 200:
+ log.info("Consul is already populated")
+ return
+
+ put("database", data_config["database"])
+ put("messenger", data_config["messenger"])
+ put("slave", data_config["slave"])
+ put("docker", data_config["docker"])
+ put("logging", data_config["logging"])
+ put("components_port_start", data_config["components"]["port_start"])
+
+ for _key, _value in data_config["components"].items():
+ if type(_value) is dict:
+ put("components/{}".format(_key), data_config["components"][_key])
+
+ for _key, _value in data_config["plugins"].items():
+ put("plugins/{}".format(_key), data_config["plugins"][_key])
+
+ for _key, _value in data_config["openstack"].items():
+ put("openstack/{}".format(_key), data_config["openstack"][_key])
+
+
+def start_database():
+ cmd = ["docker", "run", "-dti", "--net=moon", "--hostname=db", "--name=db",
+ "-e", "MYSQL_ROOT_PASSWORD=p4sswOrd1", "-e", "MYSQL_DATABASE=moon", "-e", "MYSQL_USER=moon",
+ "-e", "MYSQL_PASSWORD=p4sswOrd1", "-p", "3306:3306", "mysql:latest"]
+ output = subprocess.run(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ if output.returncode != 0:
+ log.info(cmd)
+ log.error(output.stderr)
+ log.error(output.stdout)
+ raise Exception("Error starting DB container!")
+ for database in get("database"):
+ database_url = database['url']
+ match = re.search("(?P<proto>^[\\w+]+):\/\/(?P<user>\\w+):(?P<password>.+)@(?P<host>\\w+):*(?P<port>\\d*)",
+ database_url)
+ config = match.groupdict()
+ while True:
+ try:
+ conn = mysql.connector.connect(
+ host=config["host"],
+ user=config["user"],
+ password=config["password"],
+ database="moon"
+ )
+ conn.close()
+ except mysql.connector.errors.InterfaceError:
+ log.info("Waiting for Database ({})".format(config["host"]))
+ time.sleep(1)
+ continue
+ else:
+ log.info("Database is up, populating it...")
+ output = subprocess.run(["moon_db_manager", "upgrade"],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ if output.returncode != 0:
+ raise Exception("Error populating the database!")
+ break
+
+
+def start_keystone():
+ output = subprocess.run(["docker", "run", "-dti", "--net=moon", "--hostname=keystone", "--name=keystone",
+ "-e", "DB_HOST=db", "-e", "DB_PASSWORD_ROOT=p4sswOrd1", "-p", "35357:35357",
+ "-p", "5000:5000", "keystone:mitaka"],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ if output.returncode != 0:
+ raise Exception("Error starting Keystone container!")
+ # TODO: Keystone answers request too quickly
+ # even if it is not fully loaded
+ # we must test if a token retrieval is possible or not
+ # to see if Keystone is truly up and running
+ for config in get("openstack/keystone"):
+ while True:
+ try:
+ time.sleep(1)
+ req = requests.get(config["url"])
+ except requests.exceptions.ConnectionError:
+ log.info("Waiting for Keystone ({})".format(config["url"]))
+ time.sleep(1)
+ continue
+ else:
+ log.info("Keystone is up")
+ break
+
+
+def start_moon(data_config):
+ cmds = [
+ # ["docker", "run", "-dti", "--net=moon", "--name=wrapper", "--hostname=wrapper", "-p",
+ # "{0}:{0}".format(data_config['components']['wrapper']['port']),
+ # data_config['components']['wrapper']['container']],
+ ["docker", "run", "-dti", "--net=moon", "--name=manager",
+ "--hostname=manager", "-p",
+ "{0}:{0}".format(data_config['components']['manager']['port']),
+ data_config['components']['manager']['container']],
+ ["docker", "run", "-dti", "--net=moon", "--name=interface",
+ "--hostname=interface", "-p",
+ "{0}:{0}".format(data_config['components']['interface']['port']),
+ data_config['components']['interface']['container']],
+ ]
+ for cmd in cmds:
+ log.warning("Start {}".format(cmd[-1]))
+ # answer = input()
+ # if answer.lower() in ("y", "yes", "o", "oui"):
+ output = subprocess.run(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ time.sleep(3)
+ if output.returncode != 0:
+ log.info(" ".join(cmd))
+ log.info(output.returncode)
+ log.error(output.stderr)
+ log.error(output.stdout)
+ raise Exception("Error starting {} container!".format(cmd[-1]))
+ subprocess.run(["docker", "ps"])
+
+
+def main():
+ data_config = search_config_file()
+ subprocess.run(["docker", "rm", "-f", "consul", "db", "manager", "wrapper", "interface", "authz*", "keystone"])
+ start_consul(data_config)
+ start_database()
+ start_keystone()
+ start_moon(data_config)
+
+main()
+
diff --git a/tools/bin/build_all.sh b/tools/bin/build_all.sh
new file mode 100644
index 00000000..5bbf6a19
--- /dev/null
+++ b/tools/bin/build_all.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+VERSION=v4.1
+export DOCKER_HOST=tcp://172.88.88.1:2376
+
+
+mkdir $MOON_HOME/moon_orchestrator/dist 2>/dev/null
+
+echo Building Moon_Orchestrator
+cd $MOON_HOME/moon_orchestrator
+docker build -t wukongsun/moon_orchestrator:${VERSION} .
+
+echo Building Moon_Interface
+cd $MOON_HOME/moon_interface
+docker build -t wukongsun/moon_interface:${VERSION} .
+
+echo Building Moon_Security_Router
+cd $MOON_HOME/moon_secrouter
+docker build -t wukongsun/moon_router:${VERSION} .
+
+echo Building Moon_Manager
+cd $MOON_HOME/moon_manager
+docker build -t wukongsun/moon_manager:${VERSION} .
+
+echo Building Moon_Authz
+cd $MOON_HOME/moon_authz
+docker build -t wukongsun/moon_authz:${VERSION} .
+
+
+echo Building Moon_DB
+cd $MOON_HOME/moon_db
+python3 setup.py sdist bdist_wheel > /tmp/moon_db.log
+
+echo Building Moon_Utilities
+cd $MOON_HOME/moon_utilities
+python3 setup.py sdist bdist_wheel > /tmp/moon_utilities.log
diff --git a/tools/bin/build_all_pip.sh b/tools/bin/build_all_pip.sh
new file mode 100644
index 00000000..2b415bf0
--- /dev/null
+++ b/tools/bin/build_all_pip.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+
+echo Building Moon_DB
+cd $MOON_HOME/moon_db
+python3 setup.py sdist bdist_wheel> /tmp/moon_db.log
+
+
+echo Building Moon_Utilities
+cd $MOON_HOME/moon_utilities
+python3 setup.py sdist bdist_wheel> /tmp/moon_utilities.log
+
+
+echo Building Moon_Orchestrator
+cd $MOON_HOME/moon_orchestrator
+python3 setup.py sdist bdist_wheel> /tmp/moon_orchestrator.log
\ No newline at end of file
diff --git a/tools/bin/delete_orchestrator.sh b/tools/bin/delete_orchestrator.sh
new file mode 100644
index 00000000..95fcfddd
--- /dev/null
+++ b/tools/bin/delete_orchestrator.sh
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+set +x
+
+kubectl delete -n moon -f kubernetes/templates/moon_orchestrator.yaml
+for i in $(kubectl get deployments -n moon | grep wrapper | cut -d " " -f 1 | xargs); do
+ kubectl delete deployments/$i -n moon;
+done
+for i in $(kubectl get deployments -n moon | grep interface | cut -d " " -f 1 | xargs); do
+ kubectl delete deployments/$i -n moon;
+done
+for i in $(kubectl get deployments -n moon | grep authz | cut -d " " -f 1 | xargs); do
+ kubectl delete deployments/$i -n moon;
+done
+for i in $(kubectl get services -n moon | grep wrapper | cut -d " " -f 1 | xargs); do
+ kubectl delete services/$i -n moon;
+done
+for i in $(kubectl get services -n moon | grep interface | cut -d " " -f 1 | xargs); do
+ kubectl delete services/$i -n moon;
+done
+for i in $(kubectl get services -n moon | grep authz | cut -d " " -f 1 | xargs); do
+ kubectl delete services/$i -n moon;
+done
+
+if [ "$1" = "build" ]; then
+
+ DOCKER_ARGS=""
+
+ cd moon_manager
+ docker build -t wukongsun/moon_manager:v4.3.1 . ${DOCKER_ARGS}
+ if [ "$2" = "push" ]; then
+ docker push wukongsun/moon_manager:v4.3.1
+ fi
+ cd -
+
+ cd moon_orchestrator
+ docker build -t wukongsun/moon_orchestrator:v4.3 . ${DOCKER_ARGS}
+ if [ "$2" = "push" ]; then
+ docker push wukongsun/moon_orchestrator:v4.3
+ fi
+ cd -
+
+ cd moon_interface
+ docker build -t wukongsun/moon_interface:v4.3 . ${DOCKER_ARGS}
+ if [ "$2" = "push" ]; then
+ docker push wukongsun/moon_interface:v4.3
+ fi
+ cd -
+
+ cd moon_authz
+ docker build -t wukongsun/moon_authz:v4.3 . ${DOCKER_ARGS}
+ if [ "$2" = "push" ]; then
+ docker push wukongsun/moon_authz:v4.3
+ fi
+ cd -
+
+ cd moon_wrapper
+ docker build -t wukongsun/moon_wrapper:v4.3 . ${DOCKER_ARGS}
+ if [ "$2" = "push" ]; then
+ docker push wukongsun/moon_wrapper:v4.3
+ fi
+ cd -
+fi
diff --git a/tools/bin/moon_lib_update.sh b/tools/bin/moon_lib_update.sh
new file mode 100644
index 00000000..3925e336
--- /dev/null
+++ b/tools/bin/moon_lib_update.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# usage: moon_update.sh {build,upload,copy} {python_moondb,python_moonutilities} <GPG_ID>
+
+CMD=$1
+COMPONENT=$2
+GPG_ID=$3
+
+VERSION=${COMPONENT}-$(grep __version__ ${COMPONENT}/${COMPONENT}/__init__.py | cut -d "\"" -f 2)
+
+cd ${COMPONENT}
+
+python3 setup.py sdist bdist_wheel
+
+if [ "$CMD" = "upload" ]; then
+ # Instead of "A0A96E75", use your own GPG ID
+ rm dist/*.asc 2>/dev/null
+ gpg --detach-sign -u "${GPG_ID}" -a dist/${VERSION}-py3-none-any.whl
+ gpg --detach-sign -u "${GPG_ID}" -a dist/${VERSION/_/-}.tar.gz
+ twine upload dist/${VERSION}-py3-none-any.whl dist/${VERSION}-py3-none-any.whl.asc
+ twine upload dist/${VERSION/_/-}.tar.gz dist/${VERSION/_/-}.tar.gz.asc
+fi
+
+rm -f ../moon_manager/dist/${COMPONENT}*
+rm -f ../moon_orchestrator/dist/${COMPONENT}*
+rm -f ../moon_wrapper/dist/${COMPONENT}*
+rm -f ../moon_interface/dist/${COMPONENT}*
+rm -f ../moon_authz/dist/${COMPONENT}*
+
+
+if [ "$CMD" = "copy" ]; then
+ mkdir -p ../moon_manager/dist/ 2>/dev/null
+ cp -v dist/${VERSION}-py3-none-any.whl ../moon_manager/dist/
+ mkdir -p ../moon_orchestrator/dist/ 2>/dev/null
+ cp -v dist/${VERSION}-py3-none-any.whl ../moon_orchestrator/dist/
+ mkdir -p ../moon_wrapper/dist/ 2>/dev/null
+ cp -v dist/${VERSION}-py3-none-any.whl ../moon_wrapper/dist/
+ mkdir -p ../moon_interface/dist/ 2>/dev/null
+ cp -v dist/${VERSION}-py3-none-any.whl ../moon_interface/dist/
+ mkdir -p ../moon_authz/dist/ 2>/dev/null
+ cp -v dist/${VERSION}-py3-none-any.whl ../moon_authz/dist/
+fi
+
diff --git a/tools/bin/set_auth.src b/tools/bin/set_auth.src
new file mode 100644
index 00000000..d955e30b
--- /dev/null
+++ b/tools/bin/set_auth.src
@@ -0,0 +1,7 @@
+export OS_USERNAME=admin
+export OS_PASSWORD=p4ssw0rd
+export OS_REGION_NAME=Orange
+export OS_TENANT_NAME=admin
+export OS_AUTH_URL=http://keystone:5000/v3
+export OS_DOMAIN_NAME=Default
+export MOON_URL=http://172.18.0.11:38001
diff --git a/tools/bin/start.sh b/tools/bin/start.sh
new file mode 100755
index 00000000..e95ac393
--- /dev/null
+++ b/tools/bin/start.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+VERSION=4.1
+export DOCKER_HOST=tcp://172.88.88.1:2376
+
+echo -e "\033[31mDeleting previous dockers\033[m"
+docker rm -f $(docker ps -a | grep moon | cut -d " " -f 1) 2>/dev/null
+docker rm -f messenger db keystone consul 2>/dev/null
+
+echo -e "\033[32mStarting Messenger\033[m"
+docker run -dti --net=moon --hostname messenger --name messenger -e RABBITMQ_DEFAULT_USER=moon -e RABBITMQ_DEFAULT_PASS=p4sswOrd1 -e RABBITMQ_NODENAME=rabbit@messenger -e RABBITMQ_DEFAULT_VHOST=moon -e RABBITMQ_HIPE_COMPILE=1 -p 5671:5671 -p 5672:5672 -p 8080:15672 rabbitmq:3-management
+
+echo -e "\033[32mStarting DB manager\033[m"
+docker run -dti --net=moon --hostname db --name db -e MYSQL_ROOT_PASSWORD=p4sswOrd1 -e MYSQL_DATABASE=moon -e MYSQL_USER=moon -e MYSQL_PASSWORD=p4sswOrd1 -p 3306:3306 mysql:latest
+
+docker run -d --net=moon --name=consul --hostname=consul -p 8500:8500 consul
+
+echo "waiting for Database (it may takes time)..."
+echo -e "\033[35m"
+sed '/ready for connections/q' <(docker logs db -f)
+echo -e "\033[m"
+
+echo "waiting for Messenger (it may takes time)..."
+echo -e "\033[35m"
+sed '/Server startup complete;/q' <(docker logs messenger -f)
+echo -e "\033[m"
+
+docker run -dti --net moon --hostname keystone --name keystone -e DB_HOST=db -e DB_PASSWORD_ROOT=p4sswOrd1 -p 35357:35357 -p 5000:5000 keystone:mitaka
+
+echo -e "\033[32mConfiguring Moon platform\033[m"
+sudo pip install moon_db
+moon_db_manager upgrade
+
+cd ${MOON_HOME}/moon_orchestrator
+python3 populate_consul.py
+
+echo -e "\033[32mStarting Moon platform\033[m"
+
+docker container run -dti --net moon --hostname orchestrator --name orchestrator wukongsun/moon_orchestrator:${VERSION}
--
cgit