moonv4 cleanup

Change-Id: Icef927f3236d985ac13ff7376f6ce6314b2b39b0
Signed-off-by: WuKong <rebirthmonkey@gmail.com>

8 files changed, 444 insertions, 0 deletions

diff --git a/bin/README.md b/bin/README.md
new file mode 100644
index 00000000..3125c468
--- /dev/null
+++ b/bin/README.md
@@ -0,0 +1,5 @@
+# Automated Tools/Scripts
+
+## moon_utilities_update
+- update moon_utilities to PIP: `./moon_utilities_update.sh upload`
+- locally update moon_utilities for each moon Python package: `./moon_utilities_update.sh copy`
\ No newline at end of file
diff --git a/bin/bootstrap.py b/bin/bootstrap.py
new file mode 100644
index 00000000..6f2a5e03
--- /dev/null
+++ b/bin/bootstrap.py
@@ -0,0 +1,235 @@
+import os
+import sys
+import time
+import requests
+import yaml
+import logging
+import json
+import base64
+import mysql.connector
+import re
+import subprocess
+
+logging.basicConfig(level=logging.INFO)
+log = logging.getLogger("moon.bootstrap")
+requests_log = logging.getLogger("requests.packages.urllib3")
+requests_log.setLevel(logging.WARNING)
+requests_log.propagate = True
+
+if len(sys.argv) == 2:
+    if os.path.isfile(sys.argv[1]):
+        CONF_FILENAME = sys.argv[1]
+        CONSUL_HOST = "consul"
+    else:
+        CONF_FILENAME = "moon.conf"
+        CONSUL_HOST = sys.argv[1]
+    CONSUL_PORT = 8500
+else:
+    CONSUL_HOST = sys.argv[1] if len(sys.argv) > 1 else "consul"
+    CONSUL_PORT = sys.argv[2] if len(sys.argv) > 2 else 8500
+    CONF_FILENAME = sys.argv[3] if len(sys.argv) > 3 else "moon.conf"
+HEADERS = {"content-type": "application/json"}
+
+
+def search_config_file():
+    data_config = None
+    for _file in (
+            CONF_FILENAME,
+            "conf/moon.conf",
+            "../moon.conf",
+            "../conf/moon.conf",
+            "/etc/moon/moon.conf",
+    ):
+        try:
+            data_config = yaml.safe_load(open(_file))
+        except FileNotFoundError:
+            data_config = None
+            continue
+        else:
+            break
+    if not data_config:
+        raise Exception("Configuration file not found...")
+    return data_config
+
+
+def put(key, value):
+    url = "http://{host}:{port}/v1/kv/{key}".format(host=CONSUL_HOST, port=CONSUL_PORT, key=key)
+    log.info(url)
+    req = requests.put(
+        url,
+        headers=HEADERS,
+        json=value
+    )
+    if req.status_code != 200:
+        raise Exception("Error connecting to Consul ({}, {})".format(req.status_code, req.text))
+
+
+def get(key):
+    url = "http://{host}:{port}/v1/kv/{key}".format(host=CONSUL_HOST, port=CONSUL_PORT, key=key)
+    req = requests.get(url)
+    data = req.json()
+    for item in data:
+        log.info("{} {} -> {}".format(
+            req.status_code,
+            item["Key"],
+            json.loads(base64.b64decode(item["Value"]).decode("utf-8"))
+        ))
+        yield json.loads(base64.b64decode(item["Value"]).decode("utf-8"))
+
+
+def start_consul(data_config):
+    cmd = ["docker", "run", "-d", "--net=moon", "--name=consul", "--hostname=consul", "-p", "8500:8500", "consul"]
+    output = subprocess.run(cmd,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE)
+    if output.returncode != 0:
+        log.info(" ".join(cmd))
+        log.info(output.returncode)
+        log.error(output.stderr)
+        log.error(output.stdout)
+        raise Exception("Error starting Consul container!")
+    while True:
+        try:
+            req = requests.get("http://{}:{}/ui".format(CONSUL_HOST, CONSUL_PORT))
+        except requests.exceptions.ConnectionError:
+            log.info("Waiting for Consul ({}:{})".format(CONSUL_HOST, CONSUL_PORT))
+            time.sleep(1)
+            continue
+        else:
+            break
+        # if req.status_code in (302, 200):
+        #     break
+        # log.info("Waiting for Consul ({}:{})".format(CONSUL_HOST, CONSUL_PORT))
+        # time.sleep(1)
+    log.info("Consul is up")
+
+    req = requests.get("http://{}:{}/v1/kv/database".format(CONSUL_HOST, CONSUL_PORT))
+    if req.status_code == 200:
+        log.info("Consul is already populated")
+        return
+
+    put("database", data_config["database"])
+    put("messenger", data_config["messenger"])
+    put("slave", data_config["slave"])
+    put("docker", data_config["docker"])
+    put("logging", data_config["logging"])
+    put("components_port_start", data_config["components"]["port_start"])
+
+    for _key, _value in data_config["components"].items():
+        if type(_value) is dict:
+            put("components/{}".format(_key), data_config["components"][_key])
+
+    for _key, _value in data_config["plugins"].items():
+        put("plugins/{}".format(_key), data_config["plugins"][_key])
+
+    for _key, _value in data_config["openstack"].items():
+        put("openstack/{}".format(_key), data_config["openstack"][_key])
+
+
+def start_database():
+    cmd = ["docker", "run", "-dti", "--net=moon", "--hostname=db", "--name=db",
+           "-e", "MYSQL_ROOT_PASSWORD=p4sswOrd1", "-e", "MYSQL_DATABASE=moon", "-e", "MYSQL_USER=moon",
+           "-e", "MYSQL_PASSWORD=p4sswOrd1", "-p", "3306:3306", "mysql:latest"]
+    output = subprocess.run(cmd,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE)
+    if output.returncode != 0:
+        log.info(cmd)
+        log.error(output.stderr)
+        log.error(output.stdout)
+        raise Exception("Error starting DB container!")
+    for database in get("database"):
+        database_url = database['url']
+        match = re.search("(?P<proto>^[\\w+]+):\/\/(?P<user>\\w+):(?P<password>.+)@(?P<host>\\w+):*(?P<port>\\d*)",
+                          database_url)
+        config = match.groupdict()
+        while True:
+            try:
+                conn = mysql.connector.connect(
+                    host=config["host"],
+                    user=config["user"],
+                    password=config["password"],
+                    database="moon"
+                )
+                conn.close()
+            except mysql.connector.errors.InterfaceError:
+                log.info("Waiting for Database ({})".format(config["host"]))
+                time.sleep(1)
+                continue
+            else:
+                log.info("Database is up, populating it...")
+                output = subprocess.run(["moon_db_manager", "upgrade"],
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.PIPE)
+                if output.returncode != 0:
+                    raise Exception("Error populating the database!")
+                break
+
+
+def start_keystone():
+    output = subprocess.run(["docker", "run", "-dti", "--net=moon", "--hostname=keystone", "--name=keystone",
+                             "-e", "DB_HOST=db", "-e", "DB_PASSWORD_ROOT=p4sswOrd1", "-p", "35357:35357",
+                             "-p", "5000:5000", "keystone:mitaka"],
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE)
+    if output.returncode != 0:
+        raise Exception("Error starting Keystone container!")
+    # TODO: Keystone answers request too quickly
+    #       even if it is not fully loaded
+    #       we must test if a token retrieval is possible or not
+    #       to see if Keystone is truly up and running
+    for config in get("openstack/keystone"):
+        while True:
+            try:
+                time.sleep(1)
+                req = requests.get(config["url"])
+            except requests.exceptions.ConnectionError:
+                log.info("Waiting for Keystone ({})".format(config["url"]))
+                time.sleep(1)
+                continue
+            else:
+                log.info("Keystone is up")
+                break
+
+
+def start_moon(data_config):
+    cmds = [
+        # ["docker", "run", "-dti", "--net=moon", "--name=wrapper", "--hostname=wrapper", "-p",
+        #  "{0}:{0}".format(data_config['components']['wrapper']['port']),
+        #  data_config['components']['wrapper']['container']],
+        ["docker", "run", "-dti", "--net=moon", "--name=manager",
+         "--hostname=manager", "-p",
+         "{0}:{0}".format(data_config['components']['manager']['port']),
+         data_config['components']['manager']['container']],
+        ["docker", "run", "-dti", "--net=moon", "--name=interface",
+         "--hostname=interface", "-p",
+         "{0}:{0}".format(data_config['components']['interface']['port']),
+         data_config['components']['interface']['container']],
+    ]
+    for cmd in cmds:
+        log.warning("Start {}".format(cmd[-1]))
+        # answer = input()
+        # if answer.lower() in ("y", "yes", "o", "oui"):
+        output = subprocess.run(cmd,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
+        time.sleep(3)
+        if output.returncode != 0:
+            log.info(" ".join(cmd))
+            log.info(output.returncode)
+            log.error(output.stderr)
+            log.error(output.stdout)
+            raise Exception("Error starting {} container!".format(cmd[-1]))
+    subprocess.run(["docker", "ps"])
+
+
+def main():
+    data_config = search_config_file()
+    subprocess.run(["docker", "rm", "-f", "consul", "db", "manager", "wrapper", "interface", "authz*", "keystone"])
+    start_consul(data_config)
+    start_database()
+    start_keystone()
+    start_moon(data_config)
+
+main()
+
diff --git a/bin/build_all.sh b/bin/build_all.sh
new file mode 100644
index 00000000..5bbf6a19
--- /dev/null
+++ b/bin/build_all.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+VERSION=v4.1
+export DOCKER_HOST=tcp://172.88.88.1:2376
+
+
+mkdir $MOON_HOME/moon_orchestrator/dist 2>/dev/null
+
+echo Building Moon_Orchestrator
+cd $MOON_HOME/moon_orchestrator
+docker build -t wukongsun/moon_orchestrator:${VERSION} .
+
+echo Building Moon_Interface
+cd $MOON_HOME/moon_interface
+docker build -t wukongsun/moon_interface:${VERSION} .
+
+echo Building Moon_Security_Router
+cd $MOON_HOME/moon_secrouter
+docker build -t wukongsun/moon_router:${VERSION} .
+
+echo Building Moon_Manager
+cd $MOON_HOME/moon_manager
+docker build -t wukongsun/moon_manager:${VERSION} .
+
+echo Building Moon_Authz
+cd $MOON_HOME/moon_authz
+docker build -t wukongsun/moon_authz:${VERSION} .
+
+
+echo Building Moon_DB
+cd $MOON_HOME/moon_db
+python3 setup.py sdist bdist_wheel > /tmp/moon_db.log
+
+echo Building Moon_Utilities
+cd $MOON_HOME/moon_utilities
+python3 setup.py sdist bdist_wheel > /tmp/moon_utilities.log
diff --git a/bin/build_all_pip.sh b/bin/build_all_pip.sh
new file mode 100644
index 00000000..2b415bf0
--- /dev/null
+++ b/bin/build_all_pip.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+
+echo Building Moon_DB
+cd $MOON_HOME/moon_db
+python3 setup.py sdist bdist_wheel> /tmp/moon_db.log
+
+
+echo Building Moon_Utilities
+cd $MOON_HOME/moon_utilities
+python3 setup.py sdist bdist_wheel> /tmp/moon_utilities.log
+
+
+echo Building Moon_Orchestrator
+cd $MOON_HOME/moon_orchestrator
+python3 setup.py sdist bdist_wheel> /tmp/moon_orchestrator.log
\ No newline at end of file
diff --git a/bin/delete_orchestrator.sh b/bin/delete_orchestrator.sh
new file mode 100644
index 00000000..95fcfddd
--- /dev/null
+++ b/bin/delete_orchestrator.sh
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+set +x
+
+kubectl delete -n moon -f kubernetes/templates/moon_orchestrator.yaml
+for i in $(kubectl get deployments -n moon | grep wrapper | cut -d  " " -f 1 | xargs); do
+    kubectl delete deployments/$i -n moon;
+done
+for i in $(kubectl get deployments -n moon | grep interface | cut -d  " " -f 1 | xargs); do
+    kubectl delete deployments/$i -n moon;
+done
+for i in $(kubectl get deployments -n moon | grep authz | cut -d  " " -f 1 | xargs); do
+    kubectl delete deployments/$i -n moon;
+done
+for i in $(kubectl get services -n moon | grep wrapper | cut -d  " " -f 1 | xargs); do
+    kubectl delete services/$i -n moon;
+done
+for i in $(kubectl get services -n moon | grep interface | cut -d  " " -f 1 | xargs); do
+    kubectl delete services/$i -n moon;
+done
+for i in $(kubectl get services -n moon | grep authz | cut -d  " " -f 1 | xargs); do
+    kubectl delete services/$i -n moon;
+done
+
+if [ "$1" = "build" ]; then
+
+    DOCKER_ARGS=""
+
+    cd moon_manager
+    docker build -t wukongsun/moon_manager:v4.3.1 . ${DOCKER_ARGS}
+    if [ "$2" = "push" ]; then
+        docker push wukongsun/moon_manager:v4.3.1
+    fi
+    cd -
+
+    cd moon_orchestrator
+    docker build -t wukongsun/moon_orchestrator:v4.3 . ${DOCKER_ARGS}
+    if [ "$2" = "push" ]; then
+        docker push wukongsun/moon_orchestrator:v4.3
+    fi
+    cd -
+
+    cd moon_interface
+    docker build -t wukongsun/moon_interface:v4.3 . ${DOCKER_ARGS}
+    if [ "$2" = "push" ]; then
+        docker push wukongsun/moon_interface:v4.3
+    fi
+    cd -
+
+    cd moon_authz
+    docker build -t wukongsun/moon_authz:v4.3 . ${DOCKER_ARGS}
+    if [ "$2" = "push" ]; then
+        docker push wukongsun/moon_authz:v4.3
+    fi
+    cd -
+
+    cd moon_wrapper
+    docker build -t wukongsun/moon_wrapper:v4.3 . ${DOCKER_ARGS}
+    if [ "$2" = "push" ]; then
+        docker push wukongsun/moon_wrapper:v4.3
+    fi
+    cd -
+fi
diff --git a/bin/moon_lib_update.sh b/bin/moon_lib_update.sh
new file mode 100644
index 00000000..3925e336
--- /dev/null
+++ b/bin/moon_lib_update.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# usage: moon_update.sh {build,upload,copy} {python_moondb,python_moonutilities} <GPG_ID>
+
+CMD=$1
+COMPONENT=$2
+GPG_ID=$3
+
+VERSION=${COMPONENT}-$(grep __version__ ${COMPONENT}/${COMPONENT}/__init__.py | cut -d "\"" -f 2)
+
+cd ${COMPONENT}
+
+python3 setup.py sdist bdist_wheel
+
+if [ "$CMD" = "upload" ]; then
+    # Instead of "A0A96E75", use your own GPG ID
+    rm dist/*.asc 2>/dev/null
+    gpg --detach-sign -u "${GPG_ID}" -a dist/${VERSION}-py3-none-any.whl
+    gpg --detach-sign -u "${GPG_ID}" -a dist/${VERSION/_/-}.tar.gz
+    twine upload dist/${VERSION}-py3-none-any.whl dist/${VERSION}-py3-none-any.whl.asc
+    twine upload dist/${VERSION/_/-}.tar.gz dist/${VERSION/_/-}.tar.gz.asc
+fi
+
+rm -f ../moon_manager/dist/${COMPONENT}*
+rm -f ../moon_orchestrator/dist/${COMPONENT}*
+rm -f ../moon_wrapper/dist/${COMPONENT}*
+rm -f ../moon_interface/dist/${COMPONENT}*
+rm -f ../moon_authz/dist/${COMPONENT}*
+
+
+if [ "$CMD" = "copy" ]; then
+    mkdir -p ../moon_manager/dist/ 2>/dev/null
+    cp -v dist/${VERSION}-py3-none-any.whl ../moon_manager/dist/
+    mkdir -p ../moon_orchestrator/dist/ 2>/dev/null
+    cp -v dist/${VERSION}-py3-none-any.whl ../moon_orchestrator/dist/
+    mkdir -p ../moon_wrapper/dist/ 2>/dev/null
+    cp -v dist/${VERSION}-py3-none-any.whl ../moon_wrapper/dist/
+    mkdir -p ../moon_interface/dist/ 2>/dev/null
+    cp -v dist/${VERSION}-py3-none-any.whl ../moon_interface/dist/
+    mkdir -p ../moon_authz/dist/ 2>/dev/null
+    cp -v dist/${VERSION}-py3-none-any.whl ../moon_authz/dist/
+fi
+
diff --git a/bin/set_auth.src b/bin/set_auth.src
new file mode 100644
index 00000000..d955e30b
--- /dev/null
+++ b/bin/set_auth.src
@@ -0,0 +1,7 @@
+export OS_USERNAME=admin
+export OS_PASSWORD=p4ssw0rd
+export OS_REGION_NAME=Orange
+export OS_TENANT_NAME=admin
+export OS_AUTH_URL=http://keystone:5000/v3
+export OS_DOMAIN_NAME=Default
+export MOON_URL=http://172.18.0.11:38001
diff --git a/bin/start.sh b/bin/start.sh
new file mode 100755
index 00000000..e95ac393
--- /dev/null
+++ b/bin/start.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+VERSION=4.1
+export DOCKER_HOST=tcp://172.88.88.1:2376
+
+echo -e "\033[31mDeleting previous dockers\033[m"
+docker rm -f $(docker ps -a | grep moon | cut -d " " -f 1) 2>/dev/null
+docker rm -f messenger db keystone consul 2>/dev/null
+
+echo -e "\033[32mStarting Messenger\033[m"
+docker run -dti --net=moon --hostname messenger --name messenger -e RABBITMQ_DEFAULT_USER=moon -e RABBITMQ_DEFAULT_PASS=p4sswOrd1 -e RABBITMQ_NODENAME=rabbit@messenger -e RABBITMQ_DEFAULT_VHOST=moon -e RABBITMQ_HIPE_COMPILE=1 -p 5671:5671 -p 5672:5672 -p 8080:15672 rabbitmq:3-management
+
+echo -e "\033[32mStarting DB manager\033[m"
+docker run -dti --net=moon --hostname db        --name db        -e MYSQL_ROOT_PASSWORD=p4sswOrd1 -e MYSQL_DATABASE=moon -e MYSQL_USER=moon -e MYSQL_PASSWORD=p4sswOrd1 -p 3306:3306 mysql:latest
+
+docker run -d --net=moon --name=consul --hostname=consul -p 8500:8500 consul
+
+echo "waiting for Database (it may takes time)..."
+echo -e "\033[35m"
+sed '/ready for connections/q' <(docker logs db -f)
+echo -e "\033[m"
+
+echo "waiting for Messenger (it may takes time)..."
+echo -e "\033[35m"
+sed '/Server startup complete;/q' <(docker logs messenger -f)
+echo -e "\033[m"
+
+docker run -dti --net moon --hostname keystone  --name keystone  -e DB_HOST=db -e DB_PASSWORD_ROOT=p4sswOrd1 -p 35357:35357 -p 5000:5000 keystone:mitaka
+
+echo -e "\033[32mConfiguring Moon platform\033[m"
+sudo pip install moon_db
+moon_db_manager upgrade
+
+cd ${MOON_HOME}/moon_orchestrator
+python3 populate_consul.py
+
+echo -e "\033[32mStarting Moon platform\033[m"
+
+docker container run -dti --net moon --hostname orchestrator --name orchestrator wukongsun/moon_orchestrator:${VERSION}