From 7e83d0876ddb84a45e130eeba28bc40ef53c074b Mon Sep 17 00:00:00 2001 From: Yaron Yogev Date: Thu, 27 Jul 2017 09:02:54 +0300 Subject: Calipso initial release for OPNFV Change-Id: I7210c244b0c10fa80bfa8c77cb86c9d6ddf8bc88 Signed-off-by: Yaron Yogev --- app/install/calipso-installer.py | 380 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 380 insertions(+) create mode 100644 app/install/calipso-installer.py (limited to 'app/install/calipso-installer.py') diff --git a/app/install/calipso-installer.py b/app/install/calipso-installer.py new file mode 100644 index 0000000..bccddae --- /dev/null +++ b/app/install/calipso-installer.py @@ -0,0 +1,380 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from pymongo import MongoClient, ReturnDocument +from pymongo.errors import ConnectionFailure +from urllib.parse import quote_plus +import docker +import argparse +import dockerpycreds +# note : not used, useful for docker api security if used +import time +import json + + +class MongoComm: + # deals with communication from host/installer server to mongoDB, includes methods for future use + try: + + def __init__(self, host, user, password, port): + self.uri = "mongodb://%s:%s@%s:%s/%s" % ( + quote_plus(user), quote_plus(password), host, port, "calipso") + self.client = MongoClient(self.uri) + + def find(self, coll, key, val): + collection = self.client.calipso[coll] + doc = collection.find({key: val}) + return doc + + def get(self, coll, doc_name): + collection = self.client.calipso[coll] + doc = collection.find_one({"name": doc_name}) + return doc + + def insert(self, coll, doc): + collection = self.client.calipso[coll] + doc_id = collection.insert(doc) + return doc_id + + def remove_doc(self, coll, doc): + collection = self.client.calipso[coll] + collection.remove(doc) + + def remove_coll(self, coll): + collection = self.client.calipso[coll] + collection.remove() + + def find_update(self, coll, key, val, data): + collection = self.client.calipso[coll] + collection.find_one_and_update( + {key: val}, + {"$set": data}, + upsert=True + ) + + def update(self, coll, doc, upsert=False): + collection = self.client.calipso[coll] + doc_id = collection.update_one({'_id': doc['_id']},{'$set': doc}, upsert=upsert) + return doc_id + + except ConnectionFailure: + print("MongoDB Server not available") + + +DockerClient = docker.from_env() # using local host docker environment parameters + +# use the below example for installer against a remote docker host: +# DockerClient = docker.DockerClient(base_url='tcp://korlev-calipso-testing.cisco.com:2375') + + +def copy_file(filename): + c = MongoComm(args.hostname, args.dbuser, args.dbpassword, args.dbport) + txt = open('db/'+filename+'.json') + data = json.load(txt) + c.remove_coll(filename) + doc_id = c.insert(filename, data) + print("Copied", filename, "mongo doc_ids:\n\n", doc_id, "\n\n") + time.sleep(1) + + +C_MONGO_CONFIG = "/local_dir/calipso_mongo_access.conf" +H_MONGO_CONFIG = "/home/calipso/calipso_mongo_access.conf" +PYTHONPATH = "/home/scan/calipso_prod/app" +C_LDAP_CONFIG = "/local_dir/ldap.conf" +H_LDAP_CONFIG = "/home/calipso/ldap.conf" + +# functions to check and start calipso containers: +def start_mongo(dbport): + if not DockerClient.containers.list(all=True, filters={"name": "calipso-mongo"}): + print("\nstarting container calipso-mongo, please wait...\n") + image = DockerClient.images.list(all=True, name="korenlev/calipso:mongo") + if image: + print(image, "exists...not downloading...") + else: + print("image korenlev/calipso:mongo missing, hold on while downloading first...\n") + image = DockerClient.images.pull("korenlev/calipso:mongo") + print("Downloaded", image, "\n\n") + mongocontainer = DockerClient.containers.run('korenlev/calipso:mongo', detach=True, name="calipso-mongo", + ports={'27017/tcp': dbport, '28017/tcp': 28017}, + restart_policy={"Name": "always"}) + # wait a bit till mongoDB is up before starting to copy the json files from 'db' folder: + time.sleep(5) + enable_copy = input("create initial calipso DB ? (copy json files from 'db' folder to mongoDB -" + " 'c' to copy, 'q' to skip):") + if enable_copy == "c": + print("\nstarting to copy json files to mongoDB...\n\n") + print("-----------------------------------------\n\n") + time.sleep(1) + copy_file("attributes_for_hover_on_data") + copy_file("clique_constraints") + copy_file("clique_types") + copy_file("cliques") + copy_file("constants") + copy_file("environments_config") + copy_file("inventory") + copy_file("link_types") + copy_file("links") + copy_file("messages") + copy_file("meteor_accounts_loginServiceConfiguration") + copy_file("users") + copy_file("monitoring_config") + copy_file("monitoring_config_templates") + copy_file("network_agent_types") + copy_file("roles") + copy_file("scans") + copy_file("scheduled_scans") + copy_file("statistics") + copy_file("supported_environments") + + # note : 'messages', 'roles', 'users' and some of the 'constants' are filled by calipso-ui at runtime + # some other docs are filled later by scanning, logging and monitoring + else: + return + else: + print("container named calipso-mongo already exists, please deal with it using docker...\n") + return + + +def start_listen(): + if not DockerClient.containers.list(all=True, filters={"name": "calipso-listen"}): + print("\nstarting container calipso-listen...\n") + image = DockerClient.images.list(all=True, name="korenlev/calipso:listen") + if image: + print(image, "exists...not downloading...") + else: + print("image korenlev/calipso:listen missing, hold on while downloading first...\n") + image = DockerClient.images.pull("korenlev/calipso:listen") + print("Downloaded", image, "\n\n") + listencontainer = DockerClient.containers.run('korenlev/calipso:listen', detach=True, name="calipso-listen", + ports={'22/tcp': 50022}, + restart_policy={"Name": "always"}, + environment=["PYTHONPATH=" + PYTHONPATH, + "MONGO_CONFIG=" + C_MONGO_CONFIG], + volumes={'/home/calipso': {'bind': '/local_dir', 'mode': 'rw'}}) + else: + print("container named calipso-listen already exists, please deal with it using docker...\n") + return + + +def start_ldap(): + if not DockerClient.containers.list(all=True, filters={"name": "calipso-ldap"}): + print("\nstarting container calipso-ldap...\n") + image = DockerClient.images.list(all=True, name="korenlev/calipso:ldap") + if image: + print(image, "exists...not downloading...") + else: + print("image korenlev/calipso:ldap missing, hold on while downloading first...\n") + image = DockerClient.images.pull("korenlev/calipso:ldap") + print("Downloaded", image, "\n\n") + ldapcontainer = DockerClient.containers.run('korenlev/calipso:ldap', detach=True, name="calipso-ldap", + ports={'389/tcp': 389, '389/udp': 389}, + restart_policy={"Name": "always"}, + volumes={'/home/calipso/': {'bind': '/local_dir/', 'mode': 'rw'}}) + else: + print("container named calipso-ldap already exists, please deal with it using docker...\n") + return + + +def start_api(): + if not DockerClient.containers.list(all=True, filters={"name": "calipso-api"}): + print("\nstarting container calipso-api...\n") + image = DockerClient.images.list(all=True, name="korenlev/calipso:api") + if image: + print(image, "exists...not downloading...") + else: + print("image korenlev/calipso:api missing, hold on while downloading first...\n") + image = DockerClient.images.pull("korenlev/calipso:api") + print("Downloaded", image, "\n\n") + apicontainer = DockerClient.containers.run('korenlev/calipso:api', detach=True, name="calipso-api", + ports={'8000/tcp': 8000, '22/tcp': 40022}, + restart_policy={"Name": "always"}, + environment=["PYTHONPATH=" + PYTHONPATH, + "MONGO_CONFIG=" + C_MONGO_CONFIG, + "LDAP_CONFIG=" + C_LDAP_CONFIG, + "LOG_LEVEL=DEBUG"], + volumes={'/home/calipso/': {'bind': '/local_dir/', 'mode': 'rw'}}) + else: + print("container named calipso-api already exists, please deal with it using docker...\n") + return + + +def start_scan(): + if not DockerClient.containers.list(all=True, filters={"name": "calipso-scan"}): + print("\nstarting container calipso-scan...\n") + image = DockerClient.images.list(all=True, name="korenlev/calipso:scan") + if image: + print(image, "exists...not downloading...") + else: + print("image korenlev/calipso:scan missing, hold on while downloading first...\n") + image = DockerClient.images.pull("korenlev/calipso:scan") + print("Downloaded", image, "\n\n") + scancontainer = DockerClient.containers.run('korenlev/calipso:scan', detach=True, name="calipso-scan", + ports={'22/tcp': 30022}, + restart_policy={"Name": "always"}, + environment=["PYTHONPATH=" + PYTHONPATH, + "MONGO_CONFIG=" + C_MONGO_CONFIG], + volumes={'/home/calipso/': {'bind': '/local_dir/', 'mode': 'rw'}}) + else: + print("container named calipso-scan already exists, please deal with it using docker...\n") + return + + +def start_sensu(): + if not DockerClient.containers.list(all=True, filters={"name": "calipso-sensu"}): + print("\nstarting container calipso-sensu...\n") + image = DockerClient.images.list(all=True, name="korenlev/calipso:sensu") + if image: + print(image, "exists...not downloading...") + else: + print("image korenlev/calipso:sensu missing, hold on while downloading first...\n") + image = DockerClient.images.pull("korenlev/calipso:sensu") + print("Downloaded", image, "\n\n") + sensucontainer = DockerClient.containers.run('korenlev/calipso:sensu', detach=True, name="calipso-sensu", + ports={'22/tcp': 20022, '3000/tcp': 3000, '4567/tcp': 4567, + '5671/tcp': 5671, '15672/tcp': 15672}, + restart_policy={"Name": "always"}, + environment=["PYTHONPATH=" + PYTHONPATH], + volumes={'/home/calipso/': {'bind': '/local_dir/', 'mode': 'rw'}}) + else: + print("container named calipso-sensu already exists, please deal with it using docker...\n") + return + + +def start_ui(host, dbuser, dbpassword, webport, dbport): + if not DockerClient.containers.list(all=True, filters={"name": "calipso-ui"}): + print("\nstarting container calipso-ui...\n") + image = DockerClient.images.list(all=True, name="korenlev/calipso:ui") + if image: + print(image, "exists...not downloading...") + else: + print("image korenlev/calipso:ui missing, hold on while downloading first...\n") + image = DockerClient.images.pull("korenlev/calipso:ui") + print("Downloaded", image, "\n\n") + uicontainer = DockerClient.containers.run('korenlev/calipso:ui', detach=True, name="calipso-ui", + ports={'3000/tcp': webport}, + restart_policy={"Name": "always"}, + environment=["ROOT_URL=http://{}:{}".format(host, str(webport)), + "MONGO_URL=mongodb://{}:{}@{}:{}/calipso".format + (dbuser, dbpassword, host, str(dbport)), + "LDAP_CONFIG=" + C_LDAP_CONFIG]) + else: + print("container named calipso-ui already exists, please deal with it using docker...\n") + return + + +# function to check and stop calipso containers: + +def container_stop(container_name): + if DockerClient.containers.list(all=True, filters={"name": container_name}): + print("fetching container name", container_name, "...\n") + c = DockerClient.containers.get(container_name) + if c.status != "running": + print(container_name, "is not running...") + time.sleep(1) + print("removing container name", c.name, "...\n") + c.remove() + else: + print("killing container name", c.name, "...\n") + c.kill() + time.sleep(1) + print("removing container name", c.name, "...\n") + c.remove() + else: + print("no container named", container_name, "found...") + + +# parser for getting optional command arguments: +parser = argparse.ArgumentParser() +parser.add_argument("--hostname", help="Hostname or IP address of the server (default=172.17.0.1)",type=str, + default="172.17.0.1", required=False) +parser.add_argument("--webport", help="Port for the Calipso WebUI (default=80)",type=int, + default="80", required=False) +parser.add_argument("--dbport", help="Port for the Calipso MongoDB (default=27017)",type=int, + default="27017", required=False) +parser.add_argument("--dbuser", help="User for the Calipso MongoDB (default=calipso)",type=str, + default="calipso", required=False) +parser.add_argument("--dbpassword", help="Password for the Calipso MongoDB (default=calipso_default)",type=str, + default="calipso_default", required=False) +args = parser.parse_args() + +container = "" +action = "" +container_names = ["all", "calipso-mongo", "calipso-scan", "calipso-listen", "calipso-ldap", "calipso-api", + "calipso-sensu", "calipso-ui"] +container_actions = ["stop", "start"] +while action not in container_actions: + action = input("Action? (stop, start, or 'q' to quit):\n") + if action == "q": + exit() +while container not in container_names: + container = input("Container? (all, calipso-mongo, calipso-scan, calipso-listen, calipso-ldap, calipso-api, " + "calipso-sensu, calipso-ui or 'q' to quit):\n") + if container == "q": + exit() + +# starting the containers per arguments: +if action == "start": + # building /home/calipso/calipso_mongo_access.conf and /home/calipso/ldap.conf files, per the arguments: + calipso_mongo_access_text = "server " + args.hostname + "\nuser " + args.dbuser + "\npassword " + \ + args.dbpassword + "\nauth_db calipso" + ldap_text = "user admin" + "\npassword password" + "\nurl ldap://" + args.hostname + ":389" + \ + "\nuser_id_attribute CN" + "\nuser_pass_attribute userpassword" + \ + "\nuser_objectclass inetOrgPerson" + \ + "\nuser_tree_dn OU=Users,DC=openstack,DC=org" + "\nquery_scope one" + \ + "\ntls_req_cert allow" + \ + "\ngroup_member_attribute member" + print("creating default", H_MONGO_CONFIG, "file...\n") + calipso_mongo_access_file = open(H_MONGO_CONFIG, "w+") + time.sleep(1) + calipso_mongo_access_file.write(calipso_mongo_access_text) + calipso_mongo_access_file.close() + print("creating default", H_LDAP_CONFIG, "file...\n") + ldap_file = open(H_LDAP_CONFIG, "w+") + time.sleep(1) + ldap_file.write(ldap_text) + ldap_file.close() + + if container == "calipso-mongo" or container == "all": + start_mongo(args.dbport) + time.sleep(1) + if container == "calipso-listen" or container == "all": + start_listen() + time.sleep(1) + if container == "calipso-ldap" or container == "all": + start_ldap() + time.sleep(1) + if container == "calipso-api" or container == "all": + start_api() + time.sleep(1) + if container == "calipso-scan" or container == "all": + start_scan() + time.sleep(1) + if container == "calipso-sensu" or container == "all": + start_sensu() + time.sleep(1) + if container == "calipso-ui" or container == "all": + start_ui(args.hostname, args.dbuser, args.dbpassword, args.webport, args.dbport) + time.sleep(1) + +# stopping the containers per arguments: +if action == "stop": + if container == "calipso-mongo" or container == "all": + container_stop("calipso-mongo") + if container == "calipso-listen" or container == "all": + container_stop("calipso-listen") + if container == "calipso-ldap" or container == "all": + container_stop("calipso-ldap") + if container == "calipso-api" or container == "all": + container_stop("calipso-api") + if container == "calipso-scan" or container == "all": + container_stop("calipso-scan") + if container == "calipso-sensu" or container == "all": + container_stop("calipso-sensu") + if container == "calipso-ui" or container == "all": + container_stop("calipso-ui") -- cgit 1.2.3-korg