summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJustin chi <chigang@huawei.com>2017-11-03 08:43:12 +0000
committerGerrit Code Review <gerrit@opnfv.org>2017-11-03 08:43:12 +0000
commitd143e4895b1674f35738b579eec0704bc50cd5fa (patch)
tree4d9b3dbb9562843d60c22d7378589e55798409db
parentfce3a0b2765f1e521b7f8d31772f443849c2bb12 (diff)
parent8120b2583c459c85116d69211038e0110d71e5b7 (diff)
Merge "Add compass-tasks"
-rw-r--r--compass-tasks/Dockerfile12
-rw-r--r--compass-tasks/README.md1
-rw-r--r--compass-tasks/actions/__init__.py13
-rw-r--r--compass-tasks/actions/clean.py195
-rw-r--r--compass-tasks/actions/cli.py179
-rw-r--r--compass-tasks/actions/delete.py148
-rw-r--r--compass-tasks/actions/deploy.py182
-rw-r--r--compass-tasks/actions/health_check/__init__.py13
-rw-r--r--compass-tasks/actions/health_check/base.py57
-rw-r--r--compass-tasks/actions/health_check/check.py96
-rw-r--r--compass-tasks/actions/health_check/check_apache.py89
-rw-r--r--compass-tasks/actions/health_check/check_celery.py115
-rw-r--r--compass-tasks/actions/health_check/check_dhcp.py184
-rw-r--r--compass-tasks/actions/health_check/check_dns.py139
-rw-r--r--compass-tasks/actions/health_check/check_hds.py97
-rw-r--r--compass-tasks/actions/health_check/check_misc.py219
-rw-r--r--compass-tasks/actions/health_check/check_os_installer.py151
-rw-r--r--compass-tasks/actions/health_check/check_package_installer.py68
-rw-r--r--compass-tasks/actions/health_check/check_squid.py128
-rw-r--r--compass-tasks/actions/health_check/check_tftp.py96
-rw-r--r--compass-tasks/actions/health_check/utils.py114
-rw-r--r--compass-tasks/actions/install_callback.py181
-rw-r--r--compass-tasks/actions/patch.py69
-rw-r--r--compass-tasks/actions/poll_switch.py162
-rw-r--r--compass-tasks/actions/reinstall.py38
-rw-r--r--compass-tasks/actions/search.py46
-rw-r--r--compass-tasks/actions/update_progress.py298
-rw-r--r--compass-tasks/actions/util.py342
-rw-r--r--compass-tasks/apiclient/__init__.py0
-rwxr-xr-xcompass-tasks/apiclient/example.py463
-rw-r--r--compass-tasks/apiclient/restful.py1102
-rw-r--r--compass-tasks/apiclient/v1/__init__.py0
-rwxr-xr-xcompass-tasks/apiclient/v1/example.py305
-rw-r--r--compass-tasks/apiclient/v1/restful.py655
-rwxr-xr-xcompass-tasks/build.sh51
-rw-r--r--compass-tasks/db/__init__.py13
-rw-r--r--compass-tasks/db/api/__init__.py13
-rw-r--r--compass-tasks/db/api/adapter.py313
-rw-r--r--compass-tasks/db/api/adapter_holder.py155
-rw-r--r--compass-tasks/db/api/cluster.py2444
-rw-r--r--compass-tasks/db/api/database.py264
-rw-r--r--compass-tasks/db/api/health_check_report.py190
-rw-r--r--compass-tasks/db/api/host.py1120
-rw-r--r--compass-tasks/db/api/machine.py317
-rw-r--r--compass-tasks/db/api/metadata.py517
-rw-r--r--compass-tasks/db/api/metadata_holder.py731
-rw-r--r--compass-tasks/db/api/network.py160
-rw-r--r--compass-tasks/db/api/permission.py357
-rw-r--r--compass-tasks/db/api/switch.py1213
-rw-r--r--compass-tasks/db/api/user.py553
-rw-r--r--compass-tasks/db/api/user_log.py82
-rw-r--r--compass-tasks/db/api/utils.py1286
-rw-r--r--compass-tasks/db/callback.py204
-rw-r--r--compass-tasks/db/config_validation/__init__.py0
-rw-r--r--compass-tasks/db/config_validation/default_validator.py131
-rw-r--r--compass-tasks/db/config_validation/extension/__init__.py0
-rw-r--r--compass-tasks/db/config_validation/extension/openstack.py18
-rw-r--r--compass-tasks/db/exception.py116
-rw-r--r--compass-tasks/db/models.py1924
-rw-r--r--compass-tasks/db/v1/model.py724
-rw-r--r--compass-tasks/db/validator.py195
-rw-r--r--compass-tasks/deployment/__init__.py15
-rw-r--r--compass-tasks/deployment/deploy_manager.py237
-rw-r--r--compass-tasks/deployment/installers/__init__.py21
-rw-r--r--compass-tasks/deployment/installers/config_manager.py527
-rw-r--r--compass-tasks/deployment/installers/installer.py291
-rw-r--r--compass-tasks/deployment/installers/os_installers/__init__.py13
-rw-r--r--compass-tasks/deployment/installers/os_installers/cobbler/__init__.py13
-rw-r--r--compass-tasks/deployment/installers/os_installers/cobbler/cobbler.py449
-rw-r--r--compass-tasks/deployment/installers/pk_installers/__init__.py13
-rw-r--r--compass-tasks/deployment/installers/pk_installers/ansible_installer/__init__.py0
-rw-r--r--compass-tasks/deployment/installers/pk_installers/ansible_installer/ansible_installer.py441
-rw-r--r--compass-tasks/deployment/utils/__init__.py15
-rw-r--r--compass-tasks/deployment/utils/constants.py84
-rw-r--r--compass-tasks/hdsdiscovery/SNMP_CONFIG.md33
-rw-r--r--compass-tasks/hdsdiscovery/__init__.py13
-rw-r--r--compass-tasks/hdsdiscovery/base.py185
-rw-r--r--compass-tasks/hdsdiscovery/error.py26
-rw-r--r--compass-tasks/hdsdiscovery/hdmanager.py171
-rw-r--r--compass-tasks/hdsdiscovery/utils.py289
-rw-r--r--compass-tasks/hdsdiscovery/vendors/__init__.py13
-rw-r--r--compass-tasks/hdsdiscovery/vendors/appliance/__init__.py0
-rw-r--r--compass-tasks/hdsdiscovery/vendors/appliance/appliance.py34
-rw-r--r--compass-tasks/hdsdiscovery/vendors/appliance/plugins/__init__.py0
-rw-r--r--compass-tasks/hdsdiscovery/vendors/appliance/plugins/mac.py48
-rw-r--r--compass-tasks/hdsdiscovery/vendors/arista/__init__.py13
-rw-r--r--compass-tasks/hdsdiscovery/vendors/arista/arista.py33
-rw-r--r--compass-tasks/hdsdiscovery/vendors/arista/plugins/__init__.py13
-rw-r--r--compass-tasks/hdsdiscovery/vendors/arista/plugins/mac.py24
-rw-r--r--compass-tasks/hdsdiscovery/vendors/hp/__init__.py13
-rw-r--r--compass-tasks/hdsdiscovery/vendors/hp/hp.py33
-rw-r--r--compass-tasks/hdsdiscovery/vendors/hp/plugins/__init__.py13
-rw-r--r--compass-tasks/hdsdiscovery/vendors/hp/plugins/mac.py23
-rw-r--r--compass-tasks/hdsdiscovery/vendors/huawei/__init__.py13
-rw-r--r--compass-tasks/hdsdiscovery/vendors/huawei/huawei.py33
-rw-r--r--compass-tasks/hdsdiscovery/vendors/huawei/plugins/__init__.py13
-rw-r--r--compass-tasks/hdsdiscovery/vendors/huawei/plugins/mac.py63
-rw-r--r--compass-tasks/hdsdiscovery/vendors/ovswitch/__init__.py13
-rw-r--r--compass-tasks/hdsdiscovery/vendors/ovswitch/ovswitch.py76
-rw-r--r--compass-tasks/hdsdiscovery/vendors/ovswitch/plugins/__init__.py13
-rw-r--r--compass-tasks/hdsdiscovery/vendors/ovswitch/plugins/mac.py87
-rw-r--r--compass-tasks/hdsdiscovery/vendors/pica8/__init__.py13
-rw-r--r--compass-tasks/hdsdiscovery/vendors/pica8/pica8.py33
-rw-r--r--compass-tasks/hdsdiscovery/vendors/pica8/plugins/__init__.py13
-rw-r--r--compass-tasks/hdsdiscovery/vendors/pica8/plugins/mac.py24
-rw-r--r--compass-tasks/log_analyzor/__init__.py13
-rw-r--r--compass-tasks/log_analyzor/adapter_matcher.py126
-rw-r--r--compass-tasks/log_analyzor/environment.py29
-rw-r--r--compass-tasks/log_analyzor/file_matcher.py252
-rw-r--r--compass-tasks/log_analyzor/line_matcher.py206
-rw-r--r--compass-tasks/log_analyzor/progress_calculator.py208
-rw-r--r--compass-tasks/misc/Dockerfile53
-rw-r--r--compass-tasks/misc/compass_install.repo5
-rw-r--r--compass-tasks/requirements.txt23
-rw-r--r--compass-tasks/setup.py97
-rwxr-xr-xcompass-tasks/start.sh4
-rw-r--r--compass-tasks/supervisord.conf135
-rw-r--r--compass-tasks/tasks/__init__.py13
-rw-r--r--compass-tasks/tasks/client.py33
-rw-r--r--compass-tasks/tasks/tasks.py326
-rw-r--r--compass-tasks/utils/__init__.py13
-rw-r--r--compass-tasks/utils/celeryconfig_wrapper.py44
-rw-r--r--compass-tasks/utils/flags.py91
-rw-r--r--compass-tasks/utils/logsetting.py108
-rw-r--r--compass-tasks/utils/setting_wrapper.py175
-rw-r--r--compass-tasks/utils/util.py395
126 files changed, 24887 insertions, 0 deletions
diff --git a/compass-tasks/Dockerfile b/compass-tasks/Dockerfile
new file mode 100644
index 0000000..3beaf57
--- /dev/null
+++ b/compass-tasks/Dockerfile
@@ -0,0 +1,12 @@
+FROM centos:7
+
+ADD . /root/compass-tasks
+
+RUN /root/compass-tasks/build.sh
+
+EXPOSE 6379
+
+VOLUME ["/var/ansible", "/etc/compass/machine_list", "/etc/compass/switch_list"]
+
+ENTRYPOINT ["/bin/bash", "-c"]
+CMD ["/usr/local/bin/start.sh"]
diff --git a/compass-tasks/README.md b/compass-tasks/README.md
new file mode 100644
index 0000000..952f4b5
--- /dev/null
+++ b/compass-tasks/README.md
@@ -0,0 +1 @@
+# compass-tasks \ No newline at end of file
diff --git a/compass-tasks/actions/__init__.py b/compass-tasks/actions/__init__.py
new file mode 100644
index 0000000..4ee55a4
--- /dev/null
+++ b/compass-tasks/actions/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/compass-tasks/actions/clean.py b/compass-tasks/actions/clean.py
new file mode 100644
index 0000000..a4e9bc9
--- /dev/null
+++ b/compass-tasks/actions/clean.py
@@ -0,0 +1,195 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to clean installers
+"""
+import logging
+import xmlrpclib
+
+from compass.actions import util
+
+try:
+ import chef
+except ImportError:
+ pass
+
+class CobblerInstaller(object):
+ """cobbler installer"""
+ CREDENTIALS = "credentials"
+ USERNAME = 'username'
+ PASSWORD = 'password'
+
+ INSTALLER_URL = "cobbler_url"
+
+ def __init__(self, settings):
+ username = settings[self.CREDENTIALS][self.USERNAME]
+ password = settings[self.CREDENTIALS][self.PASSWORD]
+ cobbler_url = settings[self.INSTALLER_URL]
+ try:
+ self.remote = xmlrpclib.Server(cobbler_url)
+ self.token = self.remote.login(username, password)
+ logging.info('cobbler %s client created', cobbler_url)
+ except Exception as error:
+ logging.error(
+ 'failed to login %s with (%s, %s)',
+ cobbler_url, username, password
+ )
+ logging.exception(error)
+
+ def clean(self):
+ systems = self.remote.get_systems()
+ for system in systems:
+ system_name = system['name']
+ try:
+ self.remote.remove_system(system_name, self.token)
+ logging.info('system %s is removed', system_name)
+ except Exception as error:
+ logging.error(
+ 'failed to remove system %s', system_name
+ )
+ logging.exception(error)
+
+
+class AnsibleInstaller(object):
+
+ def __init__(self, settings):
+ return
+
+ def clean(self):
+ pass
+
+
+class ChefInstaller(object):
+ DATABAGS = "databags"
+ CHEFSERVER_URL = "chef_url"
+ CHEFSERVER_DNS = "chef_server_dns"
+ CHEFSERVER_IP = "chef_server_ip"
+ KEY_DIR = "key_dir"
+ CLIENT = "client_name"
+
+ def __init__(self, settings):
+ installer_url = settings.get(self.CHEFSERVER_URL, None)
+ key_dir = settings.get(self.KEY_DIR, None)
+ client = settings.get(self.CLIENT, None)
+ try:
+ if installer_url and key_dir and client:
+ self.api = chef.ChefAPI(installer_url, key_dir, client)
+ else:
+ self.api = chef.autoconfigure()
+ logging.info(
+ 'chef client created %s(%s, %s)',
+ installer_url, key_dir, client
+ )
+ except Exception as error:
+ logging.error(
+ 'failed to create chef client %s(%s, %s)',
+ installer_url, key_dir, client
+ )
+ logging.exception(error)
+
+ def clean(self):
+ try:
+ for node_name in chef.Node.list(api=self.api):
+ node = chef.Node(node_name, api=self.api)
+ node.delete()
+ logging.info('delete node %s', node_name)
+ except Exception as error:
+ logging.error('failed to delete some nodes')
+ logging.exception(error)
+
+ try:
+ for client_name in chef.Client.list(api=self.api):
+ if client_name in ['chef-webui', 'chef-validator']:
+ continue
+ client = chef.Client(client_name, api=self.api)
+ client.delete()
+ logging.info('delete client %s', client_name)
+ except Exception as error:
+ logging.error('failed to delete some clients')
+ logging.exception(error)
+
+ try:
+ for env_name in chef.Environment.list(api=self.api):
+ if env_name == '_default':
+ continue
+ env = chef.Environment(env_name, api=self.api)
+ env.delete()
+ logging.info('delete env %s', env_name)
+ except Exception as error:
+ logging.error('failed to delete some envs')
+ logging.exception(error)
+
+ try:
+ for databag_name in chef.DataBag.list(api=self.api):
+ databag = chef.DataBag(databag_name, api=self.api)
+ for item_name, item in databag.items():
+ item.delete()
+ logging.info(
+ 'delete item %s from databag %s',
+ item_name, databag_name
+ )
+ except Exception as error:
+ logging.error('failed to delete some databag items')
+ logging.exception(error)
+
+
+OS_INSTALLERS = {
+ 'cobbler': CobblerInstaller
+}
+PK_INSTALLERS = {
+ 'chef_installer': ChefInstaller,
+ 'ansible_installer': AnsibleInstaller
+}
+
+
+def clean_os_installer(
+ os_installer_name, os_installer_settings
+):
+ with util.lock('serialized_action', timeout=100) as lock:
+ if not lock:
+ raise Exception(
+ 'failed to acquire lock to clean os installer'
+ )
+
+ if os_installer_name not in OS_INSTALLERS:
+ logging.error(
+ '%s not found in os_installers',
+ os_installer_name
+ )
+
+ os_installer = OS_INSTALLERS[os_installer_name](
+ os_installer_settings
+ )
+ os_installer.clean()
+
+
+def clean_package_installer(
+ package_installer_name, package_installer_settings
+):
+ with util.lock('serialized_action', timeout=100) as lock:
+ if not lock:
+ raise Exception(
+ 'failed to acquire lock to clean package installer'
+ )
+
+ if package_installer_name not in PK_INSTALLERS:
+ logging.error(
+ '%s not found in os_installers',
+ package_installer_name
+ )
+
+ package_installer = PK_INSTALLERS[package_installer_name](
+ package_installer_settings
+ )
+ package_installer.clean()
diff --git a/compass-tasks/actions/cli.py b/compass-tasks/actions/cli.py
new file mode 100644
index 0000000..c9058ed
--- /dev/null
+++ b/compass-tasks/actions/cli.py
@@ -0,0 +1,179 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Compass Command Line Interface"""
+import logging
+import subprocess
+import sys
+
+from compass.actions.health_check import check
+from compass.db.api import database
+
+from compass.utils import flags
+from compass.utils import logsetting
+from compass.utils import setting_wrapper as setting
+from compass.utils.util import pretty_print
+
+
+ACTION_MAP = {
+ "check": "apache celery dhcp dns hds misc os_installer "
+ "package_installer squid tftp".split(" "),
+ "refresh": "db sync".split(" "),
+}
+
+
+class BootCLI(object):
+ """CLI to do compass check."""
+
+ def __init__(self):
+ return
+
+ def run(self, args):
+ """cli takes the commands and calls respective modules."""
+ action = self.get_action(args)
+ if action is None:
+ self.print_help()
+ else:
+ module = self.get_module(action, args)
+ if module == "invalid":
+ self.print_help(action)
+ else:
+ method = "self.run_" + action + "(module)"
+ eval(method)
+
+ @classmethod
+ def get_action(cls, args):
+ """This method returns an action type.
+
+ .. note::
+ For 'compass check dhcp' command, it will return 'check'.
+ """
+ if len(args) == 1:
+ return None
+ elif args[1] in ACTION_MAP.keys():
+ return args[1]
+ return None
+
+ @classmethod
+ def get_module(cls, action, args):
+ """This method returns a module.
+
+ .. note::
+ For 'compass check dhcp' command, it will return 'dhcp'.
+ """
+ if len(args) <= 2:
+ return None
+ elif args[2] in ACTION_MAP[action]:
+ return args[2]
+ return "invalid"
+
+ def run_check(self, module=None):
+ """This provides a flexible sanity check.
+
+ .. note::
+ param module default set to None.
+ if parameter module is none. Compass checks all modules.
+ If module specified, Compass will only check such module.
+ """
+ if module is None:
+ pretty_print("Starting: Compass Health Check",
+ "==============================")
+ chk = check.BootCheck()
+ res = chk.run()
+ self.output_check_result(res)
+
+ else:
+ pretty_print("Checking Module: %s" % module,
+ "============================")
+ chk = check.BootCheck()
+ method = "chk._check_" + module + "()"
+ res = eval(method)
+ print "\n".join(msg for msg in res[1])
+
+ @classmethod
+ def output_check_result(cls, result):
+ """output check result."""
+ if result == {}:
+ return
+ pretty_print("\n",
+ "===============================",
+ "* Compass Health Check Report *",
+ "===============================")
+ successful = True
+ for key in result.keys():
+ if result[key][0] == 0:
+ successful = False
+ print "%s" % "\n".join(item for item in result[key][1])
+
+ print "===================="
+ if successful is True:
+ print "Compass Check completes. No problems found, all systems go"
+ sys.exit(0)
+ else:
+ print (
+ "Compass has ERRORS shown above. Please fix them before "
+ "deploying!")
+ sys.exit(1)
+
+ @classmethod
+ def run_refresh(cls, action=None):
+ """Run refresh."""
+ # TODO(xicheng): replace refresh.sh with refresh.py
+ if action is None:
+ pretty_print("Refreshing Compass...",
+ "=================")
+ subprocess.Popen(
+ ['/opt/compass/bin/refresh.sh'], shell=True)
+ elif action == "db":
+ pretty_print("Refreshing Compass Database...",
+ "===================")
+ subprocess.Popen(
+ ['/opt/compass/bin/manage_db.py createdb'], shell=True)
+ else:
+ pretty_print("Syncing with Installers...",
+ "================")
+ subprocess.Popen(
+ ['/opt/compass/bin/manage_db.py sync_from_installers'],
+ shell=True
+ )
+
+ @classmethod
+ def print_help(cls, module_help=""):
+ """print help."""
+ if module_help == "":
+ pretty_print("usage\n=====",
+ "compass <refresh|check>",
+ "type 'compass {action} --help' for detailed "
+ "command list")
+
+ elif module_help == "refresh":
+ pretty_print("usage\n=====",
+ "compass refresh [%s]" %
+ "|".join(action for action in ACTION_MAP['refresh']))
+
+ else:
+ pretty_print("usage\n=====",
+ "compass check [%s]" %
+ "|".join(action for action in ACTION_MAP['check']))
+ sys.exit(2)
+
+
+def main():
+ """Compass cli entry point."""
+ flags.init()
+ logsetting.init()
+ database.init()
+ cli = BootCLI()
+ output = cli.run(sys.argv)
+ return sys.exit(output)
diff --git a/compass-tasks/actions/delete.py b/compass-tasks/actions/delete.py
new file mode 100644
index 0000000..d89994d
--- /dev/null
+++ b/compass-tasks/actions/delete.py
@@ -0,0 +1,148 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to delete a given cluster
+"""
+import logging
+
+from compass.actions import util
+from compass.db.api import cluster as cluster_api
+from compass.db.api import host as host_api
+from compass.db.api import user as user_db
+from compass.deployment.deploy_manager import DeployManager
+from compass.deployment.utils import constants as const
+
+
+def delete_cluster(
+ cluster_id, host_id_list,
+ username=None, delete_underlying_host=False
+):
+ """Delete cluster and all clusterhosts on it.
+
+ :param cluster_id: id of the cluster.
+ :type cluster_id: int
+ :param host_id_list: list of host id.
+ :type host_id_list: list of int.
+
+ If delete_underlying_host is set, all underlying hosts will
+ be deleted.
+
+ .. note::
+ The function should be called out of database session.
+ """
+ with util.lock('serialized_action', timeout=100) as lock:
+ if not lock:
+ raise Exception('failed to acquire lock to delete cluster')
+
+ user = user_db.get_user_object(username)
+
+ cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
+ adapter_id = cluster_info[const.ADAPTER_ID]
+
+ adapter_info = util.ActionHelper.get_adapter_info(
+ adapter_id, cluster_id, user)
+ hosts_info = util.ActionHelper.get_hosts_info(
+ cluster_id, host_id_list, user)
+
+ deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info)
+
+ deploy_manager.remove_hosts(
+ package_only=not delete_underlying_host,
+ delete_cluster=True
+ )
+ util.ActionHelper.delete_cluster(
+ cluster_id, host_id_list, user,
+ delete_underlying_host
+ )
+
+
+def delete_cluster_host(
+ cluster_id, host_id,
+ username=None, delete_underlying_host=False
+):
+ """Delete clusterhost.
+
+ :param cluster_id: id of the cluster.
+ :type cluster_id: int
+ :param host_id: id of the host.
+ :type host_id: int
+
+ If delete_underlying_host is set, the underlying host
+ will be deleted too.
+
+ .. note::
+ The function should be called out of database session.
+ """
+ with util.lock('serialized_action', timeout=100) as lock:
+ if not lock:
+ raise Exception('failed to acquire lock to delete clusterhost')
+
+ user = user_db.get_user_object(username)
+ cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
+ adapter_id = cluster_info[const.ADAPTER_ID]
+
+ adapter_info = util.ActionHelper.get_adapter_info(
+ adapter_id, cluster_id, user)
+ hosts_info = util.ActionHelper.get_hosts_info(
+ cluster_id, [host_id], user)
+
+ deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info)
+
+ deploy_manager.remove_hosts(
+ package_only=not delete_underlying_host,
+ delete_cluster=False
+ )
+ util.ActionHelper.delete_cluster_host(
+ cluster_id, host_id, user,
+ delete_underlying_host
+ )
+
+
+def delete_host(
+ host_id, cluster_id_list, username=None
+):
+ """Delete host and all clusterhosts on it.
+
+ :param host_id: id of the host.
+ :type host_id: int
+
+ .. note::
+ The function should be called out of database session.
+ """
+ with util.lock('serialized_action', timeout=100) as lock:
+ if not lock:
+ raise Exception('failed to acquire lock to delete host')
+
+ user = user_db.get_user_object(username)
+ for cluster_id in cluster_id_list:
+ cluster_info = util.ActionHelper.get_cluster_info(
+ cluster_id, user)
+ adapter_id = cluster_info[const.ADAPTER_ID]
+
+ adapter_info = util.ActionHelper.get_adapter_info(
+ adapter_id, cluster_id, user)
+ hosts_info = util.ActionHelper.get_hosts_info(
+ cluster_id, [host_id], user)
+
+ deploy_manager = DeployManager(
+ adapter_info, cluster_info, hosts_info)
+
+ deploy_manager.remove_hosts(
+ package_only=True,
+ delete_cluster=False
+ )
+
+ util.ActionHelper.delete_host(
+ host_id, user
+ )
diff --git a/compass-tasks/actions/deploy.py b/compass-tasks/actions/deploy.py
new file mode 100644
index 0000000..53179f5
--- /dev/null
+++ b/compass-tasks/actions/deploy.py
@@ -0,0 +1,182 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to deploy a given cluster
+"""
+import logging
+
+from compass.actions import util
+from compass.db.api import cluster as cluster_db
+from compass.db.api import health_check_report as health_check_db
+from compass.db.api import user as user_db
+from compass.deployment.deploy_manager import DeployManager
+from compass.deployment.utils import constants as const
+
+
+def deploy(cluster_id, hosts_id_list, username=None):
+ """Deploy clusters.
+
+ :param cluster_hosts: clusters and hosts in each cluster to deploy.
+ :type cluster_hosts: dict of int or str to list of int or str
+
+ .. note::
+ The function should be called out of database session.
+ """
+ with util.lock('serialized_action', timeout=1000) as lock:
+ if not lock:
+ raise Exception('failed to acquire lock to deploy')
+
+ user = user_db.get_user_object(username)
+
+ cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
+ adapter_id = cluster_info[const.ADAPTER_ID]
+
+ adapter_info = util.ActionHelper.get_adapter_info(
+ adapter_id, cluster_id, user)
+ hosts_info = util.ActionHelper.get_hosts_info(
+ cluster_id, hosts_id_list, user)
+
+ deploy_successful = True
+ try:
+ deploy_manager = DeployManager(
+ adapter_info, cluster_info, hosts_info)
+ # deploy_manager.prepare_for_deploy()
+ logging.debug('Created deploy manager with %s %s %s'
+ % (adapter_info, cluster_info, hosts_info))
+ deployed_config = deploy_manager.deploy()
+ except Exception as error:
+ logging.exception(error)
+ deploy_successful = False
+
+ if deploy_successful:
+ util.ActionHelper.save_deployed_config(deployed_config, user)
+ util.ActionHelper.update_state(
+ cluster_id, hosts_id_list, user, state='INSTALLING'
+ )
+ else:
+ util.ActionHelper.update_state(
+ cluster_id, hosts_id_list, user, state='ERROR',
+ message='failed to start deployment', severity='ERROR'
+ )
+
+
+def redeploy(cluster_id, username=None):
+ """Deploy clusters.
+
+ :param cluster_hosts: clusters and hosts in each cluster to deploy.
+ :type cluster_hosts: dict of int or str to list of int or str
+ """
+ with util.lock('serialized_action') as lock:
+ if not lock:
+ raise Exception('failed to acquire lock to deploy')
+
+ user = user_db.get_user_object(username)
+ cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
+ adapter_id = cluster_info[const.ADAPTER_ID]
+
+ adapter_info = util.ActionHelper.get_adapter_info(
+ adapter_id, cluster_id, user)
+
+ cluster_hosts = cluster_db.list_cluster_hosts(cluster_id, user)
+ hosts_id_list = [host['id'] for host in cluster_hosts]
+
+ hosts_info = util.ActionHelper.get_hosts_info(
+ cluster_id, hosts_id_list, user)
+
+ deploy_successful = True
+ try:
+ deploy_manager = DeployManager(
+ adapter_info, cluster_info, hosts_info)
+ # deploy_manager.prepare_for_deploy()
+ deploy_manager.redeploy()
+ except Exception as error:
+ logging.exception(error)
+ deploy_successful = False
+ if deploy_successful:
+ util.ActionHelper.update_state(
+ cluster_id, hosts_id_list, user, state='INSTALLING',
+ )
+ else:
+ util.ActionHelper.update_state(
+ cluster_id, hosts_id_list, user, state='ERROR',
+ message='failed to start redeployment', severity='ERROR'
+ )
+
+
+def health_check(cluster_id, report_uri, username):
+ with util.lock('cluster_health_check') as lock:
+ if not lock:
+ raise Exception('failed to acquire lock to check health')
+
+ user = user_db.get_user_object(username)
+ cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
+ adapter_id = cluster_info[const.ADAPTER_ID]
+
+ adapter_info = util.ActionHelper.get_adapter_info(
+ adapter_id, cluster_id, user
+ )
+
+ deploy_manager = DeployManager(adapter_info, cluster_info, None)
+ try:
+ deploy_manager.check_cluster_health(report_uri)
+ except Exception as exc:
+ logging.error("health_check exception: ============= %s" % exc)
+ data = {'state': 'error', 'error_message': str(exc), 'report': {}}
+ reports = health_check_db.list_health_reports(
+ cluster_id, user=user)
+ if not reports:
+ # Exception before executing command remotely for health check.
+ # No reports names sending back yet. Create a report
+ name = 'pre_remote_health_check'
+ health_check_db.add_report_record(
+ cluster_id, name, user=user, **data
+ )
+
+ health_check_db.update_multi_reports(cluster_id, user=user, **data)
+
+
+class ServerPowerMgmt(object):
+ """Power management for bare-metal machines by IPMI command."""
+ @staticmethod
+ def poweron(machine_id, user):
+ """Power on the specified machine."""
+ pass
+
+ @staticmethod
+ def poweroff(machine_id, user):
+ pass
+
+ @staticmethod
+ def reset(machine_id, user):
+ pass
+
+
+class HostPowerMgmt(object):
+ """Power management for hosts installed OS by OS installer. OS installer
+
+ will poweron/poweroff/reset host.
+
+ """
+ @staticmethod
+ def poweron(host_id, user):
+ """Power on the specified host."""
+ pass
+
+ @staticmethod
+ def poweroff(host_id, user):
+ pass
+
+ @staticmethod
+ def reset(host_id, user):
+ pass
diff --git a/compass-tasks/actions/health_check/__init__.py b/compass-tasks/actions/health_check/__init__.py
new file mode 100644
index 0000000..4ee55a4
--- /dev/null
+++ b/compass-tasks/actions/health_check/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/compass-tasks/actions/health_check/base.py b/compass-tasks/actions/health_check/base.py
new file mode 100644
index 0000000..22b6fae
--- /dev/null
+++ b/compass-tasks/actions/health_check/base.py
@@ -0,0 +1,57 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Base class for Compass Health Check."""
+from compass.actions.health_check import utils as health_check_utils
+from compass.db.api import adapter as adapter_api
+from compass.utils import setting_wrapper as setting
+
+
+class BaseCheck(object):
+ """health check base class."""
+
+ def __init__(self):
+ self.config = setting
+ self.code = 1
+ self.messages = []
+ self.dist, self.version, self.release = health_check_utils.get_dist()
+ adapter_api.load_adapters_internal()
+ self.os_installer = self._get_os_installer()
+ self.package_installer = self._get_package_installer()
+
+ def _get_os_installer(self):
+ installer = adapter_api.OS_INSTALLERS.values()[0]
+ os_installer = {}
+ os_installer['name'] = health_check_utils.strip_name(
+ installer['name'])
+ os_installer.update(installer['settings'])
+ return os_installer
+
+ def _get_package_installer(self):
+ package_installer = {}
+ installer = adapter_api.PACKAGE_INSTALLERS.values()[0]
+ package_installer = {}
+ package_installer['name'] = health_check_utils.strip_name(
+ installer['name'])
+ package_installer.update(installer['settings'])
+ return package_installer
+
+ def _set_status(self, code, message):
+ """set status."""
+ self.code = code
+ self.messages.append(message)
+
+ def get_status(self):
+ """get status."""
+ return (self.code, self.messages)
diff --git a/compass-tasks/actions/health_check/check.py b/compass-tasks/actions/health_check/check.py
new file mode 100644
index 0000000..c1adbc6
--- /dev/null
+++ b/compass-tasks/actions/health_check/check.py
@@ -0,0 +1,96 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Main Entry Point of Compass Health Check."""
+from compass.actions.health_check import base
+from compass.actions.health_check import check_apache
+from compass.actions.health_check import check_celery
+from compass.actions.health_check import check_dhcp
+from compass.actions.health_check import check_dns
+from compass.actions.health_check import check_hds
+from compass.actions.health_check import check_misc
+from compass.actions.health_check import check_os_installer
+from compass.actions.health_check import check_package_installer
+from compass.actions.health_check import check_squid
+from compass.actions.health_check import check_tftp
+
+
+class BootCheck(base.BaseCheck):
+ """health check for all components."""
+
+ def run(self):
+ """do health check."""
+ status = {}
+ status['apache'] = self._check_apache()
+ status['celery'] = self._check_celery()
+ status['dhcp'] = self._check_dhcp()
+ status['dns'] = self._check_dns()
+ status['hds'] = self._check_hds()
+ status['os_installer'] = self._check_os_installer()
+ status['package_installer'] = self._check_package_installer()
+ status['squid'] = self._check_squid()
+ status['tftp'] = self._check_tftp()
+ status['other'] = self._check_misc()
+
+ return status
+
+ def _check_apache(self):
+ """do apache health check."""
+ checker = check_apache.ApacheCheck()
+ return checker.run()
+
+ def _check_celery(self):
+ """do celery health check."""
+ checker = check_celery.CeleryCheck()
+ return checker.run()
+
+ def _check_dhcp(self):
+ """do dhcp health check."""
+ checker = check_dhcp.DhcpCheck()
+ return checker.run()
+
+ def _check_dns(self):
+ """do dns health check."""
+ checker = check_dns.DnsCheck()
+ return checker.run()
+
+ def _check_hds(self):
+ """do hds health check."""
+ checker = check_hds.HdsCheck()
+ return checker.run()
+
+ def _check_os_installer(self):
+ """do os installer health check."""
+ checker = check_os_installer.OsInstallerCheck()
+ return checker.run()
+
+ def _check_package_installer(self):
+ """do package installer health check."""
+ checker = check_package_installer.PackageInstallerCheck()
+ return checker.run()
+
+ def _check_squid(self):
+ """do squid health check."""
+ checker = check_squid.SquidCheck()
+ return checker.run()
+
+ def _check_tftp(self):
+ """do tftp health check."""
+ checker = check_tftp.TftpCheck()
+ return checker.run()
+
+ def _check_misc(self):
+ """do misc health check."""
+ checker = check_misc.MiscCheck()
+ return checker.run()
diff --git a/compass-tasks/actions/health_check/check_apache.py b/compass-tasks/actions/health_check/check_apache.py
new file mode 100644
index 0000000..294d6f9
--- /dev/null
+++ b/compass-tasks/actions/health_check/check_apache.py
@@ -0,0 +1,89 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Health Check module for Apache service."""
+
+import socket
+import urllib2
+
+from compass.actions.health_check import base
+from compass.actions.health_check import utils as health_check_utils
+
+
+class ApacheCheck(base.BaseCheck):
+ """apache server health check class."""
+ NAME = "Apache Check"
+
+ def run(self):
+ """do the healthcheck."""
+ if self.dist in ("centos", "redhat", "fedora", "scientific linux"):
+ apache_service = 'httpd'
+ else:
+ apache_service = 'apache2'
+ self.check_apache_conf(apache_service)
+ print "[Done]"
+ self.check_apache_running(apache_service)
+ print "[Done]"
+ if self.code == 1:
+ self.messages.append(
+ "[%s]Info: Apache health check has completed. "
+ "No problems found, all systems go." % self.NAME)
+ return (self.code, self.messages)
+
+ def check_apache_conf(self, apache_service):
+ """Validates if Apache settings.
+
+ :param apache_service : service type of apache, os dependent.
+ e.g. httpd or apache2
+ :type apache_service : string
+
+ """
+ print "Checking Apache Config......",
+ conf_err_msg = health_check_utils.check_path(
+ self.NAME,
+ "/etc/%s/conf.d/ods-server.conf" % apache_service)
+ if not conf_err_msg == "":
+ self._set_status(0, conf_err_msg)
+
+ wsgi_err_msg = health_check_utils.check_path(
+ self.NAME,
+ '/var/www/compass/compass.wsgi')
+ if not wsgi_err_msg == "":
+ self._set_status(0, wsgi_err_msg)
+
+ return True
+
+ def check_apache_running(self, apache_service):
+ """Checks if Apache service is running on port 80."""
+
+ print "Checking Apache service......",
+ serv_err_msg = health_check_utils.check_service_running(self.NAME,
+ apache_service)
+ if not serv_err_msg == "":
+ self._set_status(0, serv_err_msg)
+ if 'http' != socket.getservbyport(80):
+ self._set_status(
+ 0,
+ "[%s]Error: Apache is not listening on port 80."
+ % self.NAME)
+ try:
+ html = urllib2.urlopen('http://localhost')
+ html.geturl()
+ except Exception:
+ self._set_status(
+ 0,
+ "[%s]Error: Apache is not listening on port 80."
+ % self.NAME)
+
+ return True
diff --git a/compass-tasks/actions/health_check/check_celery.py b/compass-tasks/actions/health_check/check_celery.py
new file mode 100644
index 0000000..2d8d27c
--- /dev/null
+++ b/compass-tasks/actions/health_check/check_celery.py
@@ -0,0 +1,115 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Health Check module for Celery."""
+import commands
+import os
+
+from celery.task.control import inspect
+
+from compass.actions.health_check import base
+from compass.actions.health_check import utils as health_check_utils
+
+
+class CeleryCheck(base.BaseCheck):
+ """celery health check class."""
+ NAME = "Celery Check."
+
+ def run(self):
+ """do health check."""
+ self.check_compass_celery_setting()
+ print "[Done]"
+ self.check_celery_backend()
+ print "[Done]"
+ if self.code == 1:
+ self.messages.append("[%s]Info: Celery health check "
+ "has completed. No problems found, "
+ "all systems go." % self.NAME)
+ return (self.code, self.messages)
+
+ def check_compass_celery_setting(self):
+ """Validates Celery settings."""
+
+ print "Checking Celery setting......",
+ setting_map = {
+ 'logfile': 'CELERY_LOGFILE',
+ 'configdir': 'CELERYCONFIG_DIR',
+ 'configfile': 'CELERYCONFIG_FILE',
+ }
+ unset = []
+ res = health_check_utils.validate_setting('Celery',
+ self.config,
+ 'CELERY_LOGFILE')
+ if res is False:
+ unset.append(setting_map["logfile"])
+ self._set_status(0, res)
+
+ res = health_check_utils.validate_setting('Celery',
+ self.config,
+ 'CELERYCONFIG_DIR')
+ if res is False:
+ unset.append(setting_map["configdir"])
+ self._set_status(0, res)
+
+ res = health_check_utils.validate_setting('Celery',
+ self.config,
+ 'CELERYCONFIG_FILE')
+ if res is False:
+ unset.append(setting_map["configdir"])
+ self._set_status(0, res)
+
+ if len(unset) != 0:
+ self._set_status(0,
+ "[%s]Error: Unset celery settings: %s"
+ " in /etc/compass/setting"
+ % (self.NAME, ', '.join(item for item in unset)))
+ return True
+
+ def check_celery_backend(self):
+ """Checks if Celery backend is running and configured properly."""
+
+ print "Checking Celery Backend......",
+ if 'celery worker' not in commands.getoutput('ps -ef'):
+ self._set_status(0, "[%s]Error: celery is not running" % self.NAME)
+ return True
+
+ if not os.path.exists('/etc/compass/celeryconfig'):
+ self._set_status(
+ 0,
+ "[%s]Error: No celery config file found for Compass"
+ % self.NAME)
+ return True
+
+ try:
+ insp = inspect()
+ celery_stats = inspect.stats(insp)
+ if not celery_stats:
+ self._set_status(
+ 0,
+ "[%s]Error: No running Celery workers were found."
+ % self.NAME)
+ except IOError as error:
+ self._set_status(
+ 0,
+ "[%s]Error: Failed to connect to the backend: %s"
+ % (self.NAME, str(error)))
+ from errno import errorcode
+ if (
+ len(error.args) > 0 and
+ errorcode.get(error.args[0]) == 'ECONNREFUSED'
+ ):
+ self.messages.append(
+ "[%s]Error: RabbitMQ server isn't running"
+ % self.NAME)
+ return True
diff --git a/compass-tasks/actions/health_check/check_dhcp.py b/compass-tasks/actions/health_check/check_dhcp.py
new file mode 100644
index 0000000..e3bae1e
--- /dev/null
+++ b/compass-tasks/actions/health_check/check_dhcp.py
@@ -0,0 +1,184 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Health Check module for DHCP service."""
+import commands
+import os
+import re
+import socket
+import xmlrpclib
+
+from compass.actions.health_check import base
+
+
+class DhcpCheck(base.BaseCheck):
+ """dhcp health check class."""
+
+ NAME = "DHCP Check"
+
+ def run(self):
+ """do health check."""
+ method_name = "self.check_" + self.os_installer['name'] + "_dhcp()"
+ return eval(method_name)
+
+ def check_cobbler_dhcp(self):
+ """Checks if Cobbler has taken over DHCP service."""
+
+ try:
+ remote = xmlrpclib.Server(
+ self.os_installer['cobbler_url'],
+ allow_none=True)
+ credentials = self.os_installer['credentials']
+ remote.login(
+ credentials['username'], credentials['password'])
+ except Exception:
+ self._set_status(
+ 0,
+ "[%s]Error: Cannot login to Cobbler with "
+ "the tokens provided in the config file" % self.NAME)
+ return (self.code, self.messages)
+
+ cobbler_settings = remote.get_settings()
+ if cobbler_settings['manage_dhcp'] == 0:
+ self.messages.append(
+ "[%s]Info: DHCP service is "
+ "not managed by Compass" % self.NAME)
+ self.code = 0
+ return (self.code, self.messages)
+
+ self.check_cobbler_dhcp_template()
+ print "[Done]"
+ self.check_dhcp_service()
+ self.check_dhcp_netmask()
+ print "[Done]"
+ if self.code == 1:
+ self.messages.append(
+ "[%s]Info: DHCP health check has completed. "
+ "No problems found, all systems go." % self.NAME)
+
+ return (self.code, self.messages)
+
+ def check_cobbler_dhcp_template(self):
+ """Validates Cobbler's DHCP template file."""
+ print "Checking DHCP template......",
+ if os.path.exists("/etc/cobbler/dhcp.template"):
+ var_map = {
+ "match_next_server": False,
+ "match_subnet": False,
+ "match_filename": False,
+ "match_range": False,
+ }
+
+ ip_regex = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$')
+
+ dhcp_template = open("/etc/cobbler/dhcp.template")
+ for line in dhcp_template.readlines():
+ if line.find("next_server") != -1:
+ elmlist = line.split(" ")
+ for elm in elmlist:
+ if ";" in elm:
+ elm = elm[:-2]
+
+ if "$next_server" in elm or ip_regex.match(elm):
+ var_map["match_next_server"] = True
+
+ elif line.find("subnet") != -1 and line.find("{") != -1:
+ elmlist = line.split(" ")
+ for elm in elmlist:
+ if ip_regex.match(elm):
+ if elm[-1] == "0" and "255" not in elm:
+ var_map["match_subnet"] = True
+ elif elm[-1] != "0":
+ self.messages.append(
+ "[%s]Error: Subnet should be set "
+ "in the form of 192.168.0.0 in"
+ "/etc/cobbler/dhcp.template" % self.NAME)
+
+ elif line.find("filename") != -1:
+ var_map["match_filename"] = True
+ elif line.find("range dynamic-bootp") != -1:
+ elmlist = line.split(" ")
+ ip_count = 0
+ for elm in elmlist:
+ if ";" in elm and "\n" in elm:
+ elm = elm[:-2]
+
+ if ip_regex.match(elm):
+ ip_count += 1
+
+ if ip_count != 2:
+ self.messages.append(
+ "[%s]Error: DHCP range should be set "
+ "between two IP addresses in "
+ "/etc/cobbler/dhcp.template" % self.NAME)
+ else:
+ var_map["match_range"] = True
+
+ dhcp_template.close()
+ fails = []
+ for var in var_map.keys():
+ if var_map[var] is False:
+ fails.append(var)
+
+ if len(fails) != 0:
+ self._set_status(
+ 0,
+ "[%s]Info: DHCP template file "
+ "failed components: %s" % (
+ self.NAME, ' '.join(failed for failed in fails)))
+
+ else:
+ self._set_status(
+ 0,
+ "[%s]Error: DHCP template file doesn't exist, "
+ "health check failed." % self.NAME)
+
+ return True
+
+ def check_dhcp_netmask(self):
+ with open('/etc/dhcp/dhcpd.conf') as conf_reader:
+ lines = conf_reader.readlines()
+ for line in lines:
+ if re.search('^subnet', line):
+ elm_list = line.split(' ')
+ break
+ subnet_ip = elm_list[1]
+ netmask = elm_list[-2]
+ subnet_ip_elm = subnet_ip.split('.')
+ netmask_elm = netmask.split('.')
+ for index, digit in enumerate(subnet_ip_elm):
+ if int(digit) & int(netmask_elm[index]) != int(digit):
+ self._set_status(
+ 0,
+ "[%s]Info: DHCP subnet IP and "
+ "netmask do not match" % self.NAME)
+ break
+ return True
+
+ def check_dhcp_service(self):
+ """Checks if DHCP is running on port 67."""
+ print "Checking DHCP service......",
+ if not commands.getoutput('pgrep dhcp'):
+ self._set_status(
+ 0,
+ "[%s]Error: dhcp service does not "
+ "seem to be running" % self.NAME)
+
+ if socket.getservbyport(67) != 'bootps':
+ self._set_status(
+ 0,
+ "[%s]Error: bootps is not listening "
+ "on port 67" % self.NAME)
+
+ return True
diff --git a/compass-tasks/actions/health_check/check_dns.py b/compass-tasks/actions/health_check/check_dns.py
new file mode 100644
index 0000000..843d7e2
--- /dev/null
+++ b/compass-tasks/actions/health_check/check_dns.py
@@ -0,0 +1,139 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Health Check module for DNS service."""
+
+import commands
+import os
+import socket
+import xmlrpclib
+
+from compass.actions.health_check import base
+
+
+class DnsCheck(base.BaseCheck):
+ """dns health check class."""
+ NAME = "DNS Check"
+
+ def run(self):
+ """do health check."""
+ method_name = "self.check_" + self.os_installer['name'] + "_dns()"
+ return eval(method_name)
+
+ def check_cobbler_dns(self):
+ """Checks if Cobbler has taken over DNS service."""
+ try:
+ remote = xmlrpclib.Server(
+ self.os_installer['cobbler_url'],
+ allow_none=True)
+ credentials = self.os_installer['credentials']
+ remote.login(
+ credentials['username'], credentials['password'])
+ except Exception:
+ self._set_status(0,
+ "[%s]Error: Cannot login to Cobbler "
+ "with the tokens provided in the config file"
+ % self.NAME)
+ return (self.code, self.messages)
+
+ cobbler_settings = remote.get_settings()
+ if cobbler_settings['manage_dns'] == 0:
+ self.messages.append('[DNS]Info: DNS is not managed by Compass')
+ return (0, self.messages)
+ self.check_cobbler_dns_template()
+ print "[Done]"
+ self.check_dns_service()
+ print "[Done]"
+ if self.code == 1:
+ self.messages.append(
+ "[%s]Info: DNS health check has complated. "
+ "No problems found, all systems go." % self.NAME)
+ return (self.code, self.messages)
+
+ def check_cobbler_dns_template(self):
+ """Validates Cobbler's DNS template file."""
+
+ print "Checking DNS template......",
+ if os.path.exists("/etc/cobbler/named.template"):
+ var_map = {
+ "match_port": False,
+ "match_allow_query": False,
+ }
+ named_template = open("/etc/cobbler/named.template")
+ host_ip = socket.gethostbyname(socket.gethostname())
+ missing_query = []
+ for line in named_template.readlines():
+ if "listen-on port 53" in line and host_ip in line:
+ var_map["match_port"] = True
+
+ if "allow-query" in line:
+ for subnet in ["127.0.0.0/8"]:
+ if subnet not in line:
+ missing_query.append(subnet)
+
+ named_template.close()
+
+ if var_map["match_port"] is False:
+ self.messages.append(
+ "[%s]Error: named service port "
+ "and/or IP is misconfigured in "
+ "/etc/cobbler/named.template" % self.NAME)
+
+ if len(missing_query) != 0:
+ self.messages.append(
+ "[%s]Error: Missing allow_query values in "
+ "/etc/cobbler/named.template:%s" % (
+ self.NAME,
+ ', '.join(subnet for subnet in missing_query)))
+ else:
+ var_map["match_allow_query"] = True
+
+ fails = []
+ for var in var_map.keys():
+ if var_map[var] is False:
+ fails.append(var)
+
+ if len(fails) != 0:
+ self._set_status(
+ 0,
+ "[%s]Info: DNS template failed components: "
+ "%s" % (
+ self.NAME,
+ ' '.join(failed for failed in fails)))
+
+ else:
+ self._set_status(
+ 0,
+ "[%s]Error: named template file doesn't exist, "
+ "health check failed." % self.NAME)
+
+ return True
+
+ def check_dns_service(self):
+ """Checks if DNS is running on port 53."""
+
+ print "Checking DNS service......",
+ if 'named' not in commands.getoutput('ps -ef'):
+ self._set_status(
+ 0,
+ "[%s]Error: named service does not seem to be "
+ "running" % self.NAME)
+
+ if socket.getservbyport(53) != 'domain':
+ self._set_status(
+ 0,
+ "[%s]Error: domain service is not listening on port "
+ "53" % self.NAME)
+
+ return None
diff --git a/compass-tasks/actions/health_check/check_hds.py b/compass-tasks/actions/health_check/check_hds.py
new file mode 100644
index 0000000..d176f1f
--- /dev/null
+++ b/compass-tasks/actions/health_check/check_hds.py
@@ -0,0 +1,97 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Health Check module for Hardware Discovery."""
+import logging
+
+from compass.actions.health_check import base
+from compass.actions.health_check import utils as health_check_utils
+
+
+class HdsCheck(base.BaseCheck):
+ """hds health check class."""
+ NAME = "HDS Check"
+
+ def run(self):
+ """do health check."""
+ if self.dist in ("centos", "redhat", "fedora", "scientific linux"):
+ pkg_type = "yum"
+ else:
+ pkg_type = "apt"
+
+ try:
+ pkg_module = __import__(pkg_type)
+ except Exception:
+ self._set_status(
+ 0, "[%s]Error: No module named %s please install it first."
+ % (self.NAME, pkg_type)
+ )
+ return (self.code, self.messages)
+
+ logging.info('import %s: %s', pkg_type, pkg_module)
+ method_name = 'self.check_' + pkg_type + '_snmp(pkg_module)'
+ eval(method_name)
+ print "[Done]"
+ self.check_snmp_mibs()
+ print "[Done]"
+ if self.code == 1:
+ self.messages.append("[%s]Info: hds health check has complated. "
+ "No problems found, all systems go."
+ % self.NAME)
+
+ return (self.code, self.messages)
+
+ def check_yum_snmp(self, pkg_module):
+ """Check if SNMP yum dependencies are installed
+
+ :param pkg_module : python yum library
+ :type pkg_module : python module
+
+ """
+ print "Checking SNMP Packages......",
+ yum_base = pkg_module.YumBase()
+ uninstalled = []
+ for package in ['net-snmp-utils', 'net-snmp', 'net-snmp-python']:
+ if len(yum_base.rpmdb.searchNevra(name=package)) == 0:
+ self.messages.append("[%s]Error: %s package is required "
+ "for HDS" % (self.NAME, package))
+ uninstalled.append(package)
+
+ if len(uninstalled) != 0:
+ self._set_status(0, "[%s]Info: Uninstalled packages: %s"
+ % (self.NAME,
+ ', '.join(item for item in uninstalled)))
+
+ return True
+
+ def check_apt_snmp(self, pkg_module):
+ """do apt health check."""
+ return None
+
+ def check_snmp_mibs(self):
+ """Checks if SNMP MIB files are properly placed."""
+
+ print "Checking SNMP MIBs......",
+ conf_err_msg = health_check_utils.check_path(self.NAME,
+ '/etc/snmp/snmp.conf')
+ if not conf_err_msg == "":
+ self._set_status(0, conf_err_msg)
+
+ mibs_err_msg = health_check_utils.check_path(
+ self.NAME,
+ '/usr/local/share/snmp/mibs')
+ if not mibs_err_msg == "":
+ self._set_status(0, mibs_err_msg)
+
+ return True
diff --git a/compass-tasks/actions/health_check/check_misc.py b/compass-tasks/actions/health_check/check_misc.py
new file mode 100644
index 0000000..b8beb1b
--- /dev/null
+++ b/compass-tasks/actions/health_check/check_misc.py
@@ -0,0 +1,219 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Miscellaneous Health Check for Compass."""
+import logging
+
+from compass.actions.health_check import base
+from compass.actions.health_check import utils as health_check_utils
+
+
+class MiscCheck(base.BaseCheck):
+ """health check for misc."""
+ NAME = "Miscellaneous Check"
+
+ MISC_MAPPING = {
+ "yum": "rsyslog ntp iproute openssh-clients python git wget "
+ "python-setuptools "
+ "amqp mod_wsgi httpd squid "
+ "dhcp bind rsync yum-utils xinetd tftp-server gcc "
+ "net-snmp-utils net-snmp".split(" "),
+ "pip": "netaddr flask flask_script flask_restful amqplib "
+ "flask_sqlalchemy paramiko mock celery six discover daemon "
+ "unittest2 chef".split(" "),
+ "disable": "iptables ip6tables".split(" "),
+ "enable": "httpd squid xinetd dhcpd named sshd rsyslog cobblerd "
+ "ntpd compass-celeryd compass-progress-updated".split(" "),
+ }
+
+ def run(self):
+ """do health check."""
+ self.check_linux_dependencies()
+ print "[Done]"
+ self.check_pip_dependencies()
+ print "[Done]"
+ self.check_ntp()
+ print "[Done]"
+ self.check_rsyslogd()
+ print "[Done]"
+ self.check_chkconfig()
+ print "[Done]"
+ self.check_selinux()
+ print "[Done]"
+
+ if self.code == 1:
+ self.messages.append(
+ "[%s]Info: Miscellaneous check has completed "
+ "No problems found, all systems go." % self.NAME)
+ return (self.code, self.messages)
+
+ def check_linux_dependencies(self):
+ """Checks if dependencies are installed."""
+ print "Checking Linux dependencies....",
+ if self.dist in ("centos", "redhat", "fedora", "scientific linux"):
+ pkg_type = "yum"
+ else:
+ pkg_type = "apt"
+
+ try:
+ pkg_module = __import__(pkg_type)
+ except Exception:
+ self._set_status(
+ 0,
+ "[%s]Error: No module named %s, "
+ "please install it first." % (self.NAME, pkg_type))
+ return True
+
+ logging.info('import %s: %s', pkg_type, pkg_module)
+ method_name = 'self.check_' + pkg_type + '_dependencies(pkg_module)'
+ eval(method_name)
+
+ def check_yum_dependencies(self, pkg_module):
+ """Checks if yum dependencies are installed.
+
+ :param pkg_module : python yum library
+ :type pkg_module : python module
+
+ """
+ print "Checking Yum dependencies......",
+ yum_base = pkg_module.YumBase()
+ uninstalled = []
+ for package in self.MISC_MAPPING["yum"]:
+ if len(yum_base.rpmdb.searchNevra(name=package)) == 0:
+ self._set_status(
+ 0,
+ "[%s]Error: %s package is required"
+ % (self.NAME, package))
+ uninstalled.append(package)
+
+ if len(uninstalled) != 0:
+ self._set_status(
+ 0,
+ "[%s]Info: Uninstalled yum packages: %s"
+ % (self.NAME, ', '.join(item for item in uninstalled)))
+
+ return True
+
+ def check_pip_dependencies(self):
+ """Checks if required pip packages are installed."""
+ print "Checking pip dependencies......",
+ uninstalled = []
+ for module in self.MISC_MAPPING['pip']:
+ try:
+ __import__(module)
+ except Exception:
+ self._set_status(
+ 0,
+ "[%s]Error: pip package %s is requred"
+ % (self.NAME, module))
+ uninstalled.append(module)
+
+ if len(uninstalled) != 0:
+ self._set_status(
+ 0,
+ "[%s]Info: Uninstalled pip packages: %s"
+ % (self.NAME, ', '.join(item for item in uninstalled)))
+
+ return True
+
+ def check_ntp(self):
+ """Validates ntp configuration and service."""
+
+ print "Checking NTP......",
+ conf_err_msg = health_check_utils.check_path(self.NAME,
+ '/etc/ntp.conf')
+ if not conf_err_msg == "":
+ self._set_status(0, conf_err_msg)
+
+ serv_err_msg = health_check_utils.check_service_running(self.NAME,
+ 'ntpd')
+ if not serv_err_msg == "":
+ self._set_status(0, serv_err_msg)
+
+ return True
+
+ def check_rsyslogd(self):
+ """Validates rsyslogd configuration and service."""
+
+ print "Checking rsyslog......",
+ conf_err_msg = health_check_utils.check_path(self.NAME,
+ '/etc/rsyslog.conf')
+ if not conf_err_msg == "":
+ self._set_status(0, conf_err_msg)
+
+ dir_err_msg = health_check_utils.check_path(self.NAME,
+ '/etc/rsyslog.d/')
+ if not dir_err_msg == "":
+ self._set_status(0, dir_err_msg)
+
+ serv_err_msg = health_check_utils.check_service_running(self.NAME,
+ 'rsyslogd')
+ if not serv_err_msg == "":
+ self._set_status(0, serv_err_msg)
+
+ return True
+
+ def check_chkconfig(self):
+ """Check if required services are enabled on the start up."""
+
+ print "Checking chkconfig......",
+ serv_to_disable = []
+ for serv in self.MISC_MAPPING["disable"]:
+ if health_check_utils.check_chkconfig(serv) is True:
+ self._set_status(
+ 0,
+ "[%s]Error: %s is not disabled"
+ % (self.NAME, serv))
+ serv_to_disable.append(serv)
+
+ if len(serv_to_disable) != 0:
+ self._set_status(
+ 0,
+ "[%s]Info: You need to disable these services "
+ "on system start-up: %s"
+ % (self.NAME,
+ ", ".join(item for item in serv_to_disable)))
+
+ serv_to_enable = []
+ for serv in self.MISC_MAPPING["enable"]:
+ if health_check_utils.check_chkconfig(serv) is False:
+ self._set_status(
+ 0, "[%s]Error: %s is disabled" % (self.NAME, serv))
+ serv_to_enable.append(serv)
+
+ if len(serv_to_enable) != 0:
+ self._set_status(0, "[%s]Info: You need to enable these "
+ "services on system start-up: %s"
+ % (self.NAME,
+ ", ".join(item for item in serv_to_enable)))
+
+ return True
+
+ def check_selinux(self):
+ """Check if SELinux is disabled."""
+ print "Checking Selinux......",
+ disabled = False
+ with open("/etc/selinux/config") as selinux:
+ for line in selinux:
+ if "SELINUX=disabled" in line:
+ disabled = True
+ break
+
+ if disabled is False:
+ self._set_status(
+ 0,
+ "[%s]Selinux is not disabled, "
+ "please disable it in /etc/selinux/config." % self.NAME)
+
+ return True
diff --git a/compass-tasks/actions/health_check/check_os_installer.py b/compass-tasks/actions/health_check/check_os_installer.py
new file mode 100644
index 0000000..6ef9818
--- /dev/null
+++ b/compass-tasks/actions/health_check/check_os_installer.py
@@ -0,0 +1,151 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Compass Health Check module for OS Installer."""
+
+import os
+import xmlrpclib
+
+from compass.actions.health_check import base
+
+
+class OsInstallerCheck(base.BaseCheck):
+ """os installer health check."""
+ NAME = "OS Installer Check"
+
+ def run(self):
+ """do health check."""
+ method_name = 'self.' + self.os_installer['name'] + '_check()'
+ return eval(method_name)
+
+ def cobbler_check(self):
+ """Runs cobbler check from xmlrpc client."""
+ try:
+ remote = xmlrpclib.Server(
+ self.os_installer['cobbler_url'],
+ allow_none=True)
+ credentials = self.os_installer['credentials']
+ token = remote.login(
+ credentials['username'], credentials['password'])
+ except Exception:
+ self.code = 0
+ self.messages.append(
+ "[%s]Error: Cannot login to Cobbler with "
+ "the tokens provided in the config file"
+ % self.NAME)
+ self.messages.append(
+ "[%s]Error: Failed to connect to Cobbler "
+ "API, please check if /etc/cobbler/setting "
+ "is properly configured" % self.NAME)
+ return (self.code, self.messages)
+
+ check_result = remote.check(token)
+
+ for index, message in enumerate(check_result):
+ if "SELinux" in message:
+ check_result.pop(index)
+
+ if len(check_result) != 0:
+ self.code = 0
+ for error_msg in check_result:
+ self.messages.append("[%s]Error: " % self.NAME + error_msg)
+
+ if len(remote.get_distros()) == 0:
+ self._set_status(0,
+ "[%s]Error: No Cobbler distros found" % self.NAME)
+
+ if len(remote.get_profiles()) == 0:
+ self._set_status(0,
+ "[%s]Error: No Cobbler profiles found"
+ % self.NAME)
+
+ found_ppa = False
+ if len(remote.get_repos()) != 0:
+ for repo in remote.get_repos():
+ if 'ppa_repo' in repo['mirror']:
+ found_ppa = True
+ break
+
+ if found_ppa is False:
+ self._set_status(0,
+ "[%s]Error: No repository ppa_repo found"
+ % self.NAME)
+
+ path_map = {
+ 'match_kickstart': (
+ '/var/lib/cobbler/kickstarts/',
+ ['default.ks', 'default.seed']
+ ),
+ 'match_snippets': (
+ '/var/lib/cobbler/snippets/',
+ [
+ 'kickstart_done',
+ 'kickstart_start',
+ 'kickstart_pre_partition_disks',
+ 'kickstart_partition_disks',
+ 'kickstart_pre_anamon',
+ 'kickstart_post_anamon',
+ 'kickstart_pre_install_network_config',
+ 'kickstart_network_config',
+ 'kickstart_post_install_network_config',
+ 'kickstart_chef',
+ 'kickstart_ntp',
+ 'kickstart_yum_repo_config',
+ 'preseed_pre_partition_disks',
+ 'preseed_partition_disks',
+ 'preseed_pre_anamon',
+ 'preseed_post_anamon',
+ 'preseed_pre_install_network_config',
+ 'preseed_network_config',
+ 'preseed_post_install_network_config',
+ 'preseed_chef',
+ 'preseed_ntp',
+ 'preseed_apt_repo_config',
+ ]
+ ),
+ 'match_ks_mirror': (
+ '/var/www/cobbler/',
+ ['ks_mirror']
+ ),
+ 'match_repo_mirror': (
+ '/var/www/cobbler/',
+ ['repo_mirror']
+ ),
+ 'match_iso': (
+ '/var/lib/cobbler/',
+ ['iso']
+ ),
+ }
+ not_exists = []
+ for key in path_map.keys():
+ for path in path_map[key][1]:
+ if not os.path.exists(path_map[key][0] + path):
+ not_exists.append(path_map[key][0] + path)
+
+ if len(not_exists) != 0:
+ self._set_status(
+ 0,
+ "[%s]Error: These locations do not exist: "
+ "%s" % (
+ self.NAME,
+ ', '.join(item for item in not_exists)
+ )
+ )
+
+ if self.code == 1:
+ self.messages.append(
+ "[%s]Info: OS Installer health check has completed."
+ " No problems found, all systems go." % self.NAME)
+
+ return (self.code, self.messages)
diff --git a/compass-tasks/actions/health_check/check_package_installer.py b/compass-tasks/actions/health_check/check_package_installer.py
new file mode 100644
index 0000000..efcd8e8
--- /dev/null
+++ b/compass-tasks/actions/health_check/check_package_installer.py
@@ -0,0 +1,68 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Health Check module for Package Installer."""
+import logging
+import os
+import requests
+
+from compass.actions.health_check import base
+from compass.actions.health_check import utils as health_check_utils
+
+
+class PackageInstallerCheck(base.BaseCheck):
+ """package installer health check class."""
+ NAME = "Package Installer Check"
+
+ def run(self):
+ """do health check."""
+ method_name = "self." + self.package_installer['name'] + "_check()"
+ return eval(method_name)
+
+ def chef_check(self):
+ """Checks chef setting, cookbooks and roles."""
+ self.check_chef_config_dir()
+ print "[Done]"
+ if self.code == 1:
+ self.messages.append(
+ "[%s]Info: Package installer health check "
+ "has completed. No problems found, all systems "
+ "go." % self.NAME)
+
+ return (self.code, self.messages)
+
+ def check_chef_config_dir(self):
+ """Validates chef configuration directories."""
+
+ print "Checking Chef configurations......",
+ message = health_check_utils.check_path(self.NAME, '/etc/chef-server/')
+ if not message == "":
+ self._set_status(0, message)
+
+ message = health_check_utils.check_path(self.NAME, '/opt/chef-server/')
+ if not message == "":
+ self._set_status(0, message)
+
+ return None
+
+ def ansible_check(self):
+ """Placeholder for ansible check."""
+ print "Checking ansible......"
+ print ("[Done]")
+ self.code == 1
+ self.messages.append(
+ "[%s]Info: Package installer health check "
+ "has completed. No problems found, all systems "
+ "go." % self.NAME)
+ return (self.code, self.messages)
diff --git a/compass-tasks/actions/health_check/check_squid.py b/compass-tasks/actions/health_check/check_squid.py
new file mode 100644
index 0000000..5628a63
--- /dev/null
+++ b/compass-tasks/actions/health_check/check_squid.py
@@ -0,0 +1,128 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Health Check module for Squid service."""
+import commands
+import os
+import pwd
+import socket
+
+from compass.actions.health_check import base
+from compass.actions.health_check import utils as health_check_utils
+
+
+class SquidCheck(base.BaseCheck):
+ """Squid health check class."""
+ NAME = "Squid Check"
+
+ def run(self):
+ """do health check."""
+ self.check_squid_files()
+ print "[Done]"
+ self.check_squid_service()
+ print "[Done]"
+ if self.code == 1:
+ self.messages.append(
+ "[%s]Info: Squid health check has completed. "
+ "No problems found, all systems go." % self.NAME)
+ return (self.code, self.messages)
+
+ def check_squid_files(self):
+ """Validates squid config, cache directory and ownership."""
+ print "Checking Squid Files......",
+ var_map = {
+ 'match_squid_conf': False,
+ 'match_squid_cache': False,
+ 'match_squid_ownership': False,
+ }
+
+ conf_err_msg = health_check_utils.check_path(
+ self.NAME,
+ "/etc/squid/squid.conf")
+ if not conf_err_msg == "":
+ self._set_status(0, conf_err_msg)
+ elif int(oct(os.stat('/etc/squid/squid.conf').st_mode)) < 100644:
+ self._set_status(
+ 0,
+ "[%s]Error: squid.conf has incorrect "
+ "file permissions" % self.NAME)
+ else:
+ var_map['match_squid_conf'] = True
+
+ squid_path_err_msg = health_check_utils.check_path(
+ self.NAME, '/var/squid/')
+ if not squid_path_err_msg == "":
+ self._set_status(0, squid_path_err_msg)
+ elif health_check_utils.check_path(
+ self.NAME,
+ '/var/squid/cache'
+ ) != "":
+ self._set_status(
+ 0,
+ health_check_utils.check_path(
+ self.NAME,
+ '/var/squid/cache'
+ )
+ )
+ else:
+ var_map['match_squid_cache'] = True
+ uid = os.stat('/var/squid/').st_uid
+ gid = os.stat('/var/squid/').st_gid
+ if uid != gid or pwd.getpwuid(23).pw_name != 'squid':
+ self._set_status(
+ 0,
+ "[%s]Error: /var/squid directory ownership "
+ "misconfigured" % self.NAME)
+ else:
+ var_map['match_squid_ownership'] = True
+
+ fails = []
+ for key in var_map.keys():
+ if var_map[key] is False:
+ fails.append(key)
+
+ if len(fails) != 0:
+ self.messages.append(
+ "[%s]Info: Failed components for squid config: "
+ "%s" % (
+ self.NAME,
+ ', '.join(item for item in fails)
+ )
+ )
+ return True
+
+ def check_squid_service(self):
+ """Checks if squid is running on port 3128."""
+
+ print "Checking Squid service......",
+ if 'squid' not in commands.getoutput('ps -ef'):
+ self._set_status(
+ 0,
+ "[%s]Error: squid service does not seem "
+ "running" % self.NAME)
+
+ try:
+ if 'squid' != socket.getservbyport(3128):
+ self._set_status(
+ 0,
+ "[%s]Error: squid is not listening on "
+ "3128" % self.NAME)
+
+ except Exception:
+ self._set_status(
+ 0,
+ "[%s]Error: No service is listening on 3128, "
+ "squid failed" % self.NAME)
+
+ return True
diff --git a/compass-tasks/actions/health_check/check_tftp.py b/compass-tasks/actions/health_check/check_tftp.py
new file mode 100644
index 0000000..7ca6405
--- /dev/null
+++ b/compass-tasks/actions/health_check/check_tftp.py
@@ -0,0 +1,96 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Health Check module for TFTP service."""
+import os
+import socket
+import xmlrpclib
+
+from compass.actions.health_check import base
+from compass.actions.health_check import utils as health_check_utils
+
+
+class TftpCheck(base.BaseCheck):
+ """tftp health check class."""
+ NAME = "TFTP Check"
+
+ def run(self):
+ """do health check."""
+ method_name = "self.check_" + self.os_installer['name'] + "_tftp()"
+ return eval(method_name)
+
+ def check_cobbler_tftp(self):
+ """Checks if Cobbler manages TFTP service.
+
+ :note: we assume TFTP service is running at the
+ same machine where this health check runs at
+ """
+
+ try:
+ remote = xmlrpclib.Server(
+ self.os_installer['cobbler_url'],
+ allow_none=True)
+ credentials = self.os_installer['credentials']
+ remote.login(
+ credentials['username'], credentials['password'])
+ except Exception:
+ self._set_status(
+ 0,
+ "[%s]Error: Cannot login to Cobbler with the tokens "
+ " provided in the config file" % self.NAME)
+ return (self.code, self.messages)
+
+ cobbler_settings = remote.get_settings()
+ if cobbler_settings['manage_tftp'] == 0:
+ self.messages.append(
+ '[TFTP]Info: tftp service is not managed by Compass')
+ return (0, self.messages)
+ self.check_tftp_dir()
+ print "[Done]"
+ self.check_tftp_service()
+ print "[Done]"
+ if self.code == 1:
+ self.messages.append(
+ "[%s]Info: tftp service health check has completed. "
+ "No problems found, all systems go." % self.NAME)
+
+ return (self.code, self.messages)
+
+ def check_tftp_dir(self):
+ """Validates TFTP directories and configurations."""
+ print "Checking TFTP directories......",
+ if not os.path.exists('/var/lib/tftpboot/'):
+ self._set_status(
+ 0,
+ "[%s]Error: No tftp-boot libraries found, "
+ "please check if tftp server is properly "
+ "installed/managed" % self.NAME)
+
+ return True
+
+ def check_tftp_service(self):
+ """Checks if TFTP is running on port 69."""
+ print "Checking TFTP services......",
+ serv_err_msg = health_check_utils.check_service_running(self.NAME,
+ 'xinetd')
+ if not serv_err_msg == "":
+ self._set_status(0, serv_err_msg)
+
+ if 'tftp' != socket.getservbyport(69):
+ self._set_status(
+ 0,
+ "[%s]Error: tftp doesn't seem to be listening "
+ "on Port 60." % self.NAME)
+
+ return True
diff --git a/compass-tasks/actions/health_check/utils.py b/compass-tasks/actions/health_check/utils.py
new file mode 100644
index 0000000..369c5b6
--- /dev/null
+++ b/compass-tasks/actions/health_check/utils.py
@@ -0,0 +1,114 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Compass Health Check heavy-lifting utilities"""
+import commands
+import os
+import platform
+import re
+
+
+def validate_setting(module, setting, param):
+ """Checks if a Compass setting exists in the config file.
+
+ :param module : module name to be checked
+ :type module : string
+ :param setting : compass setting wrapper
+ :type setting : python module
+ :param param : settings defined in compass config file
+ :type param : string
+
+ """
+ if hasattr(setting, param):
+ return True
+ else:
+ err_msg = "[%s]Error: no %s defined" % (module, param)
+ return err_msg
+
+
+def get_dist():
+ """Returns the operating system related information."""
+
+ os_version, version, release = platform.linux_distribution()
+ return (os_version.lower().strip(), version, release.lower().strip())
+
+
+def check_path(module_name, path):
+ """Checks if a directory or file exisits.
+
+ :param module_name : module name to be checked
+ :type module_name : string
+ :param path : path of the directory of file
+ :type path : string
+
+ """
+ err_msg = ""
+ if not os.path.exists(path):
+ err_msg = (
+ "[%s]Error: %s does not exist, "
+ "please check your configurations.") % (module_name, path)
+ return err_msg
+
+
+def check_service_running(module_name, service_name):
+ """Checks if a certain service is running.
+
+ :param module_name : module name to be checked
+ :type module_name : string
+ :param service_name : service name to be checked
+ :type service_name : string
+
+ """
+ err_msg = ""
+ if service_name not in commands.getoutput('ps -ef'):
+ err_msg = "[%s]Error: %s is not running." % (
+ module_name, service_name)
+
+ return err_msg
+
+
+def check_chkconfig(service_name):
+ """Checks if a service is enabled at the start up.
+
+ :param service_name : service name to be checked
+ :type service_name : string
+
+ """
+ chk_on = False
+ for service in os.listdir('/etc/rc3.d/'):
+ if service_name in service and 'S' in service:
+ chk_on = True
+ break
+
+ return chk_on
+
+
+def strip_name(name):
+ """Reformats names."""
+ if not any([s in name for s in "(,),-,_".split(',')]):
+ return name
+
+ paren_regex = re.compile("(.*?)\s*\(")
+ dash_regex = re.compile("(.*?)\s*\-")
+ under_dash_regex = re.compile("(.*?)\s*\_")
+
+ r1 = paren_regex.match(name)
+ r2 = dash_regex.match(name)
+ r3 = under_dash_regex.match(name)
+ shortest = 'AVeryLongStringForDefualt'
+ for r in [r1, r2, r3]:
+ if r and len(r.group(1)) < len(shortest):
+ shortest = r.group(1)
+
+ return shortest
diff --git a/compass-tasks/actions/install_callback.py b/compass-tasks/actions/install_callback.py
new file mode 100644
index 0000000..14d2639
--- /dev/null
+++ b/compass-tasks/actions/install_callback.py
@@ -0,0 +1,181 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to receive installation callback.
+
+ .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+import logging
+
+from compass.actions import util
+from compass.db.api import cluster as cluster_api
+from compass.db.api import host as host_api
+from compass.db.api import user as user_db
+from compass.deployment.deploy_manager import DeployManager
+from compass.deployment.utils import constants as const
+
+
+def os_installed(
+ host_id, clusterhosts_ready, clusters_os_ready,
+ username=None
+):
+ """Callback when os is installed.
+
+ :param host_id: host that os is installed.
+ :type host_id: integer
+ :param clusterhosts_ready: the clusterhosts that should trigger ready.
+ :param clusters_os_ready: the cluster that should trigger os ready.
+
+ .. note::
+ The function should be called out of database session.
+ """
+ with util.lock('serialized_action') as lock:
+ if not lock:
+ raise Exception(
+ 'failed to acquire lock to '
+ 'do the post action after os installation'
+ )
+ logging.info(
+ 'os installed on host %s '
+ 'with cluster host ready %s cluster os ready %s',
+ host_id, clusterhosts_ready, clusters_os_ready
+ )
+ if username:
+ user = user_db.get_user_object(username)
+ else:
+ user = None
+ os_installed_triggered = False
+ for cluster_id, clusterhost_ready in clusterhosts_ready.items():
+ if not clusterhost_ready and os_installed_triggered:
+ continue
+ cluster_id = int(cluster_id)
+ cluster_info = util.ActionHelper.get_cluster_info(
+ cluster_id, user)
+ adapter_id = cluster_info[const.ADAPTER_ID]
+
+ adapter_info = util.ActionHelper.get_adapter_info(
+ adapter_id, cluster_id, user)
+ hosts_info = util.ActionHelper.get_hosts_info(
+ cluster_id, [host_id], user)
+
+ deploy_manager = DeployManager(
+ adapter_info, cluster_info, hosts_info)
+
+ if not os_installed_triggered:
+ deploy_manager.os_installed()
+ util.ActionHelper.host_ready(host_id, True, user)
+ os_installed_triggered = True
+
+ if clusterhost_ready:
+ # deploy_manager.cluster_os_installed()
+ util.ActionHelper.cluster_host_ready(
+ cluster_id, host_id, False, user
+ )
+
+ if util.ActionHelper.is_cluster_os_ready(cluster_id, user):
+ logging.info("deploy_manager begin cluster_os_installed")
+ deploy_manager.cluster_os_installed()
+
+
+def package_installed(
+ cluster_id, host_id, cluster_ready,
+ host_ready, username=None
+):
+ """Callback when package is installed.
+
+ :param cluster_id: cluster id.
+ :param host_id: host id.
+ :param cluster_ready: if the cluster should trigger ready.
+ :param host_ready: if the host should trigger ready.
+
+ .. note::
+ The function should be called out of database session.
+ """
+ with util.lock('serialized_action') as lock:
+ if not lock:
+ raise Exception(
+ 'failed to acquire lock to '
+ 'do the post action after package installation'
+ )
+ logging.info(
+ 'package installed on cluster %s host %s '
+ 'with cluster ready %s host ready %s',
+ cluster_id, host_id, cluster_ready, host_ready
+ )
+
+ if username:
+ user = user_db.get_user_object(username)
+ else:
+ user = None
+ cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
+ adapter_id = cluster_info[const.ADAPTER_ID]
+
+ adapter_info = util.ActionHelper.get_adapter_info(
+ adapter_id, cluster_id, user)
+ hosts_info = util.ActionHelper.get_hosts_info(
+ cluster_id, [host_id], user)
+
+ deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info)
+
+ deploy_manager.package_installed()
+ util.ActionHelper.cluster_host_ready(cluster_id, host_id, True, user)
+ if cluster_ready:
+ util.ActionHelper.cluster_ready(cluster_id, False, user)
+ if host_ready:
+ util.ActionHelper.host_ready(host_id, False, user)
+
+
+def cluster_installed(
+ cluster_id, clusterhosts_ready,
+ username=None
+):
+ """Callback when cluster is installed.
+
+ :param cluster_id: cluster id
+ :param clusterhosts_ready: clusterhosts that should trigger ready.
+
+ .. note::
+ The function should be called out of database session.
+ """
+ with util.lock('serialized_action') as lock:
+ if not lock:
+ raise Exception(
+ 'failed to acquire lock to '
+ 'do the post action after cluster installation'
+ )
+ logging.info(
+ 'package installed on cluster %s with clusterhosts ready %s',
+ cluster_id, clusterhosts_ready
+ )
+ if username:
+ user = user_db.get_user_object(username)
+ else:
+ user = None
+ cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
+ adapter_id = cluster_info[const.ADAPTER_ID]
+
+ adapter_info = util.ActionHelper.get_adapter_info(
+ adapter_id, cluster_id, user)
+ hosts_info = util.ActionHelper.get_hosts_info(
+ cluster_id, clusterhosts_ready.keys(), user)
+
+ deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info)
+
+ deploy_manager.cluster_installed()
+ util.ActionHelper.cluster_ready(cluster_id, True, user)
+ for host_id, clusterhost_ready in clusterhosts_ready.items():
+ if clusterhost_ready:
+ util.ActionHelper.cluster_host_ready(
+ cluster_id, host_id, False, user
+ )
diff --git a/compass-tasks/actions/patch.py b/compass-tasks/actions/patch.py
new file mode 100644
index 0000000..6d29be6
--- /dev/null
+++ b/compass-tasks/actions/patch.py
@@ -0,0 +1,69 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to patch an existing cluster
+"""
+import logging
+import simplejson as json
+
+from compass.actions import util
+from compass.db.api import cluster as cluster_db
+from compass.db.api import user as user_db
+from compass.deployment.deploy_manager import Patcher
+from compass.deployment.utils import constants as const
+
+
+def patch(cluster_id, username=None):
+ """Patch cluster.
+
+ :param cluster_id: id of the cluster
+ :type cluster_id: int
+
+ .. note::
+ The function should be called out of database session.
+ """
+ with util.lock('serialized_action', timeout=1000) as lock:
+ if not lock:
+ raise Exception('failed to acquire lock to deploy')
+
+ user = user_db.get_user_object(username)
+ cluster_hosts = cluster_db.list_cluster_hosts(cluster_id, user)
+ hosts_id_list = [host['id'] for host in cluster_hosts]
+ cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
+ adapter_id = cluster_info[const.ADAPTER_ID]
+
+ adapter_info = util.ActionHelper.get_adapter_info(
+ adapter_id, cluster_id, user)
+ hosts_info = util.ActionHelper.get_hosts_info(
+ cluster_id, hosts_id_list, user)
+ patch_successful = True
+ try:
+ patcher = Patcher(
+ adapter_info, cluster_info, hosts_info, cluster_hosts)
+ patched_config = patcher.patch()
+ except Exception as error:
+ logging.exception(error)
+ patch_successful = False
+
+ if patch_successful:
+ clean_payload = '{"patched_roles": []}'
+ clean_payload = json.loads(clean_payload)
+ for cluster_host in cluster_hosts:
+ cluster_db.update_cluster_host(
+ cluster_id, cluster_host['id'], user, **clean_payload)
+ logging.info(
+ "cleaning up patched roles for host id: %s",
+ cluster_host['id']
+ )
+ logging.info("Patch successful: %s", patched_config)
diff --git a/compass-tasks/actions/poll_switch.py b/compass-tasks/actions/poll_switch.py
new file mode 100644
index 0000000..5c29b01
--- /dev/null
+++ b/compass-tasks/actions/poll_switch.py
@@ -0,0 +1,162 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to provider function to poll switch."""
+import logging
+import netaddr
+
+from compass.actions import util
+from compass.db.api import database
+from compass.db.api import switch as switch_api
+from compass.db.api import user as user_api
+from compass.hdsdiscovery.hdmanager import HDManager
+
+
+def _poll_switch(ip_addr, credentials, req_obj='mac', oper="SCAN"):
+ """Poll switch by ip addr.
+
+
+ Args:
+ ip_addr: ip addr of the switch.
+ credentials: credentials of the switch.
+
+ Returns: switch attributes dict and list of machine attributes dict.
+ """
+ under_monitoring = 'under_monitoring'
+ unreachable = 'unreachable'
+ polling_error = 'error'
+ hdmanager = HDManager()
+ vendor, state, err_msg = hdmanager.get_vendor(ip_addr, credentials)
+ if not vendor:
+ logging.info("*****error_msg: %s****", err_msg)
+ logging.error('no vendor found or match switch %s', ip_addr)
+ return (
+ {
+ 'vendor': vendor, 'state': state, 'err_msg': err_msg
+ }, {
+ }
+ )
+
+ logging.debug(
+ 'hdmanager learn switch from %s', ip_addr
+ )
+ results = []
+ try:
+ results = hdmanager.learn(
+ ip_addr, credentials, vendor, req_obj, oper
+ )
+ except Exception as error:
+ logging.exception(error)
+ state = unreachable
+ err_msg = (
+ 'SNMP walk for querying MAC addresses timedout'
+ )
+ return (
+ {
+ 'vendor': vendor, 'state': state, 'err_msg': err_msg
+ }, {
+ }
+ )
+
+ logging.info("pollswitch %s result: %s", ip_addr, results)
+ if not results:
+ logging.error(
+ 'no result learned from %s', ip_addr
+ )
+ state = polling_error
+ err_msg = 'No result learned from SNMP walk'
+ return (
+ {'vendor': vendor, 'state': state, 'err_msg': err_msg},
+ {}
+ )
+
+ logging.info('poll switch result: %s' % str(results))
+ machine_dicts = {}
+ for machine in results:
+ mac = machine['mac']
+ port = machine['port']
+ vlan = int(machine['vlan'])
+ if vlan:
+ vlans = [vlan]
+ else:
+ vlans = []
+ if mac not in machine_dicts:
+ machine_dicts[mac] = {'mac': mac, 'port': port, 'vlans': vlans}
+ else:
+ machine_dicts[mac]['port'] = port
+ machine_dicts[mac]['vlans'].extend(vlans)
+
+ logging.debug('update switch %s state to under monitoring', ip_addr)
+ state = under_monitoring
+ return (
+ {'vendor': vendor, 'state': state, 'err_msg': err_msg},
+ machine_dicts.values()
+ )
+
+
+def poll_switch(poller_email, ip_addr, credentials,
+ req_obj='mac', oper="SCAN"):
+ """Query switch and update switch machines.
+
+ .. note::
+ When polling switch succeeds, for each mac it got from polling switch,
+ A Machine record associated with the switch is added to the database.
+
+ :param ip_addr: switch ip address.
+ :type ip_addr: str
+ :param credentials: switch crednetials.
+ :type credentials: dict
+ :param req_obj: the object requested to query from switch.
+ :type req_obj: str
+ :param oper: the operation to query the switch.
+ :type oper: str, should be one of ['SCAN', 'GET', 'SET']
+
+ .. note::
+ The function should be called out of database session scope.
+ """
+ poller = user_api.get_user_object(poller_email)
+ ip_int = long(netaddr.IPAddress(ip_addr))
+ with util.lock('poll switch %s' % ip_addr, timeout=120) as lock:
+ if not lock:
+ raise Exception(
+ 'failed to acquire lock to poll switch %s' % ip_addr
+ )
+
+ # TODO(grace): before repoll the switch, set the state to repolling.
+ # and when the poll switch is timeout, set the state to error.
+ # the frontend should only consider some main state like INTIALIZED,
+ # ERROR and SUCCESSFUL, REPOLLING is as an intermediate state to
+ # indicate the switch is in learning the mac of the machines connected
+ # to it.
+ logging.debug('poll switch: %s', ip_addr)
+ switch_dict, machine_dicts = _poll_switch(
+ ip_addr, credentials, req_obj=req_obj, oper=oper
+ )
+ switches = switch_api.list_switches(ip_int=ip_int, user=poller)
+ if not switches:
+ logging.error('no switch found for %s', ip_addr)
+ return
+
+ for switch in switches:
+ for machine_dict in machine_dicts:
+ logging.info('add machine: %s', machine_dict)
+ machine_dict['owner_id'] = poller.id
+ switch_api.add_switch_machine(
+ switch['id'], False, user=poller, **machine_dict
+ )
+ switch_api.update_switch(
+ switch['id'],
+ user=poller,
+ **switch_dict
+ )
diff --git a/compass-tasks/actions/reinstall.py b/compass-tasks/actions/reinstall.py
new file mode 100644
index 0000000..62d1bcb
--- /dev/null
+++ b/compass-tasks/actions/reinstall.py
@@ -0,0 +1,38 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to reinstall a given cluster
+
+ .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+import logging
+
+from compass.actions import util
+from compass.db.api import database
+
+
+def reinstall(cluster_hosts):
+ """Reinstall clusters.
+
+ :param cluster_hosts: clusters and hosts in each cluster to reinstall.
+ :type cluster_hosts: dict of int or str to list of int or str
+
+ .. note::
+ The function should be called out of database session.
+ """
+ with util.lock('serialized_action') as lock:
+ if not lock:
+ raise Exception(
+ 'failed to acquire lock to reinstall')
+ logging.debug('reinstall cluster_hosts: %s', cluster_hosts)
diff --git a/compass-tasks/actions/search.py b/compass-tasks/actions/search.py
new file mode 100644
index 0000000..73ce1d9
--- /dev/null
+++ b/compass-tasks/actions/search.py
@@ -0,0 +1,46 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to search configs of given clusters
+
+ .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+import logging
+
+from compass.actions import util
+from compass.config_management.utils.config_manager import ConfigManager
+from compass.db.api import database
+
+
+def search(cluster_hosts, cluster_propreties_match,
+ cluster_properties_name, host_properties_match,
+ host_properties_name):
+ """search clusters.
+
+ :param cluster_hosts: clusters and hosts in each cluster to search.
+ :type cluster_hosts: dict of int or str to list of int or str
+
+ .. note::
+ The function should be called out of database session.
+ """
+ logging.debug('search cluster_hosts: %s', cluster_hosts)
+ with database.session():
+ cluster_hosts, os_versions, target_systems = (
+ util.update_cluster_hosts(cluster_hosts))
+ manager = ConfigManager()
+ return manager.filter_cluster_and_hosts(
+ cluster_hosts, os_versions,
+ target_systems, cluster_propreties_match,
+ cluster_properties_name, host_properties_match,
+ host_properties_name)
diff --git a/compass-tasks/actions/update_progress.py b/compass-tasks/actions/update_progress.py
new file mode 100644
index 0000000..67a9963
--- /dev/null
+++ b/compass-tasks/actions/update_progress.py
@@ -0,0 +1,298 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to update status and installing progress of the given cluster.
+
+ .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+import logging
+
+from compass.actions import util
+from compass.db.api import adapter_holder as adapter_api
+from compass.db.api import cluster as cluster_api
+from compass.db.api import host as host_api
+from compass.db.api import user as user_api
+from compass.log_analyzor import progress_calculator
+from compass.utils import setting_wrapper as setting
+
+
+def update_progress():
+ """Update status and installing progress of the given cluster.
+
+ :param cluster_hosts: clusters and hosts in each cluster to update.
+ :type cluster_hosts: dict of int or str to list of int or str
+
+ .. note::
+ The function should be called out of the database session scope.
+ In the function, it will update the database cluster_state and
+ host_state table for the deploying cluster and hosts.
+
+ The function will also query log_progressing_history table to get
+ the lastest installing progress and the position of log it has
+ processed in the last run. The function uses these information to
+ avoid recalculate the progress from the beginning of the log file.
+ After the progress got updated, these information will be stored back
+ to the log_progressing_history for next time run.
+ """
+ with util.lock('log_progressing', timeout=60, blocking=False) as lock:
+ if not lock:
+ logging.error(
+ 'failed to acquire lock to calculate installation progress'
+ )
+ return
+
+ logging.info('update installing progress')
+
+ user = user_api.get_user_object(setting.COMPASS_ADMIN_EMAIL)
+ hosts = host_api.list_hosts(user=user)
+ host_mapping = {}
+ for host in hosts:
+ if 'id' not in host:
+ logging.error('id is not in host %s', host)
+ continue
+ host_id = host['id']
+ if 'os_name' not in host:
+ logging.error('os_name is not in host %s', host)
+ continue
+ if 'os_installer' not in host:
+ logging.error('os_installer is not in host %s', host)
+ continue
+ host_dirname = setting.HOST_INSTALLATION_LOGDIR_NAME
+ if host_dirname not in host:
+ logging.error(
+ '%s is not in host %s', host_dirname, host
+ )
+ continue
+ host_state = host_api.get_host_state(host_id, user=user)
+ if 'state' not in host_state:
+ logging.error('state is not in host state %s', host_state)
+ continue
+ if host_state['state'] == 'INSTALLING':
+ host_log_histories = host_api.get_host_log_histories(
+ host_id, user=user
+ )
+ host_log_history_mapping = {}
+ for host_log_history in host_log_histories:
+ if 'filename' not in host_log_history:
+ logging.error(
+ 'filename is not in host log history %s',
+ host_log_history
+ )
+ continue
+ host_log_history_mapping[
+ host_log_history['filename']
+ ] = host_log_history
+ host_mapping[host_id] = (
+ host, host_state, host_log_history_mapping
+ )
+ else:
+ logging.info(
+ 'ignore host state %s since it is not in installing',
+ host_state
+ )
+ adapters = adapter_api.list_adapters(user=user)
+ adapter_mapping = {}
+ for adapter in adapters:
+ if 'id' not in adapter:
+ logging.error(
+ 'id not in adapter %s', adapter
+ )
+ continue
+ if 'package_installer' not in adapter:
+ logging.info(
+ 'package_installer not in adapter %s', adapter
+ )
+ continue
+ adapter_id = adapter['id']
+ adapter_mapping[adapter_id] = adapter
+ clusters = cluster_api.list_clusters(user=user)
+ cluster_mapping = {}
+ for cluster in clusters:
+ if 'id' not in cluster:
+ logging.error('id not in cluster %s', cluster)
+ continue
+ cluster_id = cluster['id']
+ if 'adapter_id' not in cluster:
+ logging.error(
+ 'adapter_id not in cluster %s',
+ cluster
+ )
+ continue
+ cluster_state = cluster_api.get_cluster_state(
+ cluster_id,
+ user=user
+ )
+ if 'state' not in cluster_state:
+ logging.error('state not in cluster state %s', cluster_state)
+ continue
+ cluster_mapping[cluster_id] = (cluster, cluster_state)
+ clusterhosts = cluster_api.list_clusterhosts(user=user)
+ clusterhost_mapping = {}
+ for clusterhost in clusterhosts:
+ if 'clusterhost_id' not in clusterhost:
+ logging.error(
+ 'clusterhost_id not in clusterhost %s',
+ clusterhost
+ )
+ continue
+ clusterhost_id = clusterhost['clusterhost_id']
+ if 'cluster_id' not in clusterhost:
+ logging.error(
+ 'cluster_id not in clusterhost %s',
+ clusterhost
+ )
+ continue
+ cluster_id = clusterhost['cluster_id']
+ if cluster_id not in cluster_mapping:
+ logging.info(
+ 'ignore clusterhost %s '
+ 'since the cluster_id '
+ 'is not in cluster_mapping %s',
+ clusterhost, cluster_mapping
+ )
+ continue
+ cluster, _ = cluster_mapping[cluster_id]
+ if 'flavor_name' not in cluster:
+ logging.error(
+ 'flavor_name is not in clusterhost %s related cluster',
+ clusterhost
+ )
+ continue
+ clusterhost_dirname = setting.CLUSTERHOST_INATALLATION_LOGDIR_NAME
+ if clusterhost_dirname not in clusterhost:
+ logging.error(
+ '%s is not in clusterhost %s',
+ clusterhost_dirname, clusterhost
+ )
+ continue
+ adapter_id = cluster['adapter_id']
+ if adapter_id not in adapter_mapping:
+ logging.info(
+ 'ignore clusterhost %s '
+ 'since the adapter_id %s '
+ 'is not in adaper_mapping %s',
+ clusterhost, adapter_id, adapter_mapping
+ )
+ continue
+ adapter = adapter_mapping[adapter_id]
+ if 'package_installer' not in adapter:
+ logging.info(
+ 'ignore clusterhost %s '
+ 'since the package_installer is not define '
+ 'in adapter %s',
+ clusterhost, adapter
+ )
+ continue
+ package_installer = adapter['package_installer']
+ clusterhost['package_installer'] = package_installer
+ clusterhost['adapter_name'] = adapter['name']
+ clusterhost_state = cluster_api.get_clusterhost_self_state(
+ clusterhost_id, user=user
+ )
+ if 'state' not in clusterhost_state:
+ logging.error(
+ 'state not in clusterhost_state %s',
+ clusterhost_state
+ )
+ continue
+ if clusterhost_state['state'] == 'INSTALLING':
+ clusterhost_log_histories = (
+ cluster_api.get_clusterhost_log_histories(
+ clusterhost_id, user=user
+ )
+ )
+ clusterhost_log_history_mapping = {}
+ for clusterhost_log_history in clusterhost_log_histories:
+ if 'filename' not in clusterhost_log_history:
+ logging.error(
+ 'filename not in clusterhost_log_history %s',
+ clusterhost_log_history
+ )
+ continue
+ clusterhost_log_history_mapping[
+ clusterhost_log_history['filename']
+ ] = clusterhost_log_history
+ clusterhost_mapping[clusterhost_id] = (
+ clusterhost, clusterhost_state,
+ clusterhost_log_history_mapping
+ )
+ else:
+ logging.info(
+ 'ignore clusterhost state %s '
+ 'since it is not in installing',
+ clusterhost_state
+ )
+
+ progress_calculator.update_host_progress(
+ host_mapping)
+ for host_id, (host, host_state, host_log_history_mapping) in (
+ host_mapping.items()
+ ):
+ host_api.update_host_state(
+ host_id, user=user,
+ percentage=host_state.get('percentage', 0),
+ message=host_state.get('message', ''),
+ severity=host_state.get('severity', 'INFO')
+ )
+ for filename, host_log_history in (
+ host_log_history_mapping.items()
+ ):
+ host_api.add_host_log_history(
+ host_id, filename=filename, user=user,
+ position=host_log_history.get('position', 0),
+ percentage=host_log_history.get('percentage', 0),
+ partial_line=host_log_history.get('partial_line', ''),
+ message=host_log_history.get('message', ''),
+ severity=host_log_history.get('severity', 'INFO'),
+ line_matcher_name=host_log_history.get(
+ 'line_matcher_name', 'start'
+ )
+ )
+ progress_calculator.update_clusterhost_progress(
+ clusterhost_mapping)
+ for (
+ clusterhost_id,
+ (clusterhost, clusterhost_state, clusterhost_log_history_mapping)
+ ) in (
+ clusterhost_mapping.items()
+ ):
+ cluster_api.update_clusterhost_state(
+ clusterhost_id, user=user,
+ percentage=clusterhost_state.get('percentage', 0),
+ message=clusterhost_state.get('message', ''),
+ severity=clusterhost_state.get('severity', 'INFO')
+ )
+ for filename, clusterhost_log_history in (
+ clusterhost_log_history_mapping.items()
+ ):
+ cluster_api.add_clusterhost_log_history(
+ clusterhost_id, user=user, filename=filename,
+ position=clusterhost_log_history.get('position', 0),
+ percentage=clusterhost_log_history.get('percentage', 0),
+ partial_line=clusterhost_log_history.get(
+ 'partial_line', ''),
+ message=clusterhost_log_history.get('message', ''),
+ severity=clusterhost_log_history.get('severity', 'INFO'),
+ line_matcher_name=(
+ clusterhost_log_history.get(
+ 'line_matcher_name', 'start'
+ )
+ )
+ )
+ progress_calculator.update_cluster_progress(
+ cluster_mapping)
+ for cluster_id, (cluster, cluster_state) in cluster_mapping.items():
+ cluster_api.update_cluster_state(
+ cluster_id, user=user
+ )
diff --git a/compass-tasks/actions/util.py b/compass-tasks/actions/util.py
new file mode 100644
index 0000000..4d9f855
--- /dev/null
+++ b/compass-tasks/actions/util.py
@@ -0,0 +1,342 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to provide util for actions
+
+ .. moduleauthor:: Xiaodong Wang ,xiaodongwang@huawei.com>
+"""
+import logging
+import redis
+
+from contextlib import contextmanager
+
+from compass.db.api import adapter_holder as adapter_db
+from compass.db.api import cluster as cluster_db
+from compass.db.api import host as host_db
+from compass.db.api import machine as machine_db
+from compass.deployment.utils import constants as const
+
+
+@contextmanager
+def lock(lock_name, blocking=True, timeout=10):
+ """acquire a lock to do some actions.
+
+ The lock is acquired by lock_name among the whole distributed
+ systems.
+ """
+ # TODO(xicheng): in future we should explicitly told which redis
+ # server we want to talk to make the lock works on distributed
+ # systems.
+ redis_instance = redis.Redis()
+ instance_lock = redis_instance.lock(lock_name, timeout=timeout)
+ owned = False
+ try:
+ locked = instance_lock.acquire(blocking=blocking)
+ if locked:
+ owned = True
+ logging.debug('acquired lock %s', lock_name)
+ yield instance_lock
+ else:
+ logging.info('lock %s is already hold', lock_name)
+ yield None
+
+ except Exception as error:
+ logging.info(
+ 'redis fails to acquire the lock %s', lock_name)
+ logging.exception(error)
+ yield None
+
+ finally:
+ if owned:
+ instance_lock.acquired_until = 0
+ instance_lock.release()
+ logging.debug('released lock %s', lock_name)
+ else:
+ logging.debug('nothing to release %s', lock_name)
+
+
+class ActionHelper(object):
+
+ @staticmethod
+ def get_adapter_info(adapter_id, cluster_id, user):
+ """Get adapter information. Return a dictionary as below,
+
+ {
+ "id": 1,
+ "name": "xxx",
+ "flavors": [
+ {
+ "flavor_name": "xxx",
+ "roles": ['xxx', 'yyy', ...],
+ "template": "xxx.tmpl"
+ },
+ ...
+ ],
+ "metadata": {
+ "os_config": {
+ ...
+ },
+ "package_config": {
+ ...
+ }
+ },
+ "os_installer": {
+ "name": "cobbler",
+ "settings": {....}
+ },
+ "pk_installer": {
+ "name": "chef",
+ "settings": {....}
+ },
+ ...
+ }
+ To view a complete output, please refer to backend doc.
+ """
+
+ adapter_info = adapter_db.get_adapter(adapter_id, user=user)
+ metadata = cluster_db.get_cluster_metadata(cluster_id, user=user)
+ adapter_info.update({const.METADATA: metadata})
+
+ for flavor_info in adapter_info[const.FLAVORS]:
+ roles = flavor_info[const.ROLES]
+ flavor_info[const.ROLES] = ActionHelper._get_role_names(roles)
+
+ return adapter_info
+
+ @staticmethod
+ def _get_role_names(roles):
+ return [role[const.NAME] for role in roles]
+
+ @staticmethod
+ def get_cluster_info(cluster_id, user):
+ """Get cluster information.Return a dictionary as below,
+
+ {
+ "id": 1,
+ "adapter_id": 1,
+ "os_version": "CentOS-6.5-x86_64",
+ "name": "cluster_01",
+ "flavor": {
+ "flavor_name": "zzz",
+ "template": "xx.tmpl",
+ "roles": [...]
+ }
+ "os_config": {..},
+ "package_config": {...},
+ "deployed_os_config": {},
+ "deployed_package_config": {},
+ "owner": "xxx"
+ }
+ """
+
+ cluster_info = cluster_db.get_cluster(cluster_id, user=user)
+
+ # convert roles retrieved from db into a list of role names
+ roles_info = cluster_info.setdefault(
+ const.FLAVOR, {}).setdefault(const.ROLES, [])
+ cluster_info[const.FLAVOR][const.ROLES] = \
+ ActionHelper._get_role_names(roles_info)
+
+ # get cluster config info
+ cluster_config = cluster_db.get_cluster_config(cluster_id, user=user)
+ cluster_info.update(cluster_config)
+
+ deploy_config = cluster_db.get_cluster_deployed_config(cluster_id,
+ user=user)
+ cluster_info.update(deploy_config)
+
+ return cluster_info
+
+ @staticmethod
+ def get_hosts_info(cluster_id, hosts_id_list, user):
+ """Get hosts information. Return a dictionary as below,
+
+ {
+ "hosts": {
+ 1($host_id): {
+ "reinstall_os": True,
+ "mac": "xxx",
+ "name": "xxx",
+ "roles": [xxx, yyy]
+ },
+ "networks": {
+ "eth0": {
+ "ip": "192.168.1.1",
+ "netmask": "255.255.255.0",
+ "is_mgmt": True,
+ "is_promiscuous": False,
+ "subnet": "192.168.1.0/24"
+ },
+ "eth1": {...}
+ },
+ "os_config": {},
+ "package_config": {},
+ "deployed_os_config": {},
+ "deployed_package_config": {}
+ },
+ 2: {...},
+ ....
+ }
+ }
+ """
+
+ hosts_info = {}
+ for host_id in hosts_id_list:
+ info = cluster_db.get_cluster_host(cluster_id, host_id, user=user)
+ logging.debug("checking on info %r %r" % (host_id, info))
+
+ info[const.ROLES] = ActionHelper._get_role_names(info[const.ROLES])
+
+ # TODO(grace): Is following line necessary??
+ info.setdefault(const.ROLES, [])
+
+ config = cluster_db.get_cluster_host_config(cluster_id,
+ host_id,
+ user=user)
+ info.update(config)
+
+ networks = info[const.NETWORKS]
+ networks_dict = {}
+ # Convert networks from list to dictionary format
+ for entry in networks:
+ nic_info = {}
+ nic_info = {
+ entry[const.NIC]: {
+ const.IP_ADDR: entry[const.IP_ADDR],
+ const.NETMASK: entry[const.NETMASK],
+ const.MGMT_NIC_FLAG: entry[const.MGMT_NIC_FLAG],
+ const.PROMISCUOUS_FLAG: entry[const.PROMISCUOUS_FLAG],
+ const.SUBNET: entry[const.SUBNET]
+ }
+ }
+ networks_dict.update(nic_info)
+
+ info[const.NETWORKS] = networks_dict
+
+ hosts_info[host_id] = info
+
+ return hosts_info
+
+ @staticmethod
+ def save_deployed_config(deployed_config, user):
+ """Save deployed config."""
+ cluster_config = deployed_config[const.CLUSTER]
+ cluster_id = cluster_config[const.ID]
+ del cluster_config[const.ID]
+
+ cluster_db.update_cluster_deployed_config(cluster_id, user=user,
+ **cluster_config)
+
+ hosts_id_list = deployed_config[const.HOSTS].keys()
+ for host_id in hosts_id_list:
+ config = deployed_config[const.HOSTS][host_id]
+ cluster_db.update_cluster_host_deployed_config(cluster_id,
+ host_id,
+ user=user,
+ **config)
+
+ @staticmethod
+ def update_state(
+ cluster_id, host_id_list, user, **kwargs
+ ):
+ # update all clusterhosts state
+ for host_id in host_id_list:
+ cluster_db.update_cluster_host_state(
+ cluster_id,
+ host_id,
+ user=user,
+ **kwargs
+ )
+
+ # update cluster state
+ cluster_db.update_cluster_state(
+ cluster_id,
+ user=user,
+ **kwargs
+ )
+
+ @staticmethod
+ def delete_cluster(
+ cluster_id, host_id_list, user, delete_underlying_host=False
+ ):
+ """Delete cluster.
+
+ If delete_underlying_host is set, underlying hosts will also
+ be deleted.
+ """
+ if delete_underlying_host:
+ for host_id in host_id_list:
+ host_db.del_host(
+ host_id, True, True, user=user
+ )
+ cluster_db.del_cluster(
+ cluster_id, True, True, user=user
+ )
+
+ @staticmethod
+ def delete_cluster_host(
+ cluster_id, host_id, user, delete_underlying_host=False
+ ):
+ """Delete clusterhost.
+
+ If delete_underlying_host set, also delete underlying host.
+ """
+ if delete_underlying_host:
+ host_db.del_host(
+ host_id, True, True, user=user
+ )
+ cluster_db.del_cluster_host(
+ cluster_id, host_id, True, True, user=user
+ )
+
+ @staticmethod
+ def delete_host(host_id, user):
+ host_db.del_host(
+ host_id, True, True, user=user
+ )
+
+ @staticmethod
+ def host_ready(host_id, from_database_only, user):
+ """Trigger host ready."""
+ host_db.update_host_state_internal(
+ host_id, from_database_only=from_database_only,
+ user=user, ready=True
+ )
+
+ @staticmethod
+ def cluster_host_ready(
+ cluster_id, host_id, from_database_only, user
+ ):
+ """Trigger clusterhost ready."""
+ cluster_db.update_cluster_host_state_internal(
+ cluster_id, host_id, from_database_only=from_database_only,
+ user=user, ready=True
+ )
+
+ @staticmethod
+ def is_cluster_os_ready(cluster_id, user=None):
+ return cluster_db.is_cluster_os_ready(cluster_id, user=user)
+
+ @staticmethod
+ def cluster_ready(cluster_id, from_database_only, user):
+ """Trigger cluster ready."""
+ cluster_db.update_cluster_state_internal(
+ cluster_id, from_database_only=from_database_only,
+ user=user, ready=True
+ )
+
+ @staticmethod
+ def get_machine_IPMI(machine_id, user):
+ machine_info = machine_db.get_machine(machine_id, user=user)
+ return machine_info[const.IPMI_CREDS]
diff --git a/compass-tasks/apiclient/__init__.py b/compass-tasks/apiclient/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/compass-tasks/apiclient/__init__.py
diff --git a/compass-tasks/apiclient/example.py b/compass-tasks/apiclient/example.py
new file mode 100755
index 0000000..4c01b98
--- /dev/null
+++ b/compass-tasks/apiclient/example.py
@@ -0,0 +1,463 @@
+#!/usr/bin/python
+# copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Example code to deploy a cluster by compass client api."""
+import os
+import re
+import sys
+import time
+
+# from compass.apiclient.restful import Client
+from restful import Client
+
+COMPASS_SERVER_URL = 'http://localhost/api'
+COMPASS_LOGIN_EMAIL = 'admin@huawei.com'
+COMPASS_LOGIN_PASSWORD = 'admin'
+SWITCH_IP = '172.29.8.40'
+SWITCH_SNMP_VERSION = '2c'
+SWITCH_SNMP_COMMUNITY = 'public'
+CLUSTER_NAME = 'test_cluster'
+HOST_NAME_PREFIX = 'host'
+SERVICE_USERNAME = 'service'
+SERVICE_PASSWORD = 'service'
+CONSOLE_USERNAME = 'console'
+CONSOLE_PASSWORD = 'console'
+HA_VIP = ''
+
+MANAGEMENT_IP_START = '10.145.88.130'
+MANAGEMENT_IP_END = '10.145.88.254'
+MANAGEMENT_IP_GATEWAY = '10.145.88.1'
+MANAGEMENT_NETMASK = '255.255.255.0'
+MANAGEMENT_NIC = 'eth0'
+MANAGEMENT_PROMISC = 0
+TENANT_IP_START = '192.168.10.130'
+TENANT_IP_END = '192.168.10.255'
+TENANT_IP_GATEWAY = '192.168.10.1'
+TENANT_NETMASK = '255.255.255.0'
+TENANT_NIC = 'eth0'
+TENANT_PROMISC = 0
+PUBLIC_IP_START = '12.234.32.130'
+PUBLIC_IP_END = '12.234.32.255'
+PUBLIC_IP_GATEWAY = '12.234.32.1'
+PUBLIC_NETMASK = '255.255.255.0'
+PUBLIC_NIC = 'eth1'
+PUBLIC_PROMISC = 1
+STORAGE_IP_START = '172.16.100.130'
+STORAGE_IP_END = '172.16.100.255'
+STORAGE_NETMASK = '255.255.255.0'
+STORAGE_IP_GATEWAY = '172.16.100.1'
+STORAGE_NIC = 'eth0'
+STORAGE_PROMISC = 0
+HOME_PERCENTAGE = 5
+TMP_PERCENTAGE = 5
+VAR_PERCENTAGE = 10
+HOST_OS = 'CentOS-6.5-x86_64'
+
+
+PRESET_VALUES = {
+ 'LANGUAGE': 'EN',
+ 'TIMEZONE': 'GMT',
+ 'HTTPS_PROXY': 'http://10.145.89.100:3128',
+ 'NO_PROXY': ['127.0.0.1'],
+ 'DOMAIN': 'ods.com',
+ 'NAMESERVERS': ['10.145.89.100'],
+ 'NTP_SERVER': '10.145.89.100',
+ 'GATEWAY': '10.145.88.1',
+ 'PROXY': 'http://10.145.89.100:3128',
+ 'OS_NAME_PATTERN': 'CentOS.*',
+ 'ADAPTER_NAME': 'openstack_icehouse',
+ 'FLAVOR_PATTERN': 'allinone.*',
+ 'ROLES_LIST': ['allinone-compute'],
+ 'MACHINES_TO_ADD': ['00:0c:29:a7:ea:4b'],
+ 'BUILD_TIMEOUT': 60,
+ 'SEARCH_PATH': ['ods.com'],
+ 'SERVER_USERNAME': 'root',
+ 'SERVER_PASSWORD': 'root'
+}
+for v in PRESET_VALUES:
+ if v in os.environ.keys():
+ PRESET_VALUES[v] = os.environ.get(v)
+ print (v + PRESET_VALUES[v] + " is set by env variables")
+ else:
+ print (PRESET_VALUES[v])
+
+# instantiate a client
+client = Client(COMPASS_SERVER_URL)
+
+# login
+status, response = client.login(COMPASS_LOGIN_EMAIL, COMPASS_LOGIN_PASSWORD)
+print '============================================================'
+print 'login status: %s response: %s' % (status, response)
+if status >= 400:
+ sys.exit(1)
+
+# list all switches
+status, response = client.list_switches()
+print '============================================================='
+print 'get all switches status: %s response: %s' % (status, response)
+
+# add a switch
+status, response = client.add_switch(
+ SWITCH_IP,
+ SWITCH_SNMP_VERSION,
+ SWITCH_SNMP_COMMUNITY
+)
+print '============================================'
+print 'adding a switch..status: %s, response: %s' % (status, response)
+
+# if switch already exists, get one from all switches
+switch = None
+if status < 400:
+ switch = response
+else:
+ status, response = client.list_switches()
+ print '========================================='
+ print 'list switches status %s response %s' % (status, response)
+ if status >= 400:
+ sys.exit(1)
+ for switch_ in response:
+ if switch_['ip'] == SWITCH_IP:
+ switch = switch_
+ break
+
+switch_id = switch['id']
+switch_ip = switch['ip']
+print '======================'
+print 'switch has been set as %s' % switch_ip
+
+# wait till switch state becomes under_monitoring
+while switch['state'] != 'under_monitoring':
+ print 'waiting for state to become under_monitoring'
+ client.poll_switch(switch_id)
+ status, resp = client.get_switch(switch_id)
+ print '====================================='
+ print 'poll switch status %s response %s' % (status, resp)
+ switch = resp
+ print 'switch is in state: %s' % switch['state']
+ time.sleep(5)
+
+print '========================================='
+print 'switch state now is %s' % (switch['state'])
+
+# create a machine list
+machine_macs = {}
+machines = {}
+for machine in PRESET_VALUES['MACHINES_TO_ADD']:
+ status, response = client.list_machines(mac=machine)
+ print '============================================'
+ print 'list machines status %s response %s' % (status, response)
+ if status >= 400:
+ sys.exit(1)
+ if status == 200 and response != []:
+ machine_id = response[0]['id']
+ machine_macs[machine_id] = response[0]['mac']
+ machines = response
+
+print '================================='
+print 'found machines are : %s' % machines
+
+machines_to_add = PRESET_VALUES['MACHINES_TO_ADD']
+if set(machine_macs.values()) != set(machines_to_add):
+ print 'only found macs %s while expected are %s' % (
+ machine_macs.values(), machines_to_add)
+ sys.exit(1)
+
+# list all adapters
+status, response = client.list_adapters()
+print '==============================='
+print 'all adapters are: %s' % response
+if status >= 400:
+ sys.exit(1)
+
+adapters = response
+adapter_id = None
+os_id = None
+flavor_id = None
+adapter_name = PRESET_VALUES['ADPATER_NAME']
+os_pattern = re.compile(PRESET_VALUES['OS_NAME_PATTERN'])
+flavor_pattern = re.compile(PRESET_VALUES['FLAVOR_PATTERN'])
+for adapter in adapters:
+ if adapter_name == adapter['name']:
+ adapter_id = adapter['id']
+ for supported_os in adapter['supported_oses']:
+ if os_pattern.match(supported_os['name']):
+ os_id = supported_os['id']
+ break
+ for flavor in adapter['flavors']:
+ if flavor_pattern.match(flavor['name']):
+ flavor_id = flavor['id']
+ if adapter_id and os_id and flavor_id:
+ break
+
+print '======================================================='
+print 'using adapter %s os %s flavor %s to deploy cluster' % (
+ adapter_id, os_id, flavor_id
+)
+
+# add a cluster
+status, response = client.add_cluster(
+ CLUSTER_NAME,
+ adapter_id,
+ os_id,
+ flavor_id
+)
+print '==============================================================='
+print 'add cluster %s status %s: %s' % (CLUSTER_NAME, status, response)
+if status >= 400:
+ sys.exit(1)
+
+status, response = client.list_clusters(name=CLUSTER_NAME)
+print '================================================================'
+print 'list clusters %s status %s: %s' % (CLUSTER_NAME, status, response)
+if status >= 400:
+ sys.exit(1)
+
+cluster = response[0]
+cluster_id = cluster['id']
+
+print '=================='
+print 'cluster is %s' % cluster
+
+# Add hosts to the cluster
+machines_dict = {}
+machine_id_list = []
+for machine in machines:
+ id_mapping = {}
+ id_mapping['machine_id'] = machine['id']
+ machine_id_list.append(id_mapping)
+
+machines_dict['machines'] = machine_id_list
+
+status, response = client.add_hosts_to_cluster(
+ cluster_id, machines_dict
+)
+print '==================================='
+print 'add hosts %s to cluster status %s response %s' % (
+ machines_dict, status, response)
+if status >= 400:
+ sys.exit(1)
+
+# Add two subnets
+subnet_1 = '10.145.89.0/24'
+subnet_2 = '192.168.100.0/24'
+
+status, response = client.add_subnet(subnet_1)
+print '=================='
+print 'add subnet %s status %s: %s' % (subnet_1, status, response)
+if status >= 400:
+ sys.exit(1)
+
+status, response = client.add_subnet(subnet_2)
+print '=================='
+print 'add subnet %s status %s: %s' % (subnet_2, status, response)
+if status >= 400:
+ sys.exit(1)
+
+status, subnet1 = client.list_subnets(subnet=subnet_1)
+print '==========================================================='
+print 'list subnet %s status %s: %s' % (subnet_1, status, subnet1)
+if status >= 400:
+ sys.exit(1)
+
+status, subnet2 = client.list_subnets(subnet=subnet_2)
+print '==========================================================='
+print 'list subnet %s status %s: %s' % (subnet_2, status, subnet2)
+if status >= 400:
+ sys.exit(1)
+
+subnet1_id = subnet1[0]['id']
+subnet2_id = subnet2[0]['id']
+print '========================'
+print 'subnet1 has id: %s, subnet is %s' % (subnet1_id, subnet1)
+print 'subnet2 has id: %s, subnet is %s' % (subnet2_id, subnet2)
+
+# Add host network
+status, response = client.list_cluster_hosts(cluster_id)
+print '================================================'
+print 'list cluster hosts status %s: %s' % (status, response)
+if status >= 400:
+ sys.exit(1)
+
+host = response[0]
+host_id = host['id']
+print '=================='
+print 'host is: %s' % host
+
+status, response = client.add_host_network(
+ host_id,
+ 'eth0',
+ '10.145.89.200',
+ subnet1_id,
+ is_mgmt=True
+)
+print '======================='
+print 'add eth0 network status %s: %s' % (status, response)
+if status >= 400:
+ sys.exit(1)
+
+status, response = client.add_host_network(
+ host_id,
+ 'eth1',
+ '192.168.100.200',
+ subnet2_id,
+ is_promiscuous=True
+)
+print '======================='
+print 'add eth1 network status %s: %s' % (status, response)
+if status >= 400:
+ sys.exit(1)
+
+# Update os config to cluster
+cluster_os_config = {
+ 'general': {
+ 'language': PRESET_VALUES['LANGUAGE'],
+ 'timezone': PRESET_VALUES['TIMEZONE'],
+ 'http_proxy': PRESET_VALUES['PROXY'],
+ 'https_proxy': PRESET_VALUES['HTTPS_PROXY'],
+ 'no_proxy': PRESET_VALUES['NO_PROXY'],
+ 'ntp_server': PRESET_VALUES['NTP_SERVER'],
+ 'dns_servers': PRESET_VALUES['NAMESERVERS'],
+ 'domain': PRESET_VALUES['DOMAIN'],
+ 'search_path': PRESET_VALUES['SEARCH_PATH'],
+ 'default_gateway': PRESET_VALUES['GATEWAY']
+ },
+ 'server_credentials': {
+ 'username': PRESET_VALUES['SERVER_USERNAME'],
+ 'password': PRESET_VALUES['SERVER_PASSWORD']
+ },
+ 'partition': {
+ '/var': {
+ 'percentage': VAR_PERCENTAGE,
+ },
+ '/home': {
+ 'percentage': HOME_PERCENTAGE,
+ }
+ }
+}
+
+
+cluster_package_config = {
+ 'security': {
+ 'service_credentials': {
+ 'image': {
+ 'username': SERVICE_USERNAME,
+ 'password': SERVICE_PASSWORD
+ },
+ 'compute': {
+ 'username': SERVICE_USERNAME,
+ 'password': SERVICE_PASSWORD
+ },
+ 'dashboard': {
+ 'username': SERVICE_USERNAME,
+ 'password': SERVICE_PASSWORD
+ },
+ 'identity': {
+ 'username': SERVICE_USERNAME,
+ 'password': SERVICE_PASSWORD
+ },
+ 'metering': {
+ 'username': SERVICE_USERNAME,
+ 'password': SERVICE_PASSWORD
+ },
+ 'rabbitmq': {
+ 'username': SERVICE_USERNAME,
+ 'password': SERVICE_PASSWORD
+ },
+ 'volume': {
+ 'username': SERVICE_USERNAME,
+ 'password': SERVICE_PASSWORD
+ },
+ 'mysql': {
+ 'username': SERVICE_USERNAME,
+ 'password': SERVICE_PASSWORD
+ }
+ },
+ 'console_credentials': {
+ 'admin': {
+ 'username': CONSOLE_USERNAME,
+ 'password': CONSOLE_PASSWORD
+ },
+ 'compute': {
+ 'username': CONSOLE_USERNAME,
+ 'password': CONSOLE_PASSWORD
+ },
+ 'dashboard': {
+ 'username': CONSOLE_USERNAME,
+ 'password': CONSOLE_PASSWORD
+ },
+ 'image': {
+ 'username': CONSOLE_USERNAME,
+ 'password': CONSOLE_PASSWORD
+ },
+ 'metering': {
+ 'username': CONSOLE_USERNAME,
+ 'password': CONSOLE_PASSWORD
+ },
+ 'network': {
+ 'username': CONSOLE_USERNAME,
+ 'password': CONSOLE_PASSWORD
+ },
+ 'object-store': {
+ 'username': CONSOLE_USERNAME,
+ 'password': CONSOLE_PASSWORD
+ },
+ 'volume': {
+ 'username': CONSOLE_USERNAME,
+ 'password': CONSOLE_PASSWORD
+ }
+ }
+ },
+ 'network_mapping': {
+ 'management': MANAGEMENT_NIC,
+ 'tenant': TENANT_NIC,
+ 'storage': STORAGE_NIC,
+ 'public': PUBLIC_NIC
+ }
+}
+
+status, response = client.update_cluster_config(
+ cluster_id,
+ cluster_os_config,
+ cluster_package_config
+)
+
+print '======================================='
+print 'cluster %s update status %s: %s' % (
+ cluster_id, status, response)
+if status >= 400:
+ sys.exit(1)
+
+status, response = client.update_cluster_host(
+ cluster_id, host_id, roles=PRESET_VALUES['ROLES_LIST'])
+print '================================================='
+print 'update cluster host %s/%s status %s: %s' % (
+ cluster_id, host_id, status, response)
+if status >= 400:
+ sys.exit(1)
+
+# Review and deploy
+status, response = client.review_cluster(
+ cluster_id, review={'hosts': [host_id]})
+print '======================================='
+print 'reviewing cluster status %s: %s' % (status, response)
+if status >= 400:
+ sys.exit(1)
+
+status, response = client.deploy_cluster(
+ cluster_id, deploy={'hosts': [host_id]})
+print '======================================='
+print 'deploy cluster status %s: %s' % (status, response)
+if status >= 400:
+ sys.exit(1)
diff --git a/compass-tasks/apiclient/restful.py b/compass-tasks/apiclient/restful.py
new file mode 100644
index 0000000..bb82922
--- /dev/null
+++ b/compass-tasks/apiclient/restful.py
@@ -0,0 +1,1102 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Compass api client library.
+"""
+
+import json
+import logging
+import requests
+
+
+class Client(object):
+ """compass restful api wrapper"""
+
+ def __init__(self, url, headers=None, proxies=None, stream=None):
+ logging.info('create api client %s', url)
+ self.url_ = url
+ self.session_ = requests.Session()
+
+ if headers:
+ self.session_.headers.update(headers)
+ self.session_.headers.update({
+ 'Accept': 'application/json'
+ })
+
+ if proxies is not None:
+ self.session_.proxies = proxies
+
+ if stream is not None:
+ self.session_.stream = stream
+
+ def __del__(self):
+ self.session_.close()
+
+ @classmethod
+ def _get_response(cls, resp):
+ response_object = {}
+ try:
+ response_object = resp.json()
+ except Exception as error:
+ logging.error('failed to load object from %s: %s',
+ resp.url, resp.content)
+ logging.exception(error)
+ response_object['status'] = 'Json Parsing Failed'
+ response_object['message'] = resp.content
+
+ return resp.status_code, response_object
+
+ def _get(self, req_url, data=None):
+ url = '%s%s' % (self.url_, req_url)
+ logging.debug('get %s with data %s', url, data)
+ if data:
+ resp = self.session_.get(url, params=data)
+ else:
+ resp = self.session_.get(url)
+
+ return self._get_response(resp)
+
+ def _post(self, req_url, data=None):
+ url = '%s%s' % (self.url_, req_url)
+ logging.debug('post %s with data %s', url, data)
+ if data:
+ resp = self.session_.post(url, json.dumps(data))
+ else:
+ resp = self.session_.post(url)
+
+ return self._get_response(resp)
+
+ def _put(self, req_url, data=None):
+ """encapsulate put method."""
+ url = '%s%s' % (self.url_, req_url)
+ logging.debug('put %s with data %s', url, data)
+ if data:
+ resp = self.session_.put(url, json.dumps(data))
+ else:
+ resp = self.session_.put(url)
+
+ return self._get_response(resp)
+
+ def _patch(self, req_url, data=None):
+ url = '%s%s' % (self.url_, req_url)
+ logging.debug('patch %s with data %s', url, data)
+ if data:
+ resp = self.session_.patch(url, json.dumps(data))
+ else:
+ resp = self.session_.patch(url)
+
+ return self._get_response(resp)
+
+ def _delete(self, req_url):
+ url = '%s%s' % (self.url_, req_url)
+ logging.debug('delete %s', url)
+ return self._get_response(self.session_.delete(url))
+
+ def login(self, email, password):
+ credential = {}
+ credential['email'] = email
+ credential['password'] = password
+ return self._post('/users/login', data=credential)
+
+ def get_token(self, email, password):
+ credential = {}
+ credential['email'] = email
+ credential['password'] = password
+ status, resp = self._post('/users/token', data=credential)
+ if status < 400:
+ self.session_.headers.update({'X-Auth-Token': resp['token']})
+ return status, resp
+
+ def get_users(self):
+ users = self._get('/users')
+ return users
+
+ def list_switches(
+ self,
+ switch_ips=None,
+ switch_ip_networks=None):
+ """list switches."""
+ params = {}
+ if switch_ips:
+ params['switchIp'] = switch_ips
+
+ if switch_ip_networks:
+ params['switchIpNetwork'] = switch_ip_networks
+
+ switchlist = self._get('/switches', data=params)
+ return switchlist
+
+ def get_switch(self, switch_id):
+ return self._get('/switches/%s' % switch_id)
+
+ def add_switch(
+ self,
+ switch_ip,
+ version=None,
+ community=None,
+ raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ data['ip'] = switch_ip
+ data['credentials'] = {}
+ if version:
+ data['credentials']['version'] = version
+
+ if community:
+ data['credentials']['community'] = community
+
+ return self._post('/switches', data=data)
+
+ def update_switch(self, switch_id, state='initialized',
+ version='2c', community='public', raw_data={}):
+ data = {}
+ if raw_data:
+ data = raw_data
+
+ else:
+ data['credentials'] = {}
+ if version:
+ data['credentials']['version'] = version
+
+ if community:
+ data['credentials']['community'] = community
+
+ if state:
+ data['state'] = state
+
+ return self._put('/switches/%s' % switch_id, data=data)
+
+ def delete_switch(self, switch_id):
+ return self._delete('/switches/%s' % switch_id)
+
+ def list_switch_machines(self, switch_id, port=None, vlans=None,
+ tag=None, location=None):
+ data = {}
+ if port:
+ data['port'] = port
+
+ if vlans:
+ data['vlans'] = vlans
+
+ if tag:
+ data['tag'] = tag
+
+ if location:
+ data['location'] = location
+
+ return self._get('/switches/%s/machines' % switch_id, data=data)
+
+ def get_switch_machine(self, switch_id, machine_id):
+ return self._get('/switches/%s/machines/%s' % (switch_id, machine_id))
+
+ def list_switch_machines_hosts(self, switch_id, port=None, vlans=None,
+ mac=None, tag=None, location=None,
+ os_name=None, os_id=None):
+
+ data = {}
+ if port:
+ data['port'] = port
+
+ if vlans:
+ data['vlans'] = vlans
+
+ if mac:
+ data['mac'] = mac
+
+ if tag:
+ data['tag'] = tag
+
+ if location:
+ data['location'] = location
+
+ if os_name:
+ data['os_name'] = os_name
+
+ if os_id:
+ data['os_id'] = os_id
+
+ return self._get('/switches/%s/machines-hosts' % switch_id, data=data)
+
+ def add_switch_machine(self, switch_id, mac=None, port=None,
+ vlans=None, ipmi_credentials=None,
+ tag=None, location=None, raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if mac:
+ data['mac'] = mac
+
+ if port:
+ data['port'] = port
+
+ if vlans:
+ data['vlans'] = vlans
+
+ if ipmi_credentials:
+ data['ipmi_credentials'] = ipmi_credentials
+
+ if tag:
+ data['tag'] = tag
+
+ if location:
+ data['location'] = location
+
+ return self._post('/switches/%s/machines' % switch_id, data=data)
+
+ def update_switch_machine(self, switch_id, machine_id, port=None,
+ vlans=None, ipmi_credentials=None, tag=None,
+ location=None, raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if port:
+ data['port'] = port
+
+ if vlans:
+ data['vlans'] = vlans
+
+ if ipmi_credentials:
+ data['ipmi_credentials'] = ipmi_credentials
+
+ if tag:
+ data['tag'] = tag
+
+ if location:
+ data['location'] = location
+
+ return self._put('/switches/%s/machines/%s' %
+ (switch_id, machine_id), data=data)
+
+ def delete_switch_machine(self, switch_id, machine_id):
+ return self._delete('/switches/%s/machines/%s' %
+ (switch_id, machine_id))
+
+ # test these
+ def poll_switch(self, switch_id):
+ data = {}
+ data['find_machines'] = None
+ return self._post('/switches/%s/action' % switch_id, data=data)
+
+ def add_group_switch_machines(self, switch_id, group_machine_ids):
+ data = {}
+ data['add_machines'] = group_machine_ids
+ return self._post('/switches/%s/action' % switch_id, data=data)
+
+ def remove_group_switch_machines(self, switch_id, group_machine_ids):
+ data = {}
+ data['remove_machines'] = group_machine_ids
+ return self._post('/switches/%s/action' % switch_id, data=data)
+
+ def update_group_switch_machines(self, switch_id, group_machines):
+ data = {}
+ data['set_machines'] = group_machines
+ return self._post('/switches/%s/action' % switch_id, data=data)
+ # end
+
+ def list_switchmachines(self, switch_ip_int=None, port=None, vlans=None,
+ mac=None, tag=None, location=None):
+ data = {}
+ if switch_ip_int:
+ data['switch_ip_int'] = switch_ip_int
+
+ if port:
+ data['port'] = port
+
+ if vlans:
+ data['vlans'] = vlans
+
+ if mac:
+ data['mac'] = mac
+
+ if tag:
+ data['tag'] = tag
+
+ if location:
+ data['location'] = location
+
+ return self._get('/switch-machines', data=data)
+
+ def list_switchmachines_hosts(self, switch_ip_int=None, port=None,
+ vlans=None, mac=None, tag=None,
+ location=None, os_name=None, os_id=None):
+
+ data = {}
+ if switch_ip_int:
+ data['switch_ip_int'] = switch_ip_int
+
+ if port:
+ data['port'] = port
+
+ if vlans:
+ data['vlans'] = vlans
+
+ if mac:
+ data['mac'] = mac
+
+ if tag:
+ data['tag'] = tag
+
+ if location:
+ data['location'] = location
+
+ if os_name:
+ data['os_name'] = os_name
+
+ if os_id:
+ data['os_id'] = os_id
+
+ return self._get('/switches-machines-hosts', data=data)
+
+ def show_switchmachine(self, switchmachine_id):
+ return self._get('/switch-machines/%s' % switchmachine_id)
+
+ def update_switchmachine(self, switchmachine_id,
+ port=None, vlans=None, raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if port:
+ data['port'] = port
+
+ if vlans:
+ data['vlans'] = vlans
+
+ return self._put('/switch-machines/%s' % switchmachine_id, data=data)
+
+ def patch_switchmachine(self, switchmachine_id,
+ vlans=None, raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ elif vlans:
+ data['vlans'] = vlans
+
+ return self._patch('/switch-machines/%s' % switchmachine_id, data=data)
+
+ def delete_switchmachine(self, switchmachine_id):
+ return self._delete('/switch-machines/%s' % switchmachine_id)
+
+ def list_machines(self, mac=None, tag=None, location=None):
+ data = {}
+ if mac:
+ data['mac'] = mac
+
+ if tag:
+ data['tag'] = tag
+
+ if location:
+ data['location'] = location
+
+ return self._get('/machines', data=data)
+
+ def get_machine(self, machine_id):
+ data = {}
+ if id:
+ data['id'] = id
+
+ return self._get('/machines/%s' % machine_id, data=data)
+
+ def update_machine(self, machine_id, ipmi_credentials=None, tag=None,
+ location=None, raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if ipmi_credentials:
+ data['ipmi_credentials'] = ipmi_credentials
+
+ if tag:
+ data['tag'] = tag
+
+ if location:
+ data['location'] = location
+
+ return self._put('/machines/%s' % machine_id, data=data)
+
+ def patch_machine(self, machine_id, ipmi_credentials=None,
+ tag=None, location=None,
+ raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if ipmi_credentials:
+ data['ipmi_credentials'] = ipmi_credentials
+
+ if tag:
+ data['tag'] = tag
+
+ if location:
+ data['location'] = location
+
+ return self._patch('/machines/%s' % machine_id, data=data)
+
+ def delete_machine(self, machine_id):
+ return self._delete('machines/%s' % machine_id)
+
+ def list_subnets(self, subnet=None, name=None):
+ data = {}
+ if subnet:
+ data['subnet'] = subnet
+
+ if name:
+ data['name'] = name
+
+ return self._get('/subnets', data=data)
+
+ def get_subnet(self, subnet_id):
+ return self._get('/subnets/%s' % subnet_id)
+
+ def add_subnet(self, subnet, name=None, raw_data=None):
+ data = {}
+ data['subnet'] = subnet
+ if raw_data:
+ data.update(raw_data)
+ else:
+ if name:
+ data['name'] = name
+
+ return self._post('/subnets', data=data)
+
+ def update_subnet(self, subnet_id, subnet=None,
+ name=None, raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if subnet:
+ data['subnet'] = subnet
+
+ if name:
+ data['name'] = name
+ return self._put('/subnets/%s' % subnet_id, data=data)
+
+ def delete_subnet(self, subnet_id):
+ return self._delete('/subnets/%s' % subnet_id)
+
+ def list_adapters(self, name=None):
+ data = {}
+ if name:
+ data['name'] = name
+
+ return self._get('/adapters', data=data)
+
+ def get_adapter(self, adapter_id):
+ return self._get('/adapters/%s' % adapter_id)
+
+ def get_adapter_roles(self, adapter_id):
+ return self._get('/adapters/%s/roles' % adapter_id)
+
+ def get_adapter_metadata(self, adapter_id):
+ return self._get('/adapters/%s/metadata' % adapter_id)
+
+ def get_os_metadata(self, os_id):
+ return self._get('/oses/%s/metadata' % os_id)
+
+ def list_clusters(self, name=None, os_name=None,
+ owner=None,
+ adapter_id=None):
+ data = {}
+ if name:
+ data['name'] = name
+
+ if os_name:
+ data['os_name'] = os_name
+
+ if owner:
+ data['owner'] = owner
+
+ if adapter_id:
+ data['adapter_id'] = adapter_id
+
+ return self._get('/clusters', data=data)
+
+ def get_cluster(self, cluster_id):
+ return self._get('/clusters/%s' % cluster_id)
+
+ def add_cluster(self, name, adapter_id, os_id,
+ flavor_id=None, raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if flavor_id:
+ data['flavor_id'] = flavor_id
+ data['name'] = name
+ data['adapter_id'] = adapter_id
+ data['os_id'] = os_id
+
+ return self._post('/clusters', data=data)
+
+ def update_cluster(self, cluster_id, name=None,
+ reinstall_distributed_system=None,
+ raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if name:
+ data['name'] = name
+
+ if reinstall_distributed_system:
+ data['reinstall_distributed_system'] = (
+ reinstall_distributed_system
+ )
+ return self._put('/clusters/%s' % cluster_id, data=data)
+
+ def delete_cluster(self, cluster_id):
+ return self._delete('/clusters/%s' % cluster_id)
+
+ def get_cluster_config(self, cluster_id):
+ return self._get('/clusters/%s/config' % cluster_id)
+
+ def get_cluster_metadata(self, cluster_id):
+ return self._get('/clusters/%s/metadata' % cluster_id)
+
+ def update_cluster_config(self, cluster_id, os_config=None,
+ package_config=None, config_step=None,
+ raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+
+ if os_config:
+ data['os_config'] = os_config
+
+ if package_config:
+ data['package_config'] = package_config
+
+ if config_step:
+ data['config_step'] = config_step
+
+ return self._put('/clusters/%s/config' % cluster_id, data=data)
+
+ def patch_cluster_config(self, cluster_id, os_config=None,
+ package_config=None, config_step=None,
+ raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+
+ if os_config:
+ data['os_config'] = os_config
+
+ if package_config:
+ data['package_config'] = package_config
+
+ if config_step:
+ data['config_step'] = config_step
+
+ return self._patch('/clusters/%s/config' % cluster_id, data=data)
+
+ def delete_cluster_config(self, cluster_id):
+ return self._delete('/clusters/%s/config' % cluster_id)
+
+ # test these
+ def add_hosts_to_cluster(self, cluster_id, hosts):
+ data = {}
+ data['add_hosts'] = hosts
+ return self._post('/clusters/%s/action' % cluster_id, data=data)
+
+ def set_hosts_in_cluster(self, cluster_id, hosts):
+ data = {}
+ data['set_hosts'] = hosts
+ return self._post('/clusters/%s/action' % cluster_id, data=data)
+
+ def remove_hosts_from_cluster(self, cluster_id, hosts):
+ data = {}
+ data['remove_hosts'] = hosts
+ return self._post('/clusters/%s/action' % cluster_id, data=data)
+
+ def review_cluster(self, cluster_id, review={}):
+ data = {}
+ data['review'] = review
+ return self._post('/clusters/%s/action' % cluster_id, data=data)
+
+ def deploy_cluster(self, cluster_id, deploy={}):
+ data = {}
+ data['deploy'] = deploy
+ return self._post('/clusters/%s/action' % cluster_id, data=data)
+
+ def redeploy_cluster(self, cluster_id, deploy={}):
+ data = {}
+ data['redeploy'] = deploy
+ return self._post('/clusters/%s/action' % cluster_id, data=data)
+
+ def get_cluster_state(self, cluster_id):
+ return self._get('/clusters/%s/state' % cluster_id)
+
+ def list_cluster_hosts(self, cluster_id):
+ return self._get('/clusters/%s/hosts' % cluster_id)
+
+ def list_clusterhosts(self):
+ return self._get('/clusterhosts')
+
+ def get_cluster_host(self, cluster_id, host_id):
+ return self._get('/clusters/%s/hosts/%s' % (cluster_id, host_id))
+
+ def get_clusterhost(self, clusterhost_id):
+ return self._get('/clusterhosts/%s' % clusterhost_id)
+
+ def add_cluster_host(self, cluster_id, machine_id=None, name=None,
+ reinstall_os=None, raw_data=None):
+ data = {}
+ data['machine_id'] = machine_id
+ if raw_data:
+ data.update(raw_data)
+ else:
+ if name:
+ data['name'] = name
+
+ if reinstall_os:
+ data['reinstall_os'] = reinstall_os
+
+ return self._post('/clusters/%s/hosts' % cluster_id, data=data)
+
+ def delete_cluster_host(self, cluster_id, host_id):
+ return self._delete('/clusters/%s/hosts/%s' %
+ (cluster_id, host_id))
+
+ def delete_clusterhost(self, clusterhost_id):
+ return self._delete('/clusterhosts/%s' % clusterhost_id)
+
+ def get_cluster_host_config(self, cluster_id, host_id):
+ return self._get('/clusters/%s/hosts/%s/config' %
+ (cluster_id, host_id))
+
+ def get_clusterhost_config(self, clusterhost_id):
+ return self._get('/clusterhosts/%s/config' % clusterhost_id)
+
+ def update_cluster_host_config(self, cluster_id, host_id,
+ os_config=None,
+ package_config=None,
+ raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if os_config:
+ data['os_config'] = os_config
+
+ if package_config:
+ data['package_config'] = package_config
+
+ return self._put('/clusters/%s/hosts/%s/config' %
+ (cluster_id, host_id), data=data)
+
+ def update_clusterhost_config(self, clusterhost_id, os_config=None,
+ package_config=None, raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+
+ else:
+ if os_config:
+ data['os_config'] = os_config
+
+ if package_config:
+ data['package_config'] = package_config
+
+ return self._put('/clusterhosts/%s/config' % clusterhost_id,
+ data=data)
+
+ def patch_cluster_host_config(self, cluster_id, host_id,
+ os_config=None,
+ package_config=None,
+ raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+
+ else:
+ if os_config:
+ data['os_config'] = os_config
+
+ if package_config:
+ data['package_config'] = package_config
+
+ return self._patch('/clusters/%s/hosts/%s/config' %
+ (cluster_id, host_id), data=data)
+
+ def patch_clusterhost_config(self, clusterhost_id, os_config=None,
+ package_config=None, raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+
+ else:
+ if os_config:
+ data['os_config'] = os_config
+
+ if package_config:
+ data['package_config'] = package_config
+
+ return self._patch('/clusterhosts/%s' % clusterhost_id, data=data)
+
+ def delete_cluster_host_config(self, cluster_id, host_id):
+ return self._delete('/clusters/%s/hosts/%s/config' %
+ (cluster_id, host_id))
+
+ def delete_clusterhost_config(self, clusterhost_id):
+ return self._delete('/clusterhosts/%s/config' % clusterhost_id)
+
+ def get_cluster_host_state(self, cluster_id, host_id):
+ return self._get('/clusters/%s/hosts/%s/state' %
+ (cluster_id, host_id))
+
+ def update_cluster_host(self, cluster_id, host_id,
+ roles=None, raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if roles:
+ data['roles'] = roles
+
+ return self._put('/clusters/%s/hosts/%s' %
+ (cluster_id, host_id), data=data)
+
+ def update_clusterhost(self, clusterhost_id,
+ roles=None, raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if roles:
+ data['roles'] = roles
+
+ return self._put('/clusterhosts/%s' % clusterhost_id, data=data)
+
+ def patch_cluster_host(self, cluster_id, host_id,
+ roles=None, raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if roles:
+ data['roles'] = roles
+
+ return self._patch('/clusters/%s/hosts/%s' %
+ (cluster_id, host_id), data=data)
+
+ def patch_clusterhost(self, clusterhost_id,
+ roles=None, raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if roles:
+ data['roles'] = roles
+
+ return self._patch('/clusterhosts/%s' % clusterhost_id, data=data)
+
+ def get_clusterhost_state(self, clusterhost_id):
+ return self._get('/clusterhosts/%s/state' % clusterhost_id)
+
+ def update_cluster_host_state(self, cluster_id, host_id, state=None,
+ percentage=None, message=None,
+ raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if state:
+ data['state'] = state
+
+ if percentage:
+ data['percentage'] = percentage
+
+ if message:
+ data['message'] = message
+
+ return self._put('/clusters/%s/hosts/%s/state' % (cluster_id, host_id),
+ data=data)
+
+ def update_clusterhost_state(self, clusterhost_id, state=None,
+ percentage=None, message=None,
+ raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if state:
+ data['state'] = state
+
+ if percentage:
+ data['percentage'] = percentage
+
+ if message:
+ data['message'] = message
+
+ return self._put('/clusterhosts/%s/state' % clusterhost_id, data=data)
+
+ def list_hosts(self, name=None, os_name=None, owner=None, mac=None):
+ data = {}
+ if name:
+ data['name'] = name
+
+ if os_name:
+ data['os_name'] = os_name
+
+ if owner:
+ data['owner'] = owner
+
+ if mac:
+ data['mac'] = mac
+
+ return self._get('/hosts', data=data)
+
+ def get_host(self, host_id):
+ return self._get('/hosts/%s' % host_id)
+
+ def list_machines_or_hosts(self, mac=None, tag=None,
+ location=None, os_name=None,
+ os_id=None):
+ data = {}
+ if mac:
+ data['mac'] = mac
+
+ if tag:
+ data['tag'] = tag
+
+ if location:
+ data['location'] = location
+
+ if os_name:
+ data['os_name'] = os_name
+
+ if os_id:
+ data['os_id'] = os_id
+
+ return self._get('/machines-hosts', data=data)
+
+ def get_machine_or_host(self, host_id):
+ return self._get('/machines-hosts/%s' % host_id)
+
+ def update_host(self, host_id, name=None,
+ reinstall_os=None, raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if name:
+ data['name'] = name
+
+ if reinstall_os:
+ data['reinstall_os'] = reinstall_os
+
+ return self._put('/hosts/%s' % host_id, data=data)
+
+ def delete_host(self, host_id):
+ return self._delete('/hosts/%s' % host_id)
+
+ def get_host_clusters(self, host_id):
+ return self._get('/hosts/%s/clusters' % host_id)
+
+ def get_host_config(self, host_id):
+ return self._get('/hosts/%s/config' % host_id)
+
+ def update_host_config(self, host_id, os_config, raw_data=None):
+ data = {}
+ data['os_config'] = os_config
+ if raw_data:
+ data.update(raw_data)
+
+ return self._put('/hosts/%s/config' % host_id, data=data)
+
+ def patch_host_config(self, host_id, os_config, raw_data=None):
+ data = {}
+ data['os_config'] = os_config
+ if raw_data:
+ data.update(raw_data)
+
+ return self._patch('/hosts/%s/config' % host_id, data=data)
+
+ def delete_host_config(self, host_id):
+ return self._delete('/hosts/%s/config' % host_id)
+
+ def list_host_networks(self, host_id, interface=None, ip=None,
+ subnet=None, is_mgmt=None, is_promiscuous=None):
+ data = {}
+ if interface:
+ data['interface'] = interface
+
+ if ip:
+ data['ip'] = ip
+
+ if subnet:
+ data['subnet'] = subnet
+
+ if is_mgmt:
+ data['is_mgmt'] = is_mgmt
+
+ if is_promiscuous:
+ data['is_promiscuous'] = is_promiscuous
+
+ return self._get('/hosts/%s/networks' % host_id, data=data)
+
+ def list_all_host_networks(self, interface=None, ip=None, subnet=None,
+ is_mgmt=None, is_promiscuous=None):
+ data = {}
+ if interface:
+ data['interface'] = interface
+
+ if ip:
+ data['ip'] = ip
+
+ if subnet:
+ data['subnet'] = subnet
+
+ if is_mgmt:
+ data['is_mgmt'] = is_mgmt
+
+ if is_promiscuous:
+ data['is_promiscuous'] = is_promiscuous
+
+ return self._get('/host-networks', data=data)
+
+ def get_host_network(self, host_id, host_network_id):
+ return self._get('/hosts/%s/networks/%s' %
+ (host_id, host_network_id))
+
+ def get_network_for_all_hosts(self, host_network_id):
+ return self._get('/host-networks/%s' % host_network_id)
+
+ def add_host_network(self, host_id, interface, ip, subnet_id,
+ is_mgmt=None, is_promiscuous=None,
+ raw_data=None):
+ data = {}
+ data['interface'] = interface
+ data['ip'] = ip
+ data['subnet_id'] = subnet_id
+ if raw_data:
+ data.update(raw_data)
+ else:
+ if is_mgmt:
+ data['is_mgmt'] = is_mgmt
+
+ if is_promiscuous:
+ data['is_promiscuous'] = is_promiscuous
+
+ return self._post('/hosts/%s/networks' % host_id, data=data)
+
+ def update_host_network(self, host_id, host_network_id,
+ ip=None, subnet_id=None, subnet=None,
+ is_mgmt=None, is_promiscuous=None,
+ raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if ip:
+ data['ip'] = ip
+
+ if subnet_id:
+ data['subnet_id'] = subnet_id
+
+ if subnet:
+ data['subnet'] = subnet
+
+ if is_mgmt:
+ data['is_mgmt'] = is_mgmt
+
+ if is_promiscuous:
+ data['is_promiscuous'] = is_promiscuous
+
+ return self._put('/hosts/%s/networks/%s' %
+ (host_id, host_network_id), data=data)
+
+ def update_hostnetwork(self, host_network_id, ip=None,
+ subnet_id=None, subnet=None,
+ is_mgmt=None, is_promiscuous=None,
+ raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if ip:
+ data['ip'] = ip
+
+ if subnet_id:
+ data['subnet_id'] = subnet_id
+
+ if subnet:
+ data['subnet'] = subnet
+
+ if is_mgmt:
+ data['is_mgmt'] = is_mgmt
+
+ if is_promiscuous:
+ data['is_promiscuous'] = is_promiscuous
+
+ return self._put('/host-networks/%s' % host_network_id,
+ data=data)
+
+ def delete_host_network(self, host_id, host_network_id):
+ return self._delete('/hosts/%s/networks/%s',
+ (host_id, host_network_id))
+
+ def delete_hostnetwork(self, host_network_id):
+ return self._delete('/host-networks/%s' % host_network_id)
+
+ def get_host_state(self, host_id):
+ return self._get('/hosts/%s/state' % host_id)
+
+ def update_host_state(self, host_id, state=None,
+ percentage=None, message=None,
+ raw_data=None):
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if state:
+ data['state'] = state
+
+ if percentage:
+ data['percentage'] = percentage
+
+ if message:
+ data['message'] = message
+
+ return self._put('/hosts/%s/state' % host_id, date=data)
+
+ def poweron_host(self, host_id):
+ data = {}
+ data['poweron'] = True
+
+ return self._post('/hosts/%s/action' % host_id, data=data)
+
+ def poweroff_host(self, host_id):
+ data = {}
+ data['poweroff'] = True
+
+ return self._post('/hosts/%s/action' % host_id, data=data)
+
+ def reset_host(self, host_id):
+ data = {}
+ data['reset'] = True
+
+ return self._post('/hosts/%s/action' % host_id, data=data)
+
+ def clusterhost_ready(self, clusterhost_name):
+ data = {}
+ data['ready'] = True
+
+ return self._post('/clusterhosts/%s/state_internal' %
+ clusterhost_name, data=data)
diff --git a/compass-tasks/apiclient/v1/__init__.py b/compass-tasks/apiclient/v1/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/compass-tasks/apiclient/v1/__init__.py
diff --git a/compass-tasks/apiclient/v1/example.py b/compass-tasks/apiclient/v1/example.py
new file mode 100755
index 0000000..6f7a7f7
--- /dev/null
+++ b/compass-tasks/apiclient/v1/example.py
@@ -0,0 +1,305 @@
+#!/usr/bin/python
+#
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Example code to deploy a cluster by compass client api."""
+import os
+import re
+import requests
+import sys
+import time
+
+from compass.apiclient.restful import Client
+
+
+COMPASS_SERVER_URL = 'http://127.0.0.1/api'
+SWITCH_IP = '10.145.81.220'
+SWITCH_SNMP_VERSION = 'v2c'
+SWITCH_SNMP_COMMUNITY = 'public'
+# MACHINES_TO_ADD = ['00:11:20:30:40:01']
+CLUSTER_NAME = 'cluster2'
+HOST_NAME_PREFIX = 'host'
+SERVER_USERNAME = 'root'
+SERVER_PASSWORD = 'root'
+SERVICE_USERNAME = 'service'
+SERVICE_PASSWORD = 'service'
+CONSOLE_USERNAME = 'console'
+CONSOLE_PASSWORD = 'console'
+HA_VIP = ''
+# NAMESERVERS = '192.168.10.6'
+SEARCH_PATH = 'ods.com'
+# GATEWAY = '192.168.10.6'
+# PROXY = 'http://192.168.10.6:3128'
+# NTP_SERVER = '192.168.10.6'
+MANAGEMENT_IP_START = '192.168.10.130'
+MANAGEMENT_IP_END = '192.168.10.254'
+MANAGEMENT_IP_GATEWAY = '192.168.10.1'
+MANAGEMENT_NETMASK = '255.255.255.0'
+MANAGEMENT_NIC = 'eth0'
+MANAGEMENT_PROMISC = 0
+TENANT_IP_START = '192.168.10.100'
+TENANT_IP_END = '192.168.10.255'
+TENANT_IP_GATEWAY = '192.168.10.1'
+TENANT_NETMASK = '255.255.255.0'
+TENANT_NIC = 'eth0'
+TENANT_PROMISC = 0
+PUBLIC_IP_START = '12.234.32.100'
+PUBLIC_IP_END = '12.234.32.255'
+PUBLIC_IP_GATEWAY = '12.234.32.1'
+PUBLIC_NETMASK = '255.255.255.0'
+PUBLIC_NIC = 'eth1'
+PUBLIC_PROMISC = 1
+STORAGE_IP_START = '172.16.100.100'
+STORAGE_IP_END = '172.16.100.255'
+STORAGE_NETMASK = '255.255.255.0'
+STORAGE_IP_GATEWAY = '172.16.100.1'
+STORAGE_NIC = 'eth0'
+STORAGE_PROMISC = 0
+HOME_PERCENTAGE = 5
+TMP_PERCENTAGE = 5
+VAR_PERCENTAGE = 10
+# ROLES_LIST = [['os-dashboard']]
+
+PRESET_VALUES = {
+ 'NAMESERVERS': '192.168.10.1',
+ 'NTP_SERVER': '192.168.10.1',
+ 'GATEWAY': '192.168.10.1',
+ 'PROXY': 'http://192.168.10.1:3128',
+ 'ROLES_LIST': 'os-dashboard',
+ 'MACHINES_TO_ADD': '00:11:20:30:40:01',
+ 'BUILD_TIMEOUT': 60
+}
+for v in PRESET_VALUES:
+ if v in os.environ.keys():
+ PRESET_VALUES[v] = os.environ.get(v)
+ print (v + PRESET_VALUES[v] + " is set by env variables")
+ else:
+ print (PRESET_VALUES[v])
+
+# get apiclient object.
+client = Client(COMPASS_SERVER_URL)
+
+
+# get all switches.
+status, resp = client.get_switches()
+print 'get all switches status: %s resp: %s' % (status, resp)
+
+# add a switch.
+status, resp = client.add_switch(
+ SWITCH_IP, version=SWITCH_SNMP_VERSION,
+ community=SWITCH_SNMP_COMMUNITY)
+
+print 'add a switch status: %s resp: %s' % (status, resp)
+
+if status < 400:
+ switch = resp['switch']
+else:
+ status, resp = client.get_switches()
+ print 'get all switches status: %s resp: %s' % (status, resp)
+ switch = None
+ for switch in resp['switches']:
+ if switch['ip'] == SWITCH_IP:
+ break
+
+switch_id = switch['id']
+switch_ip = switch['ip']
+
+
+# if the switch is not in under_monitoring, wait for the poll switch task
+# update the swich information and change the switch state.
+while switch['state'] != 'under_monitoring':
+ print 'waiting for the switch into under_monitoring'
+ status, resp = client.get_switch(switch_id)
+ print 'get switch %s status: %s, resp: %s' % (switch_id, status, resp)
+ switch = resp['switch']
+ time.sleep(10)
+
+
+# get machines connected to the switch.
+status, resp = client.get_machines(switch_id=switch_id)
+print 'get all machines under switch %s status: %s, resp: %s' % (
+ switch_id, status, resp)
+machines = {}
+MACHINES_TO_ADD = PRESET_VALUES['MACHINES_TO_ADD'].split()
+for machine in resp['machines']:
+ mac = machine['mac']
+ if mac in MACHINES_TO_ADD:
+ machines[machine['id']] = mac
+
+print 'machine to add: %s' % machines
+
+if set(machines.values()) != set(MACHINES_TO_ADD):
+ print 'only found macs %s while expected are %s' % (
+ machines.values(), MACHINES_TO_ADD)
+ sys.exit(1)
+
+
+# get adapters.
+status, resp = client.get_adapters()
+print 'get all adapters status: %s, resp: %s' % (status, resp)
+adapter_ids = []
+for adapter in resp['adapters']:
+ adapter_ids.append(adapter['id'])
+
+adapter_id = adapter_ids[0]
+print 'adpater for deploying a cluster: %s' % adapter_id
+
+
+# add a cluster.
+status, resp = client.add_cluster(
+ cluster_name=CLUSTER_NAME, adapter_id=adapter_id)
+print 'add cluster %s status: %s, resp: %s' % (CLUSTER_NAME, status, resp)
+cluster = resp['cluster']
+cluster_id = cluster['id']
+
+# add hosts to the cluster.
+status, resp = client.add_hosts(
+ cluster_id=cluster_id,
+ machine_ids=machines.keys())
+print 'add hosts to cluster %s status: %s, resp: %s' % (
+ cluster_id, status, resp)
+host_ids = []
+for host in resp['cluster_hosts']:
+ host_ids.append(host['id'])
+
+print 'added hosts: %s' % host_ids
+
+
+# set cluster security
+status, resp = client.set_security(
+ cluster_id, server_username=SERVER_USERNAME,
+ server_password=SERVER_PASSWORD,
+ service_username=SERVICE_USERNAME,
+ service_password=SERVICE_PASSWORD,
+ console_username=CONSOLE_USERNAME,
+ console_password=CONSOLE_PASSWORD)
+print 'set security config to cluster %s status: %s, resp: %s' % (
+ cluster_id, status, resp)
+
+
+# set cluster networking
+status, resp = client.set_networking(
+ cluster_id,
+ nameservers=PRESET_VALUES["NAMESERVERS"],
+ search_path=SEARCH_PATH,
+ gateway=PRESET_VALUES["GATEWAY"],
+ proxy=PRESET_VALUES["PROXY"],
+ ntp_server=PRESET_VALUES["NTP_SERVER"],
+ ha_vip=HA_VIP,
+ management_ip_start=MANAGEMENT_IP_START,
+ management_ip_end=MANAGEMENT_IP_END,
+ management_netmask=MANAGEMENT_NETMASK,
+ management_nic=MANAGEMENT_NIC,
+ management_gateway=MANAGEMENT_IP_GATEWAY,
+ management_promisc=MANAGEMENT_PROMISC,
+ tenant_ip_start=TENANT_IP_START,
+ tenant_ip_end=TENANT_IP_END,
+ tenant_netmask=TENANT_NETMASK,
+ tenant_nic=TENANT_NIC,
+ tenant_gateway=TENANT_IP_GATEWAY,
+ tenant_promisc=TENANT_PROMISC,
+ public_ip_start=PUBLIC_IP_START,
+ public_ip_end=PUBLIC_IP_END,
+ public_netmask=PUBLIC_NETMASK,
+ public_nic=PUBLIC_NIC,
+ public_gateway=PUBLIC_IP_GATEWAY,
+ public_promisc=PUBLIC_PROMISC,
+ storage_ip_start=STORAGE_IP_START,
+ storage_ip_end=STORAGE_IP_END,
+ storage_netmask=STORAGE_NETMASK,
+ storage_nic=STORAGE_NIC,
+ storage_gateway=STORAGE_IP_GATEWAY,
+ storage_promisc=STORAGE_PROMISC)
+print 'set networking config to cluster %s status: %s, resp: %s' % (
+ cluster_id, status, resp)
+
+
+# set partiton of each host in cluster
+status, resp = client.set_partition(
+ cluster_id,
+ home_percentage=HOME_PERCENTAGE,
+ tmp_percentage=TMP_PERCENTAGE,
+ var_percentage=VAR_PERCENTAGE)
+print 'set partition config to cluster %s status: %s, resp: %s' % (
+ cluster_id, status, resp)
+
+
+# set each host config in cluster.
+ROLES_LIST = [PRESET_VALUES['ROLES_LIST'].split()]
+for host_id in host_ids:
+ if ROLES_LIST:
+ roles = ROLES_LIST.pop(0)
+ else:
+ roles = []
+ status, resp = client.update_host_config(
+ host_id, hostname='%s%s' % (HOST_NAME_PREFIX, host_id),
+ roles=roles)
+ print 'set roles to host %s status: %s, resp: %s' % (
+ host_id, status, resp)
+
+
+# deploy cluster.
+status, resp = client.deploy_hosts(cluster_id)
+print 'deploy cluster %s status: %s, resp: %s' % (cluster_id, status, resp)
+
+
+# get intalling progress.
+BUILD_TIMEOUT = float(PRESET_VALUES['BUILD_TIMEOUT'])
+timeout = time.time() + BUILD_TIMEOUT * 60
+while True:
+ status, resp = client.get_cluster_installing_progress(cluster_id)
+ print 'get cluster %s installing progress status: %s, resp: %s' % (
+ cluster_id, status, resp)
+ progress = resp['progress']
+ if (
+ progress['state'] not in ['UNINITIALIZED', 'INSTALLING'] or
+ progress['percentage'] >= 1.0
+ ):
+ break
+ if (
+ time.time() > timeout
+ ):
+ raise Exception("Timeout! The system is not ready in time.")
+
+ for host_id in host_ids:
+ status, resp = client.get_host_installing_progress(host_id)
+ print 'get host %s installing progress status: %s, resp: %s' % (
+ host_id, status, resp)
+
+ time.sleep(60)
+
+
+status, resp = client.get_dashboard_links(cluster_id)
+print 'get cluster %s dashboardlinks status: %s, resp: %s' % (
+ cluster_id, status, resp)
+dashboardlinks = resp['dashboardlinks']
+if not dashboardlinks.keys():
+ raise Exception("Dashboard link is not found!")
+for x in dashboardlinks.keys():
+ if x in ("os-dashboard", "os-controller"):
+ dashboardurl = dashboardlinks.get(x)
+ if dashboardurl is None:
+ raise Exception("No dashboard link is found")
+ r = requests.get(dashboardurl, verify=False)
+ r.raise_for_status()
+ match = re.search(
+ r'(?m)(http://\d+\.\d+\.\d+\.\d+:5000/v2\.0)', r.text)
+ if match:
+ print 'dashboard login page can be downloaded'
+ break
+ print (
+ 'dashboard login page failed to be downloaded\n'
+ 'the context is:\n%s\n') % r.text
+ raise Exception("os-dashboard is not properly installed!")
diff --git a/compass-tasks/apiclient/v1/restful.py b/compass-tasks/apiclient/v1/restful.py
new file mode 100644
index 0000000..3fb235c
--- /dev/null
+++ b/compass-tasks/apiclient/v1/restful.py
@@ -0,0 +1,655 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Compass api client library.
+
+ .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+import json
+import logging
+import requests
+
+
+class Client(object):
+ """wrapper for compass restful api.
+
+ .. note::
+ Every api client method returns (status as int, resp as dict).
+ If the api succeeds, the status is 2xx, the resp includes
+ {'status': 'OK'} and other keys depend on method.
+ If the api fails, the status is 4xx, the resp includes {
+ 'status': '...', 'message': '...'}
+ """
+
+ def __init__(self, url, headers=None, proxies=None, stream=None):
+ """Restful api client initialization.
+
+ :param url: url to the compass web service.
+ :type url: str.
+ :param headers: http header sent in each restful request.
+ :type headers: dict of header name (str) to heade value (str).
+ :param proxies: the proxy address for each protocol.
+ :type proxies: dict of protocol (str) to proxy url (str).
+ :param stream: wether the restful response should be streamed.
+ :type stream: bool.
+ """
+ self.url_ = url
+ self.session_ = requests.Session()
+ if headers:
+ self.session_.headers = headers
+
+ if proxies is not None:
+ self.session_.proxies = proxies
+
+ if stream is not None:
+ self.session_.stream = stream
+
+ def __del__(self):
+ self.session_.close()
+
+ @classmethod
+ def _get_response(cls, resp):
+ """decapsulate the resp to status code and python formatted data."""
+ resp_obj = {}
+ try:
+ resp_obj = resp.json()
+ except Exception as error:
+ logging.error('failed to load object from %s: %s',
+ resp.url, resp.content)
+ logging.exception(error)
+ resp_obj['status'] = 'Json Parsing Failure'
+ resp_obj['message'] = resp.content
+
+ return resp.status_code, resp_obj
+
+ def _get(self, relative_url, params=None):
+ """encapsulate get method."""
+ url = '%s%s' % (self.url_, relative_url)
+ if params:
+ resp = self.session_.get(url, params=params)
+ else:
+ resp = self.session_.get(url)
+
+ return self._get_response(resp)
+
+ def _post(self, relative_url, data=None):
+ """encapsulate post method."""
+ url = '%s%s' % (self.url_, relative_url)
+ if data:
+ resp = self.session_.post(url, json.dumps(data))
+ else:
+ resp = self.session_.post(url)
+
+ return self._get_response(resp)
+
+ def _put(self, relative_url, data=None):
+ """encapsulate put method."""
+ url = '%s%s' % (self.url_, relative_url)
+ if data:
+ resp = self.session_.put(url, json.dumps(data))
+ else:
+ resp = self.session_.put(url)
+
+ return self._get_response(resp)
+
+ def _delete(self, relative_url):
+ """encapsulate delete method."""
+ url = '%s%s' % (self.url_, relative_url)
+ return self._get_response(self.session_.delete(url))
+
+ def get_switches(self, switch_ips=None, switch_networks=None, limit=None):
+ """List details for switches.
+
+ .. note::
+ The switches can be filtered by switch_ips, siwtch_networks and
+ limit. These params can be None or missing. If the param is None
+ or missing, that filter will be ignored.
+
+ :param switch_ips: Filter switch(es) with IP(s).
+ :type switch_ips: list of str. Each is as 'xxx.xxx.xxx.xxx'.
+ :param switch_networks: Filter switche(es) with network(s).
+ :type switch_networks: list of str. Each is as 'xxx.xxx.xxx.xxx/xx'.
+ :param limit: int, The maximum number of switches to return.
+ :type limit: int. 0 means unlimited.
+ """
+ params = {}
+ if switch_ips:
+ params['switchIp'] = switch_ips
+
+ if switch_networks:
+ params['switchIpNetwork'] = switch_networks
+
+ if limit:
+ params['limit'] = limit
+ return self._get('/switches', params=params)
+
+ def get_switch(self, switch_id):
+ """Lists details for a specified switch.
+
+ :param switch_id: switch id.
+ :type switch_id: int.
+ """
+ return self._get('/switches/%s' % switch_id)
+
+ def add_switch(self, switch_ip, version=None, community=None,
+ username=None, password=None, raw_data=None):
+ """Create a switch with specified details.
+
+ .. note::
+ It will trigger switch polling if successful. During
+ the polling, MAC address of the devices connected to the
+ switch will be learned by SNMP or SSH.
+
+ :param switch_ip: the switch IP address.
+ :type switch_ip: str, as xxx.xxx.xxx.xxx.
+ :param version: SNMP version when using SNMP to poll switch.
+ :type version: str, one in ['v1', 'v2c', 'v3']
+ :param community: SNMP community when using SNMP to poll switch.
+ :type community: str, usually 'public'.
+ :param username: SSH username when using SSH to poll switch.
+ :type username: str.
+ :param password: SSH password when using SSH to poll switch.
+ :type password: str.
+ """
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ data['switch'] = {}
+ data['switch']['ip'] = switch_ip
+ data['switch']['credential'] = {}
+ if version:
+ data['switch']['credential']['version'] = version
+
+ if community:
+ data['switch']['credential']['community'] = community
+
+ if username:
+ data['switch']['credential']['username'] = username
+
+ if password:
+ data['switch']['credential']['password'] = password
+
+ return self._post('/switches', data=data)
+
+ def update_switch(self, switch_id, ip_addr=None,
+ version=None, community=None,
+ username=None, password=None,
+ raw_data=None):
+ """Updates a switch with specified details.
+
+ .. note::
+ It will trigger switch polling if successful. During
+ the polling, MAC address of the devices connected to the
+ switch will be learned by SNMP or SSH.
+
+ :param switch_id: switch id
+ :type switch_id: int.
+ :param ip_addr: the switch ip address.
+ :type ip_addr: str, as 'xxx.xxx.xxx.xxx' format.
+ :param version: SNMP version when using SNMP to poll switch.
+ :type version: str, one in ['v1', 'v2c', 'v3'].
+ :param community: SNMP community when using SNMP to poll switch.
+ :type community: str, usually be 'public'.
+ :param username: username when using SSH to poll switch.
+ :type username: str.
+ :param password: password when using SSH to poll switch.
+ """
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ data['switch'] = {}
+ if ip_addr:
+ data['switch']['ip'] = ip_addr
+
+ data['switch']['credential'] = {}
+ if version:
+ data['switch']['credential']['version'] = version
+
+ if community:
+ data['switch']['credential']['community'] = community
+
+ if username:
+ data['switch']['credential']['username'] = username
+
+ if password:
+ data['switch']['credential']['password'] = password
+
+ return self._put('/switches/%s' % switch_id, data=data)
+
+ def delete_switch(self, switch_id):
+ """Not implemented in api."""
+ return self._delete('/switches/%s' % switch_id)
+
+ def get_machines(self, switch_id=None, vlan_id=None,
+ port=None, limit=None):
+ """Get the details of machines.
+
+ .. note::
+ The machines can be filtered by switch_id, vlan_id, port
+ and limit. These params can be None or missing. If the param
+ is None or missing, the filter will be ignored.
+
+ :param switch_id: Return machine(s) connected to the switch.
+ :type switch_id: int.
+ :param vlan_id: Return machine(s) belonging to the vlan.
+ :type vlan_id: int.
+ :param port: Return machine(s) connect to the port.
+ :type port: int.
+ :param limit: the maximum number of machines will be returned.
+ :type limit: int. 0 means no limit.
+ """
+ params = {}
+ if switch_id:
+ params['switchId'] = switch_id
+
+ if vlan_id:
+ params['vlanId'] = vlan_id
+
+ if port:
+ params['port'] = port
+
+ if limit:
+ params['limit'] = limit
+
+ return self._get('/machines', params=params)
+
+ def get_machine(self, machine_id):
+ """Lists the details for a specified machine.
+
+ :param machine_id: Return machine with the id.
+ :type machine_id: int.
+ """
+ return self._get('/machines/%s' % machine_id)
+
+ def get_clusters(self):
+ """Lists the details for all clusters."""
+ return self._get('/clusters')
+
+ def get_cluster(self, cluster_id):
+ """Lists the details of the specified cluster.
+
+ :param cluster_id: cluster id.
+ :type cluster_id: int.
+ """
+ return self._get('/clusters/%d' % cluster_id)
+
+ def add_cluster(self, cluster_name, adapter_id, raw_data=None):
+ """Creates a cluster by specified name and given adapter id.
+
+ :param cluster_name: cluster name.
+ :type cluster_name: str.
+ :param adapter_id: adapter id.
+ :type adapter_id: int.
+ """
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ data['cluster'] = {}
+ data['cluster']['name'] = cluster_name
+ data['cluster']['adapter_id'] = adapter_id
+ return self._post('/clusters', data=data)
+
+ def add_hosts(self, cluster_id, machine_ids, raw_data=None):
+ """add the specified machine(s) as the host(s) to the cluster.
+
+ :param cluster_id: cluster id.
+ :type cluster_id: int.
+ :param machine_ids: machine ids to add to cluster.
+ :type machine_ids: list of int, each is the id of one machine.
+ """
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ data['addHosts'] = machine_ids
+ return self._post('/clusters/%d/action' % cluster_id, data=data)
+
+ def remove_hosts(self, cluster_id, host_ids, raw_data=None):
+ """remove the specified host(s) from the cluster.
+
+ :param cluster_id: cluster id.
+ :type cluster_id: int.
+ :param host_ids: host ids to remove from cluster.
+ :type host_ids: list of int, each is the id of one host.
+ """
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ data['removeHosts'] = host_ids
+ return self._post('/clusters/%s/action' % cluster_id, data=data)
+
+ def replace_hosts(self, cluster_id, machine_ids, raw_data=None):
+ """replace the cluster hosts with the specified machine(s).
+
+ :param cluster_id: int, The unique identifier of the cluster.
+ :type cluster_id: int.
+ :param machine_ids: the machine ids to replace the hosts in cluster.
+ :type machine_ids: list of int, each is the id of one machine.
+ """
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ data['replaceAllHosts'] = machine_ids
+ return self._post('/clusters/%s/action' % cluster_id, data=data)
+
+ def deploy_hosts(self, cluster_id, raw_data=None):
+ """Deploy the cluster.
+
+ :param cluster_id: The unique identifier of the cluster
+ :type cluster_id: int.
+ """
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ data['deploy'] = []
+ return self._post('/clusters/%d/action' % cluster_id, data=data)
+
+ @classmethod
+ def parse_security(cls, kwargs):
+ """parse the arguments to security data."""
+ data = {}
+ for key, value in kwargs.items():
+ if '_' not in key:
+ continue
+ key_name, key_value = key.split('_', 1)
+ data.setdefault(
+ '%s_credentials' % key_name, {})[key_value] = value
+
+ return data
+
+ def set_security(self, cluster_id, **kwargs):
+ """Update the cluster security configuration.
+
+ :param cluster_id: cluster id.
+ :type cluster_id: int.
+ :param <security_name>_username: username of the security name.
+ :type <security_name>_username: str.
+ :param <security_name>_password: passowrd of the security name.
+ :type <security_name>_password: str.
+
+ .. note::
+ security_name should be one of ['server', 'service', 'console'].
+ """
+ data = {}
+ data['security'] = self.parse_security(kwargs)
+ return self._put('/clusters/%d/security' % cluster_id, data=data)
+
+ @classmethod
+ def parse_networking(cls, kwargs):
+ """parse arguments to network data."""
+ data = {}
+ global_keys = [
+ 'nameservers', 'search_path', 'gateway',
+ 'proxy', 'ntp_server', 'ha_vip']
+ for key, value in kwargs.items():
+ if key in global_keys:
+ data.setdefault('global', {})[key] = value
+ else:
+ if '_' not in key:
+ continue
+
+ key_name, key_value = key.split('_', 1)
+ data.setdefault(
+ 'interfaces', {}
+ ).setdefault(
+ key_name, {}
+ )[key_value] = value
+
+ return data
+
+ def set_networking(self, cluster_id, **kwargs):
+ """Update the cluster network configuration.
+
+ :param cluster_id: cluster id.
+ :type cluster_id: int.
+ :param nameservers: comma seperated nameserver ip address.
+ :type nameservers: str.
+ :param search_path: comma seperated dns name search path.
+ :type search_path: str.
+ :param gateway: gateway ip address for routing to outside.
+ :type gateway: str.
+ :param proxy: proxy url for downloading packages.
+ :type proxy: str.
+ :param ntp_server: ntp server ip address to sync timestamp.
+ :type ntp_server: str.
+ :param ha_vip: ha vip address to run ha proxy.
+ :type ha_vip: str.
+ :param <interface>_ip_start: start ip address to host's interface.
+ :type <interface>_ip_start: str.
+ :param <interface>_ip_end: end ip address to host's interface.
+ :type <interface>_ip_end: str.
+ :param <interface>_netmask: netmask to host's interface.
+ :type <interface>_netmask: str.
+ :param <interface>_nic: host physical interface name.
+ :type <interface>_nic: str.
+ :param <interface>_promisc: if the interface in promiscous mode.
+ :type <interface>_promisc: int, 0 or 1.
+
+ .. note::
+ interface should be one of ['management', 'tenant',
+ 'public', 'storage'].
+ """
+ data = {}
+ data['networking'] = self.parse_networking(kwargs)
+ return self._put('/clusters/%d/networking' % cluster_id, data=data)
+
+ @classmethod
+ def parse_partition(cls, kwargs):
+ """parse arguments to partition data."""
+ data = {}
+ for key, value in kwargs.items():
+ if key.endswith('_percentage'):
+ key_name = key[:-len('_percentage')]
+ data[key_name] = '%s%%' % value
+ elif key.endswitch('_mbytes'):
+ key_name = key[:-len('_mbytes')]
+ data[key_name] = str(value)
+
+ return ';'.join([
+ '/%s %s' % (key, value) for key, value in data.items()
+ ])
+
+ def set_partition(self, cluster_id, **kwargs):
+ """Update the cluster partition configuration.
+
+ :param cluster_id: cluster id.
+ :type cluster_id: int.
+ :param <partition>_percentage: the partiton percentage.
+ :type <partition>_percentage: float between 0 to 100.
+ :param <partition>_mbytes: the partition mbytes.
+ :type <partition>_mbytes: int.
+
+ .. note::
+ partition should be one of ['home', 'var', 'tmp'].
+ """
+ data = {}
+ data['partition'] = self.parse_partition(kwargs)
+ return self._put('/clusters/%s/partition' % cluster_id, data=data)
+
+ def get_hosts(self, hostname=None, clustername=None):
+ """Lists the details of hosts.
+
+ .. note::
+ The hosts can be filtered by hostname, clustername.
+ These params can be None or missing. If the param
+ is None or missing, the filter will be ignored.
+
+ :param hostname: The name of a host.
+ :type hostname: str.
+ :param clustername: The name of a cluster.
+ :type clustername: str.
+ """
+ params = {}
+ if hostname:
+ params['hostname'] = hostname
+
+ if clustername:
+ params['clustername'] = clustername
+
+ return self._get('/clusterhosts', params=params)
+
+ def get_host(self, host_id):
+ """Lists the details for the specified host.
+
+ :param host_id: host id.
+ :type host_id: int.
+ """
+ return self._get('/clusterhosts/%s' % host_id)
+
+ def get_host_config(self, host_id):
+ """Lists the details of the config for the specified host.
+
+ :param host_id: host id.
+ :type host_id: int.
+ """
+ return self._get('/clusterhosts/%s/config' % host_id)
+
+ def update_host_config(self, host_id, hostname=None,
+ roles=None, raw_data=None, **kwargs):
+ """Updates config for the host.
+
+ :param host_id: host id.
+ :type host_id: int.
+ :param hostname: host name.
+ :type hostname: str.
+ :param security_<security>_username: username of the security name.
+ :type security_<security>_username: str.
+ :param security_<security>_password: passowrd of the security name.
+ :type security_<security>_password: str.
+ :param networking_nameservers: comma seperated nameserver ip address.
+ :type networking_nameservers: str.
+ :param networking_search_path: comma seperated dns name search path.
+ :type networking_search_path: str.
+ :param networking_gateway: gateway ip address for routing to outside.
+ :type networking_gateway: str.
+ :param networking_proxy: proxy url for downloading packages.
+ :type networking_proxy: str.
+ :param networking_ntp_server: ntp server ip address to sync timestamp.
+ :type networking_ntp_server: str.
+ :param networking_<interface>_ip: ip address to host interface.
+ :type networking_<interface>_ip: str.
+ :param networking_<interface>_netmask: netmask to host's interface.
+ :type networking_<interface>_netmask: str.
+ :param networking_<interface>_nic: host physical interface name.
+ :type networking_<interface>_nic: str.
+ :param networking_<interface>_promisc: if the interface is promiscous.
+ :type networking_<interface>_promisc: int, 0 or 1.
+ :param partition_<partition>_percentage: the partiton percentage.
+ :type partition_<partition>_percentage: float between 0 to 100.
+ :param partition_<partition>_mbytes: the partition mbytes.
+ :type partition_<partition>_mbytes: int.
+ :param roles: host assigned roles in the cluster.
+ :type roles: list of str.
+ """
+ data = {}
+ if raw_data:
+ data = raw_data
+ else:
+ if hostname:
+ data['hostname'] = hostname
+
+ sub_kwargs = {}
+ for key, value in kwargs.items():
+ key_name, key_value = key.split('_', 1)
+ sub_kwargs.setdefault(key_name, {})[key_value] = value
+
+ if 'security' in sub_kwargs:
+ data['security'] = self.parse_security(sub_kwargs['security'])
+
+ if 'networking' in sub_kwargs:
+ data['networking'] = self.parse_networking(
+ sub_kwargs['networking'])
+ if 'partition' in sub_kwargs:
+ data['partition'] = self.parse_partition(
+ sub_kwargs['partition'])
+
+ if roles:
+ data['roles'] = roles
+
+ return self._put('/clusterhosts/%s/config' % host_id, data)
+
+ def delete_from_host_config(self, host_id, delete_key):
+ """Deletes one key in config for the host.
+
+ :param host_id: host id.
+ :type host_id: int.
+ :param delete_key: the key in host config to be deleted.
+ :type delete_key: str.
+ """
+ return self._delete('/clusterhosts/%s/config/%s' % (
+ host_id, delete_key))
+
+ def get_adapters(self, name=None):
+ """Lists details of adapters.
+
+ .. note::
+ the adapter can be filtered by name of name is given and not None.
+
+ :param name: adapter name.
+ :type name: str.
+ """
+ params = {}
+ if name:
+ params['name'] = name
+
+ return self._get('/adapters', params=params)
+
+ def get_adapter(self, adapter_id):
+ """Lists details for the specified adapter.
+
+ :param adapter_id: adapter id.
+ :type adapter_id: int.
+ """
+ return self._get('/adapters/%s' % adapter_id)
+
+ def get_adapter_roles(self, adapter_id):
+ """Lists roles to assign to hosts for the specified adapter.
+
+ :param adapter_id: adapter id.
+ :type adapter_id: int.
+ """
+ return self._get('/adapters/%s/roles' % adapter_id)
+
+ def get_host_installing_progress(self, host_id):
+ """Lists progress details for the specified host.
+
+ :param host_id: host id.
+ :type host_id: int.
+ """
+ return self._get('/clusterhosts/%s/progress' % host_id)
+
+ def get_cluster_installing_progress(self, cluster_id):
+ """Lists progress details for the specified cluster.
+
+ :param cluster_id: cluster id.
+ :param cluster_id: int.
+ """
+
+ return self._get('/clusters/%s/progress' % cluster_id)
+
+ def get_dashboard_links(self, cluster_id):
+ """Lists links for dashboards of deployed cluster.
+
+ :param cluster_id: cluster id.
+ :type cluster_id: int.
+ """
+ params = {}
+ params['cluster_id'] = cluster_id
+ return self._get('/dashboardlinks', params)
diff --git a/compass-tasks/build.sh b/compass-tasks/build.sh
new file mode 100755
index 0000000..e0dceea
--- /dev/null
+++ b/compass-tasks/build.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016-2017 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -x
+COMPASS_DIR=${BASH_SOURCE[0]%/*}
+
+rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+sed -i 's/^mirrorlist=https/mirrorlist=http/g' /etc/yum.repos.d/epel.repo
+yum update -y
+
+yum --nogpgcheck install -y python python-devel git amqp python-pip libffi-devel openssl-devel gcc python-setuptools MySQL-python supervisor redis sshpass python-keyczar vim ansible-2.2.1.0 libyaml-devel make
+
+mkdir -p $COMPASS_DIR/compass
+touch $COMPASS_DIR/compass/__init__.py
+mv $COMPASS_DIR/actions $COMPASS_DIR/compass/
+mv $COMPASS_DIR/apiclient $COMPASS_DIR/compass/
+mv $COMPASS_DIR/tasks $COMPASS_DIR/compass/
+mv $COMPASS_DIR/utils $COMPASS_DIR/compass/
+mv $COMPASS_DIR/deployment $COMPASS_DIR/compass/
+mv $COMPASS_DIR/db $COMPASS_DIR/compass/
+mv $COMPASS_DIR/hdsdiscovery $COMPASS_DIR/compass/
+mv $COMPASS_DIR/log_analyzor $COMPASS_DIR/compass/
+
+easy_install --upgrade pip
+pip install --upgrade pip
+pip install --upgrade setuptools
+pip install --upgrade Flask
+
+mkdir -p /etc/compass/
+mkdir -p /etc/compass/machine_list
+mkdir -p /etc/compass/switch_list
+mkdir -p /var/log/compass
+mkdir -p /opt/ansible_callbacks
+mkdir -p /root/.ssh;
+echo "UserKnownHostsFile /dev/null" >> /root/.ssh/config;
+echo "StrictHostKeyChecking no" >> /root/.ssh/config
+
+cd $COMPASS_DIR
+python setup.py install
+cp supervisord.conf /etc/supervisord.conf
+cp start.sh /usr/local/bin/start.sh
+
+yum clean all
+
+set -x
diff --git a/compass-tasks/db/__init__.py b/compass-tasks/db/__init__.py
new file mode 100644
index 0000000..4ee55a4
--- /dev/null
+++ b/compass-tasks/db/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/compass-tasks/db/api/__init__.py b/compass-tasks/db/api/__init__.py
new file mode 100644
index 0000000..5e42ae9
--- /dev/null
+++ b/compass-tasks/db/api/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/compass-tasks/db/api/adapter.py b/compass-tasks/db/api/adapter.py
new file mode 100644
index 0000000..c3ad48d
--- /dev/null
+++ b/compass-tasks/db/api/adapter.py
@@ -0,0 +1,313 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Adapter related database operations."""
+import logging
+import re
+
+from compass.db.api import database
+from compass.db.api import utils
+from compass.db import exception
+from compass.db import models
+
+from compass.utils import setting_wrapper as setting
+from compass.utils import util
+
+
+OSES = None
+OS_INSTALLERS = None
+PACKAGE_INSTALLERS = None
+ADAPTERS = None
+ADAPTERS_FLAVORS = None
+ADAPTERS_ROLES = None
+
+
+def _get_oses_from_configuration():
+ """Get all os configs from os configuration dir.
+
+ Example: {
+ <os_name>: {
+ 'name': <os_name>,
+ 'id': <os_name>,
+ 'os_id': <os_name>,
+ 'deployable': True
+ }
+ }
+ """
+ configs = util.load_configs(setting.OS_DIR)
+ systems = {}
+ for config in configs:
+ logging.info('get config %s', config)
+ system_name = config['NAME']
+ parent_name = config.get('PARENT', None)
+ system = {
+ 'name': system_name,
+ 'id': system_name,
+ 'os_id': system_name,
+ 'parent': parent_name,
+ 'parent_id': parent_name,
+ 'deployable': config.get('DEPLOYABLE', False)
+ }
+ systems[system_name] = system
+ parents = {}
+ for name, system in systems.items():
+ parent = system.get('parent', None)
+ parents[name] = parent
+ for name, system in systems.items():
+ util.recursive_merge_dict(name, systems, parents)
+ return systems
+
+
+def _get_installers_from_configuration(configs):
+ """Get installers from configurations.
+
+ Example: {
+ <installer_isntance>: {
+ 'alias': <instance_name>,
+ 'id': <instance_name>,
+ 'name': <name>,
+ 'settings': <dict pass to installer plugin>
+ }
+ }
+ """
+ installers = {}
+ for config in configs:
+ name = config['NAME']
+ instance_name = config.get('INSTANCE_NAME', name)
+ installers[instance_name] = {
+ 'alias': instance_name,
+ 'id': instance_name,
+ 'name': name,
+ 'settings': config.get('SETTINGS', {})
+ }
+ return installers
+
+
+def _get_os_installers_from_configuration():
+ """Get os installers from os installer config dir."""
+ configs = util.load_configs(setting.OS_INSTALLER_DIR)
+ return _get_installers_from_configuration(configs)
+
+
+def _get_package_installers_from_configuration():
+ """Get package installers from package installer config dir."""
+ configs = util.load_configs(setting.PACKAGE_INSTALLER_DIR)
+ return _get_installers_from_configuration(configs)
+
+
+def _get_adapters_from_configuration():
+ """Get adapters from adapter config dir."""
+ configs = util.load_configs(setting.ADAPTER_DIR)
+ adapters = {}
+ for config in configs:
+ logging.info('add config %s to adapter', config)
+ if 'OS_INSTALLER' in config:
+ os_installer = OS_INSTALLERS[config['OS_INSTALLER']]
+ else:
+ os_installer = None
+
+ if 'PACKAGE_INSTALLER' in config:
+ package_installer = PACKAGE_INSTALLERS[
+ config['PACKAGE_INSTALLER']
+ ]
+ else:
+ package_installer = None
+
+ adapter_name = config['NAME']
+ parent_name = config.get('PARENT', None)
+ adapter = {
+ 'name': adapter_name,
+ 'id': adapter_name,
+ 'parent': parent_name,
+ 'parent_id': parent_name,
+ 'display_name': config.get('DISPLAY_NAME', adapter_name),
+ 'os_installer': os_installer,
+ 'package_installer': package_installer,
+ 'deployable': config.get('DEPLOYABLE', False),
+ 'health_check_cmd': config.get('HEALTH_CHECK_COMMAND', None),
+ 'supported_oses': [],
+ 'roles': [],
+ 'flavors': []
+ }
+ supported_os_patterns = [
+ re.compile(supported_os_pattern)
+ for supported_os_pattern in config.get('SUPPORTED_OS_PATTERNS', [])
+ ]
+ for os_name, os in OSES.items():
+ if not os.get('deployable', False):
+ continue
+ for supported_os_pattern in supported_os_patterns:
+ if supported_os_pattern.match(os_name):
+ adapter['supported_oses'].append(os)
+ break
+ adapters[adapter_name] = adapter
+
+ parents = {}
+ for name, adapter in adapters.items():
+ parent = adapter.get('parent', None)
+ parents[name] = parent
+ for name, adapter in adapters.items():
+ util.recursive_merge_dict(name, adapters, parents)
+ return adapters
+
+
+def _add_roles_from_configuration():
+ """Get roles from roles config dir and update to adapters."""
+ configs = util.load_configs(setting.ADAPTER_ROLE_DIR)
+ for config in configs:
+ logging.info(
+ 'add config %s to role', config
+ )
+ adapter_name = config['ADAPTER_NAME']
+ adapter = ADAPTERS[adapter_name]
+ adapter_roles = ADAPTERS_ROLES.setdefault(adapter_name, {})
+ for role_dict in config['ROLES']:
+ role_name = role_dict['role']
+ display_name = role_dict.get('display_name', role_name)
+ adapter_roles[role_name] = {
+ 'name': role_name,
+ 'id': '%s:%s' % (adapter_name, role_name),
+ 'adapter_id': adapter_name,
+ 'adapter_name': adapter_name,
+ 'display_name': display_name,
+ 'description': role_dict.get('description', display_name),
+ 'optional': role_dict.get('optional', False)
+ }
+ parents = {}
+ for name, adapter in ADAPTERS.items():
+ parent = adapter.get('parent', None)
+ parents[name] = parent
+ for adapter_name, adapter_roles in ADAPTERS_ROLES.items():
+ util.recursive_merge_dict(adapter_name, ADAPTERS_ROLES, parents)
+ for adapter_name, adapter_roles in ADAPTERS_ROLES.items():
+ adapter = ADAPTERS[adapter_name]
+ adapter['roles'] = adapter_roles.values()
+
+
+def _add_flavors_from_configuration():
+ """Get flavors from flavor config dir and update to adapters."""
+ configs = util.load_configs(setting.ADAPTER_FLAVOR_DIR)
+ for config in configs:
+ logging.info('add config %s to flavor', config)
+ adapter_name = config['ADAPTER_NAME']
+ adapter = ADAPTERS[adapter_name]
+ adapter_flavors = ADAPTERS_FLAVORS.setdefault(adapter_name, {})
+ adapter_roles = ADAPTERS_ROLES[adapter_name]
+ for flavor_dict in config['FLAVORS']:
+ flavor_name = flavor_dict['flavor']
+ flavor_id = '%s:%s' % (adapter_name, flavor_name)
+ flavor = {
+ 'name': flavor_name,
+ 'id': flavor_id,
+ 'adapter_id': adapter_name,
+ 'adapter_name': adapter_name,
+ 'display_name': flavor_dict.get('display_name', flavor_name),
+ 'template': flavor_dict.get('template', None)
+ }
+ flavor_roles = flavor_dict.get('roles', [])
+ roles_in_flavor = []
+ for flavor_role in flavor_roles:
+ if isinstance(flavor_role, basestring):
+ role_name = flavor_role
+ role_in_flavor = {
+ 'name': role_name,
+ 'flavor_id': flavor_id
+ }
+ else:
+ role_in_flavor = flavor_role
+ role_in_flavor['flavor_id'] = flavor_id
+ if 'role' in role_in_flavor:
+ role_in_flavor['name'] = role_in_flavor['role']
+ del role_in_flavor['role']
+ role_name = role_in_flavor['name']
+ role = adapter_roles[role_name]
+ util.merge_dict(role_in_flavor, role, override=False)
+ roles_in_flavor.append(role_in_flavor)
+ flavor['roles'] = roles_in_flavor
+ adapter_flavors[flavor_name] = flavor
+ parents = {}
+ for name, adapter in ADAPTERS.items():
+ parent = adapter.get('parent', None)
+ parents[name] = parent
+ for adapter_name, adapter_roles in ADAPTERS_FLAVORS.items():
+ util.recursive_merge_dict(adapter_name, ADAPTERS_FLAVORS, parents)
+ for adapter_name, adapter_flavors in ADAPTERS_FLAVORS.items():
+ adapter = ADAPTERS[adapter_name]
+ adapter['flavors'] = adapter_flavors.values()
+
+
+def load_adapters_internal(force_reload=False):
+ """Load adapter related configurations into memory.
+
+ If force_reload, reload all configurations even it is loaded already.
+ """
+ global OSES
+ if force_reload or OSES is None:
+ OSES = _get_oses_from_configuration()
+ global OS_INSTALLERS
+ if force_reload or OS_INSTALLERS is None:
+ OS_INSTALLERS = _get_os_installers_from_configuration()
+ global PACKAGE_INSTALLERS
+ if force_reload or PACKAGE_INSTALLERS is None:
+ PACKAGE_INSTALLERS = _get_package_installers_from_configuration()
+ global ADAPTERS
+ if force_reload or ADAPTERS is None:
+ ADAPTERS = _get_adapters_from_configuration()
+ global ADAPTERS_ROLES
+ if force_reload or ADAPTERS_ROLES is None:
+ ADAPTERS_ROLES = {}
+ _add_roles_from_configuration()
+ global ADAPTERS_FLAVORS
+ if force_reload or ADAPTERS_FLAVORS is None:
+ ADAPTERS_FLAVORS = {}
+ _add_flavors_from_configuration()
+
+
+def get_adapters_internal(force_reload=False):
+ """Get all deployable adapters."""
+ load_adapters_internal(force_reload=force_reload)
+ adapter_mapping = {}
+ for adapter_name, adapter in ADAPTERS.items():
+ if adapter.get('deployable'):
+ # TODO(xicheng): adapter should be filtered before
+ # return to caller.
+ adapter_mapping[adapter_name] = adapter
+ else:
+ logging.info(
+ 'ignore adapter %s since it is not deployable',
+ adapter_name
+ )
+ return adapter_mapping
+
+
+def get_flavors_internal(force_reload=False):
+ """Get all deployable flavors."""
+ load_adapters_internal(force_reload=force_reload)
+ adapter_flavor_mapping = {}
+ for adapter_name, adapter_flavors in ADAPTERS_FLAVORS.items():
+ adapter = ADAPTERS.get(adapter_name, {})
+ for flavor_name, flavor in adapter_flavors.items():
+ if adapter.get('deployable'):
+ # TODO(xicheng): flavor dict should be filtered before
+ # return to caller.
+ adapter_flavor_mapping.setdefault(
+ adapter_name, {}
+ )[flavor_name] = flavor
+ else:
+ logging.info(
+ 'ignore adapter %s since it is not deployable',
+ adapter_name
+ )
+
+ return adapter_flavor_mapping
diff --git a/compass-tasks/db/api/adapter_holder.py b/compass-tasks/db/api/adapter_holder.py
new file mode 100644
index 0000000..91c65c4
--- /dev/null
+++ b/compass-tasks/db/api/adapter_holder.py
@@ -0,0 +1,155 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Adapter related object holder."""
+import logging
+
+from compass.db.api import adapter as adapter_api
+from compass.db.api import database
+from compass.db.api import permission
+from compass.db.api import user as user_api
+from compass.db.api import utils
+from compass.db import exception
+
+
+SUPPORTED_FIELDS = [
+ 'name',
+]
+RESP_FIELDS = [
+ 'id', 'name', 'roles', 'flavors',
+ 'os_installer', 'package_installer',
+ 'supported_oses', 'display_name', 'health_check_cmd'
+]
+RESP_OS_FIELDS = [
+ 'id', 'name', 'os_id'
+]
+RESP_ROLES_FIELDS = [
+ 'id', 'name', 'display_name', 'description', 'optional'
+]
+RESP_FLAVORS_FIELDS = [
+ 'id', 'adapter_id', 'adapter_name', 'name', 'display_name',
+ 'template', 'roles'
+]
+
+
+ADAPTER_MAPPING = None
+FLAVOR_MAPPING = None
+
+
+def load_adapters(force_reload=False):
+ global ADAPTER_MAPPING
+ if force_reload or ADAPTER_MAPPING is None:
+ logging.info('load adapters into memory')
+ ADAPTER_MAPPING = adapter_api.get_adapters_internal(
+ force_reload=force_reload
+ )
+
+
+def load_flavors(force_reload=False):
+ global FLAVOR_MAPPING
+ if force_reload or FLAVOR_MAPPING is None:
+ logging.info('load flavors into memory')
+ FLAVOR_MAPPING = {}
+ adapters_flavors = adapter_api.get_flavors_internal(
+ force_reload=force_reload
+ )
+ for adapter_name, adapter_flavors in adapters_flavors.items():
+ for flavor_name, flavor in adapter_flavors.items():
+ FLAVOR_MAPPING['%s:%s' % (adapter_name, flavor_name)] = flavor
+
+
+def _filter_adapters(adapter_config, filter_name, filter_value):
+ if filter_name not in adapter_config:
+ return False
+ if isinstance(filter_value, list):
+ return bool(
+ adapter_config[filter_name] in filter_value
+ )
+ elif isinstance(filter_value, dict):
+ return all([
+ _filter_adapters(
+ adapter_config[filter_name],
+ sub_filter_key, sub_filter_value
+ )
+ for sub_filter_key, sub_filter_value in filter_value.items()
+ ])
+ else:
+ return adapter_config[filter_name] == filter_value
+
+
+@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_ADAPTERS
+)
+@utils.output_filters(name=utils.general_filter_callback)
+@utils.wrap_to_dict(
+ RESP_FIELDS,
+ supported_oses=RESP_OS_FIELDS,
+ roles=RESP_ROLES_FIELDS,
+ flavors=RESP_FLAVORS_FIELDS
+)
+def list_adapters(user=None, session=None, **filters):
+ """list adapters."""
+ load_adapters()
+ return ADAPTER_MAPPING.values()
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_ADAPTERS
+)
+@utils.wrap_to_dict(
+ RESP_FIELDS,
+ supported_oses=RESP_OS_FIELDS,
+ roles=RESP_ROLES_FIELDS,
+ flavors=RESP_FLAVORS_FIELDS
+)
+def get_adapter(adapter_id, user=None, session=None, **kwargs):
+ """get adapter."""
+ load_adapters()
+ if adapter_id not in ADAPTER_MAPPING:
+ raise exception.RecordNotExists(
+ 'adpater %s does not exist' % adapter_id
+ )
+ return ADAPTER_MAPPING[adapter_id]
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_FLAVORS_FIELDS)
+def list_flavors(user=None, session=None, **filters):
+ """List flavors."""
+ load_flavors()
+ return FLAVOR_MAPPING.values()
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_FLAVORS_FIELDS)
+def get_flavor(flavor_id, user=None, session=None, **kwargs):
+ """Get flavor."""
+ load_flavors()
+ if flavor_id not in FLAVOR_MAPPING:
+ raise exception.RecordNotExists(
+ 'flavor %s does not exist' % flavor_id
+ )
+ return FLAVOR_MAPPING[flavor_id]
diff --git a/compass-tasks/db/api/cluster.py b/compass-tasks/db/api/cluster.py
new file mode 100644
index 0000000..7a7022c
--- /dev/null
+++ b/compass-tasks/db/api/cluster.py
@@ -0,0 +1,2444 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Cluster database operations."""
+import copy
+import functools
+import logging
+import re
+
+from compass.db.api import adapter_holder as adapter_api
+from compass.db.api import database
+from compass.db.api import metadata_holder as metadata_api
+from compass.db.api import permission
+from compass.db.api import user as user_api
+from compass.db.api import utils
+from compass.db import exception
+from compass.db import models
+from compass.utils import util
+
+
+SUPPORTED_FIELDS = [
+ 'name', 'os_name', 'owner',
+ 'adapter_name', 'flavor_name'
+]
+SUPPORTED_CLUSTERHOST_FIELDS = []
+RESP_FIELDS = [
+ 'id', 'name', 'os_name', 'os_id', 'adapter_id', 'flavor_id',
+ 'reinstall_distributed_system', 'flavor',
+ 'distributed_system_installed',
+ 'owner', 'adapter_name', 'flavor_name',
+ 'created_at', 'updated_at'
+]
+RESP_CLUSTERHOST_FIELDS = [
+ 'id', 'host_id', 'clusterhost_id', 'machine_id',
+ 'name', 'hostname', 'roles', 'os_installer',
+ 'cluster_id', 'clustername', 'location', 'tag',
+ 'networks', 'mac', 'switch_ip', 'port', 'switches',
+ 'os_installed', 'distributed_system_installed',
+ 'os_name', 'os_id', 'ip',
+ 'reinstall_os', 'reinstall_distributed_system',
+ 'owner', 'cluster_id',
+ 'created_at', 'updated_at',
+ 'patched_roles'
+]
+RESP_CONFIG_FIELDS = [
+ 'os_config',
+ 'package_config',
+ 'config_step',
+ 'config_validated',
+ 'created_at',
+ 'updated_at'
+]
+RESP_DEPLOYED_CONFIG_FIELDS = [
+ 'deployed_os_config',
+ 'deployed_package_config',
+ 'created_at',
+ 'updated_at'
+]
+RESP_METADATA_FIELDS = [
+ 'os_config', 'package_config'
+]
+RESP_CLUSTERHOST_CONFIG_FIELDS = [
+ 'package_config',
+ 'os_config',
+ 'config_step',
+ 'config_validated',
+ 'networks',
+ 'created_at',
+ 'updated_at'
+]
+RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS = [
+ 'deployed_os_config',
+ 'deployed_package_config',
+ 'created_at',
+ 'updated_at'
+]
+RESP_STATE_FIELDS = [
+ 'id', 'state', 'percentage', 'message', 'severity',
+ 'status', 'ready',
+ 'created_at', 'updated_at'
+]
+RESP_CLUSTERHOST_STATE_FIELDS = [
+ 'id', 'state', 'percentage', 'message', 'severity',
+ 'ready', 'created_at', 'updated_at'
+]
+RESP_REVIEW_FIELDS = [
+ 'cluster', 'hosts'
+]
+RESP_DEPLOY_FIELDS = [
+ 'status', 'cluster', 'hosts'
+]
+IGNORE_FIELDS = ['id', 'created_at', 'updated_at']
+ADDED_FIELDS = ['name', 'adapter_id', 'os_id']
+OPTIONAL_ADDED_FIELDS = ['flavor_id']
+UPDATED_FIELDS = ['name', 'reinstall_distributed_system']
+ADDED_HOST_FIELDS = ['machine_id']
+UPDATED_HOST_FIELDS = ['name', 'reinstall_os']
+UPDATED_CLUSTERHOST_FIELDS = ['roles', 'patched_roles']
+PATCHED_CLUSTERHOST_FIELDS = ['patched_roles']
+UPDATED_CONFIG_FIELDS = [
+ 'put_os_config', 'put_package_config', 'config_step'
+]
+UPDATED_DEPLOYED_CONFIG_FIELDS = [
+ 'deployed_os_config', 'deployed_package_config'
+]
+PATCHED_CONFIG_FIELDS = [
+ 'patched_os_config', 'patched_package_config', 'config_step'
+]
+UPDATED_CLUSTERHOST_CONFIG_FIELDS = [
+ 'put_os_config',
+ 'put_package_config'
+]
+PATCHED_CLUSTERHOST_CONFIG_FIELDS = [
+ 'patched_os_config',
+ 'patched_package_config'
+]
+UPDATED_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS = [
+ 'deployed_os_config',
+ 'deployed_package_config'
+]
+UPDATED_CLUSTERHOST_STATE_FIELDS = [
+ 'state', 'percentage', 'message', 'severity'
+]
+UPDATED_CLUSTERHOST_STATE_INTERNAL_FIELDS = [
+ 'ready'
+]
+UPDATED_CLUSTER_STATE_FIELDS = ['state']
+IGNORE_UPDATED_CLUSTER_STATE_FIELDS = ['percentage', 'message', 'severity']
+UPDATED_CLUSTER_STATE_INTERNAL_FIELDS = ['ready']
+RESP_CLUSTERHOST_LOG_FIELDS = [
+ 'clusterhost_id', 'id', 'host_id', 'cluster_id',
+ 'filename', 'position', 'partial_line',
+ 'percentage',
+ 'message', 'severity', 'line_matcher_name'
+]
+ADDED_CLUSTERHOST_LOG_FIELDS = [
+ 'filename'
+]
+UPDATED_CLUSTERHOST_LOG_FIELDS = [
+ 'position', 'partial_line', 'percentage',
+ 'message', 'severity', 'line_matcher_name'
+]
+
+
+@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_CLUSTERS
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def list_clusters(user=None, session=None, **filters):
+ """List clusters."""
+ clusters = utils.list_db_objects(
+ session, models.Cluster, **filters
+ )
+ logging.info('user is %s', user.email)
+ if not user.is_admin and len(clusters):
+ clusters = [c for c in clusters if c.owner == user.email]
+ return clusters
+
+
+def _get_cluster(cluster_id, session=None, **kwargs):
+ """Get cluster by id."""
+ if isinstance(cluster_id, (int, long)):
+ return utils.get_db_object(
+ session, models.Cluster, id=cluster_id, **kwargs
+ )
+ raise exception.InvalidParameter(
+ 'cluster id %s type is not int compatible' % cluster_id
+ )
+
+
+def get_cluster_internal(cluster_id, session=None, **kwargs):
+ """Helper function to get cluster.
+
+ Should be only used by other files under db/api.
+ """
+ return _get_cluster(cluster_id, session=session, **kwargs)
+
+
+def _get_cluster_host(
+ cluster_id, host_id, session=None, **kwargs
+):
+ """Get clusterhost by cluster id and host id."""
+ cluster = _get_cluster(cluster_id, session=session, **kwargs)
+ from compass.db.api import host as host_api
+ host = host_api.get_host_internal(host_id, session=session, **kwargs)
+ return utils.get_db_object(
+ session, models.ClusterHost,
+ cluster_id=cluster.id,
+ host_id=host.id,
+ **kwargs
+ )
+
+
+def _get_clusterhost(clusterhost_id, session=None, **kwargs):
+ """Get clusterhost by clusterhost id."""
+ if isinstance(clusterhost_id, (int, long)):
+ return utils.get_db_object(
+ session, models.ClusterHost,
+ clusterhost_id=clusterhost_id,
+ **kwargs
+ )
+ raise exception.InvalidParameter(
+ 'clusterhost id %s type is not int compatible' % clusterhost_id
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_CLUSTERS
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def get_cluster(
+ cluster_id, exception_when_missing=True,
+ user=None, session=None, **kwargs
+):
+ """Get cluster info."""
+ return _get_cluster(
+ cluster_id,
+ session=session,
+ exception_when_missing=exception_when_missing
+ )
+
+
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_CLUSTERS)
+def is_cluster_os_ready(
+ cluster_id, exception_when_missing=True,
+ user=None, session=None, **kwargs
+):
+ cluster = utils.get_db_object(
+ session, models.Cluster, exception_when_missing, id=cluster_id)
+
+ all_states = ([i.host.state.ready for i in cluster.clusterhosts])
+
+ logging.info("is_cluster_os_ready: all_states %s" % all_states)
+
+ return all(all_states)
+
+
+def check_cluster_validated(cluster):
+ """Check cluster is validated."""
+ if not cluster.config_validated:
+ raise exception.Forbidden(
+ 'cluster %s is not validated' % cluster.name
+ )
+
+
+def check_clusterhost_validated(clusterhost):
+ """Check clusterhost is validated."""
+ if not clusterhost.config_validated:
+ raise exception.Forbidden(
+ 'clusterhost %s is not validated' % clusterhost.name
+ )
+
+
+def check_cluster_editable(
+ cluster, user=None,
+ check_in_installing=False
+):
+ """Check if cluster is editable.
+
+ If we try to set cluster
+ reinstall_distributed_system attribute or any
+ checking to make sure the cluster is not in installing state,
+ we can set check_in_installing to True.
+ Otherwise we will make sure the cluster is not in deploying or
+ deployed.
+ If user is not admin or not the owner of the cluster, the check
+ will fail to make sure he can not update the cluster attributes.
+ """
+ if check_in_installing:
+ if cluster.state.state == 'INSTALLING':
+ raise exception.Forbidden(
+ 'cluster %s is not editable '
+ 'when state is installing' % cluster.name
+ )
+# elif (
+# cluster.flavor_name and
+# not cluster.reinstall_distributed_system
+# ):
+# raise exception.Forbidden(
+# 'cluster %s is not editable '
+# 'when not to be reinstalled' % cluster.name
+# )
+ if user and not user.is_admin and cluster.creator_id != user.id:
+ raise exception.Forbidden(
+ 'cluster %s is not editable '
+ 'when user is not admin or cluster owner' % cluster.name
+ )
+
+
+def is_cluster_editable(
+ cluster, user=None,
+ check_in_installing=False
+):
+ """Get if cluster is editble."""
+ try:
+ check_cluster_editable(
+ cluster, user=user,
+ check_in_installing=check_in_installing
+ )
+ return True
+ except exception.Forbidden:
+ return False
+
+
+@utils.supported_filters(
+ ADDED_FIELDS,
+ optional_support_keys=OPTIONAL_ADDED_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(name=utils.check_name)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_CLUSTER
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def add_cluster(
+ exception_when_existing=True,
+ name=None, adapter_id=None, flavor_id=None,
+ user=None, session=None, **kwargs
+):
+ """Create a cluster."""
+ adapter = adapter_api.get_adapter(
+ adapter_id, user=user, session=session
+ )
+ # if flavor_id is not None, also set flavor field.
+ # In future maybe we can move the use of flavor from
+ # models.py to db/api and explictly get flavor when
+ # needed instead of setting flavor into cluster record.
+ flavor = {}
+ if flavor_id:
+ flavor = adapter_api.get_flavor(
+ flavor_id,
+ user=user, session=session
+ )
+ if flavor['adapter_id'] != adapter['id']:
+ raise exception.InvalidParameter(
+ 'flavor %s is not of adapter %s' % (
+ flavor_id, adapter_id
+ )
+ )
+
+ cluster = utils.add_db_object(
+ session, models.Cluster, exception_when_existing,
+ name, user.id, adapter_id=adapter_id,
+ flavor_id=flavor_id, flavor=flavor, **kwargs
+ )
+ return cluster
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(name=utils.check_name)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_CLUSTER
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def update_cluster(cluster_id, user=None, session=None, **kwargs):
+ """Update a cluster."""
+ cluster = _get_cluster(
+ cluster_id, session=session
+ )
+ check_cluster_editable(
+ cluster, user=user,
+ check_in_installing=(
+ kwargs.get('reinstall_distributed_system', False)
+ )
+ )
+ return utils.update_db_object(session, cluster, **kwargs)
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEL_CLUSTER
+)
+@utils.wrap_to_dict(
+ RESP_FIELDS + ['status', 'cluster', 'hosts'],
+ cluster=RESP_FIELDS,
+ hosts=RESP_CLUSTERHOST_FIELDS
+)
+def del_cluster(
+ cluster_id, force=False, from_database_only=False,
+ delete_underlying_host=False, user=None, session=None, **kwargs
+):
+ """Delete a cluster.
+
+ If force, the cluster will be deleted anyway. It is used by cli to
+ force clean a cluster in any case.
+ If from_database_only, the cluster recored will only be removed from
+ database. Otherwise, a del task is sent to celery to do clean deletion.
+ If delete_underlying_host, all hosts under this cluster will also be
+ deleted.
+ The backend will call del_cluster again with from_database_only set
+ when it has done the deletion work on os installer/package installer.
+ """
+ cluster = _get_cluster(
+ cluster_id, session=session
+ )
+ logging.debug(
+ 'delete cluster %s with force=%s '
+ 'from_database_only=%s delete_underlying_host=%s',
+ cluster.id, force, from_database_only, delete_underlying_host
+ )
+ # force set cluster state to ERROR and the state of any clusterhost
+ # in the cluster to ERROR when we want to delete the cluster anyway
+ # even the cluster is in installing or already installed.
+ # It let the api know the deleting is in doing when backend is doing
+ # the real deleting.
+ # In future we may import a new state like INDELETE to indicate
+ # the deleting is processing.
+ # We need discuss about if we can delete a cluster when it is already
+ # installed by api.
+ for clusterhost in cluster.clusterhosts:
+ if clusterhost.state.state != 'UNINITIALIZED' and force:
+ clusterhost.state.state = 'ERROR'
+ if delete_underlying_host:
+ host = clusterhost.host
+ if host.state.state != 'UNINITIALIZED' and force:
+ host.state.state = 'ERROR'
+ if cluster.state.state != 'UNINITIALIZED' and force:
+ cluster.state.state = 'ERROR'
+
+ check_cluster_editable(
+ cluster, user=user,
+ check_in_installing=True
+ )
+
+ # delete underlying host if delete_underlying_host is set.
+ if delete_underlying_host:
+ for clusterhost in cluster.clusterhosts:
+ # delete underlying host only user has permission.
+ from compass.db.api import host as host_api
+ host = clusterhost.host
+ if host_api.is_host_editable(
+ host, user=user, check_in_installing=True
+ ):
+ # Delete host record directly in database when there is no need
+ # to do the deletion in backend or from_database_only is set.
+ if host.state.state == 'UNINITIALIZED' or from_database_only:
+ utils.del_db_object(
+ session, host
+ )
+
+ # Delete cluster record directly in database when there
+ # is no need to do the deletion in backend or from_database_only is set.
+ if cluster.state.state == 'UNINITIALIZED' or from_database_only:
+ return utils.del_db_object(
+ session, cluster
+ )
+ else:
+ from compass.tasks import client as celery_client
+ logging.info('send del cluster %s task to celery', cluster_id)
+ celery_client.celery.send_task(
+ 'compass.tasks.delete_cluster',
+ (
+ user.email, cluster.id,
+ [
+ clusterhost.host_id
+ for clusterhost in cluster.clusterhosts
+ ],
+ delete_underlying_host
+ ),
+ queue=user.email,
+ exchange=user.email,
+ routing_key=user.email
+ )
+ return {
+ 'status': 'delete action is sent',
+ 'cluster': cluster,
+ 'hosts': cluster.clusterhosts
+ }
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_CLUSTER_CONFIG
+)
+@utils.wrap_to_dict(RESP_CONFIG_FIELDS)
+def get_cluster_config(cluster_id, user=None, session=None, **kwargs):
+ """Get cluster config."""
+ return _get_cluster(cluster_id, session=session)
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_CLUSTER_CONFIG
+)
+@utils.wrap_to_dict(RESP_DEPLOYED_CONFIG_FIELDS)
+def get_cluster_deployed_config(cluster_id, user=None, session=None, **kwargs):
+ """Get cluster deployed config."""
+ return _get_cluster(cluster_id, session=session)
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_METADATA_FIELDS)
+def get_cluster_metadata(cluster_id, user=None, session=None, **kwargs):
+ """Get cluster metadata.
+
+ If no flavor in the cluster, it means this is a os only cluster.
+ We ignore package metadata for os only cluster.
+ """
+ cluster = _get_cluster(cluster_id, session=session)
+ metadatas = {}
+ os_name = cluster.os_name
+ if os_name:
+ metadatas.update(
+ metadata_api.get_os_metadata(
+ os_name, session=session
+ )
+ )
+ flavor_id = cluster.flavor_id
+ if flavor_id:
+ metadatas.update(
+ metadata_api.get_flavor_metadata(
+ flavor_id,
+ user=user, session=session
+ )
+ )
+
+ return metadatas
+
+
+def _cluster_os_config_validates(
+ config, cluster, session=None, user=None, **kwargs
+):
+ """Check cluster os config validation."""
+ metadata_api.validate_os_config(
+ config, cluster.os_id
+ )
+
+
+def _cluster_package_config_validates(
+ config, cluster, session=None, user=None, **kwargs
+):
+ """Check cluster package config validation."""
+ metadata_api.validate_flavor_config(
+ config, cluster.flavor_id
+ )
+
+
+@utils.input_validates_with_args(
+ put_os_config=_cluster_os_config_validates,
+ put_package_config=_cluster_package_config_validates
+)
+@utils.output_validates_with_args(
+ os_config=_cluster_os_config_validates,
+ package_config=_cluster_package_config_validates
+)
+@utils.wrap_to_dict(RESP_CONFIG_FIELDS)
+def _update_cluster_config(cluster, session=None, user=None, **kwargs):
+ """Update a cluster config."""
+ check_cluster_editable(cluster, user=user)
+ return utils.update_db_object(
+ session, cluster, **kwargs
+ )
+
+
+# replace os_config to deployed_os_config,
+# package_config to deployed_package_config
+@utils.replace_filters(
+ os_config='deployed_os_config',
+ package_config='deployed_package_config'
+)
+@utils.supported_filters(
+ optional_support_keys=UPDATED_DEPLOYED_CONFIG_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_CLUSTER_CONFIG
+)
+@utils.wrap_to_dict(RESP_DEPLOYED_CONFIG_FIELDS)
+def update_cluster_deployed_config(
+ cluster_id, user=None, session=None, **kwargs
+):
+ """Update cluster deployed config."""
+ cluster = _get_cluster(cluster_id, session=session)
+ check_cluster_editable(cluster, user=user)
+ check_cluster_validated(cluster)
+ return utils.update_db_object(
+ session, cluster, **kwargs
+ )
+
+
+# replace os_config to put_os_config,
+# package_config to put_package_config in kwargs.
+# It tells db these fields will be updated not patched.
+@utils.replace_filters(
+ os_config='put_os_config',
+ package_config='put_package_config'
+)
+@utils.supported_filters(
+ optional_support_keys=UPDATED_CONFIG_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_CLUSTER_CONFIG
+)
+def update_cluster_config(cluster_id, user=None, session=None, **kwargs):
+ """Update cluster config."""
+ cluster = _get_cluster(cluster_id, session=session)
+ return _update_cluster_config(
+ cluster, session=session, user=user, **kwargs
+ )
+
+
+# replace os_config to patched_os_config and
+# package_config to patched_package_config in kwargs.
+# It tells db these fields will be patched not updated.
+@utils.replace_filters(
+ os_config='patched_os_config',
+ package_config='patched_package_config'
+)
+@utils.supported_filters(
+ optional_support_keys=PATCHED_CONFIG_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_CLUSTER_CONFIG
+)
+def patch_cluster_config(cluster_id, user=None, session=None, **kwargs):
+ """patch cluster config."""
+ cluster = _get_cluster(cluster_id, session=session)
+ return _update_cluster_config(
+ cluster, session=session, user=user, **kwargs
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEL_CLUSTER_CONFIG
+)
+@utils.wrap_to_dict(RESP_CONFIG_FIELDS)
+def del_cluster_config(cluster_id, user=None, session=None):
+ """Delete a cluster config."""
+ cluster = _get_cluster(
+ cluster_id, session=session
+ )
+ check_cluster_editable(cluster, user=user)
+ return utils.update_db_object(
+ session, cluster, os_config={},
+ package_config={}, config_validated=False
+ )
+
+
+def _roles_validates(roles, cluster, session=None, user=None):
+ """Check roles is validated to a cluster's roles."""
+ if roles:
+ if not cluster.flavor_name:
+ raise exception.InvalidParameter(
+ 'not flavor in cluster %s' % cluster.name
+ )
+ cluster_roles = [role['name'] for role in cluster.flavor['roles']]
+ for role in roles:
+ if role not in cluster_roles:
+ raise exception.InvalidParameter(
+ 'role %s is not in cluster roles %s' % (
+ role, cluster_roles
+ )
+ )
+
+
+def _cluster_host_roles_validates(
+ value, cluster, host, session=None, user=None, **kwargs
+):
+ """Check clusterhost roles is validated by cluster and host."""
+ _roles_validates(value, cluster, session=session, user=user)
+
+
+def _clusterhost_roles_validates(
+ value, clusterhost, session=None, user=None, **kwargs
+):
+ """Check clusterhost roles is validated by clusterhost."""
+ _roles_validates(
+ value, clusterhost.cluster, session=session, user=user
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_HOST_FIELDS,
+ ignore_support_keys=UPDATED_CLUSTERHOST_FIELDS
+)
+@utils.input_validates(name=utils.check_name)
+def _add_host_if_not_exist(
+ machine_id, cluster, session=None, user=None, **kwargs
+):
+ """Add underlying host if it does not exist."""
+ from compass.db.api import host as host_api
+ host = host_api.get_host_internal(
+ machine_id, session=session, exception_when_missing=False
+ )
+ if host:
+ if kwargs:
+ # ignore update underlying host if host is not editable.
+ from compass.db.api import host as host_api
+ if host_api.is_host_editable(
+ host, user=cluster.creator,
+ check_in_installing=kwargs.get('reinstall_os', False),
+ ):
+ utils.update_db_object(
+ session, host,
+ **kwargs
+ )
+ else:
+ logging.debug(
+ 'ignore update host host %s '
+ 'since it is not editable' % host.name
+ )
+ else:
+ logging.debug('nothing to update for host %s', host.name)
+ else:
+ from compass.db.api import adapter_holder as adapter_api
+ adapter = adapter_api.get_adapter(
+ cluster.adapter_name, user=user, session=session
+ )
+ host = utils.add_db_object(
+ session, models.Host, False, machine_id,
+ os_name=cluster.os_name,
+ os_installer=adapter['os_installer'],
+ creator=cluster.creator,
+ **kwargs
+ )
+ return host
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_CLUSTERHOST_FIELDS,
+ ignore_support_keys=UPDATED_HOST_FIELDS
+)
+@utils.input_validates_with_args(
+ roles=_cluster_host_roles_validates
+)
+def _add_clusterhost_only(
+ cluster, host,
+ exception_when_existing=False,
+ session=None, user=None,
+ **kwargs
+):
+ """Get clusterhost only."""
+ if not cluster.state.state == "UNINITIALIZED":
+ cluster.state.ready = False
+ cluster.state.state = "UNINITIALIZED"
+ cluster.state.percentage = 0.0
+ utils.update_db_object(session, cluster.state, state="UNINITIALIZED")
+
+ return utils.add_db_object(
+ session, models.ClusterHost, exception_when_existing,
+ cluster.id, host.id, **kwargs
+ )
+
+
+@utils.supported_filters(
+ ADDED_HOST_FIELDS,
+ optional_support_keys=UPDATED_HOST_FIELDS + UPDATED_CLUSTERHOST_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+def _add_clusterhost(
+ cluster,
+ exception_when_existing=False,
+ session=None, user=None, machine_id=None, **kwargs
+):
+ """Add clusterhost and add underlying host if it does not exist."""
+ host = _add_host_if_not_exist(
+ machine_id, cluster, session=session,
+ user=user, **kwargs
+ )
+
+ return _add_clusterhost_only(
+ cluster, host, exception_when_existing=exception_when_existing,
+ session=session, user=user, **kwargs
+ )
+
+
+def _add_clusterhosts(cluster, machines, session=None, user=None):
+ """Add machines to cluster.
+
+ Args:
+ machines: list of dict which contains clusterost attr to update.
+
+ Examples:
+ [{'machine_id': 1, 'name': 'host1'}]
+ """
+ check_cluster_editable(
+ cluster, user=user,
+ check_in_installing=True
+ )
+ if cluster.state.state == 'SUCCESSFUL':
+ cluster.state.state == 'UPDATE_PREPARING'
+ for machine_dict in machines:
+ _add_clusterhost(
+ cluster, session=session, user=user, **machine_dict
+ )
+
+
+def _remove_clusterhosts(cluster, hosts, session=None, user=None):
+ """Remove hosts from cluster.
+
+ Args:
+ hosts: list of host id.
+ """
+ check_cluster_editable(
+ cluster, user=user,
+ check_in_installing=True
+ )
+ utils.del_db_objects(
+ session, models.ClusterHost,
+ cluster_id=cluster.id, host_id=hosts
+ )
+
+
+def _set_clusterhosts(cluster, machines, session=None, user=None):
+ """set machines to cluster.
+
+ Args:
+ machines: list of dict which contains clusterost attr to update.
+
+ Examples:
+ [{'machine_id': 1, 'name': 'host1'}]
+ """
+ check_cluster_editable(
+ cluster, user=user,
+ check_in_installing=True
+ )
+ utils.del_db_objects(
+ session, models.ClusterHost,
+ cluster_id=cluster.id
+ )
+ if cluster.state.state == 'SUCCESSFUL':
+ cluster.state.state = 'UPDATE_PREPARING'
+ for machine_dict in machines:
+ _add_clusterhost(
+ cluster, True, session=session, user=user, **machine_dict
+ )
+
+
+@utils.supported_filters(optional_support_keys=SUPPORTED_CLUSTERHOST_FIELDS)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_CLUSTERHOSTS
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS)
+def list_cluster_hosts(cluster_id, user=None, session=None, **filters):
+ """List clusterhosts of a cluster."""
+ cluster = _get_cluster(cluster_id, session=session)
+ return utils.list_db_objects(
+ session, models.ClusterHost, cluster_id=cluster.id,
+ **filters
+ )
+
+
+@utils.supported_filters(optional_support_keys=SUPPORTED_CLUSTERHOST_FIELDS)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_CLUSTERHOSTS
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS)
+def list_clusterhosts(user=None, session=None, **filters):
+ """List all clusterhosts."""
+ return utils.list_db_objects(
+ session, models.ClusterHost, **filters
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_CLUSTERHOSTS
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS)
+def get_cluster_host(
+ cluster_id, host_id, exception_when_missing=True,
+ user=None, session=None, **kwargs
+):
+ """Get clusterhost info by cluster id and host id."""
+ return _get_cluster_host(
+ cluster_id, host_id, session=session,
+ exception_when_missing=exception_when_missing,
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_CLUSTERHOSTS
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS)
+def get_clusterhost(
+ clusterhost_id, exception_when_missing=True,
+ user=None, session=None, **kwargs
+):
+ """Get clusterhost info by clusterhost id."""
+ return _get_clusterhost(
+ clusterhost_id, session=session,
+ exception_when_missing=exception_when_missing,
+ user=user
+ )
+
+
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_CLUSTER_HOSTS
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS)
+def add_cluster_host(
+ cluster_id, exception_when_existing=True,
+ user=None, session=None, **kwargs
+):
+ """Add a host to a cluster."""
+ cluster = _get_cluster(cluster_id, session=session)
+ check_cluster_editable(
+ cluster, user=user,
+ check_in_installing=True
+ )
+ if cluster.state.state == 'SUCCESSFUL':
+ cluster.state.state = 'UPDATE_PREPARING'
+ return _add_clusterhost(
+ cluster, exception_when_existing,
+ session=session, user=user, **kwargs
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_HOST_FIELDS,
+ ignore_support_keys=(
+ UPDATED_CLUSTERHOST_FIELDS +
+ PATCHED_CLUSTERHOST_FIELDS
+ )
+)
+def _update_host_if_necessary(
+ clusterhost, session=None, user=None, **kwargs
+):
+ """Update underlying host if there is something to update."""
+ host = clusterhost.host
+ if kwargs:
+ # ignore update underlying host if the host is not editable.
+ from compass.db.api import host as host_api
+ if host_api.is_host_editable(
+ host, user=clusterhost.cluster.creator,
+ check_in_installing=kwargs.get('reinstall_os', False),
+ ):
+ utils.update_db_object(
+ session, host,
+ **kwargs
+ )
+ else:
+ logging.debug(
+ 'ignore update host %s since it is not editable' % host.name
+ )
+ else:
+ logging.debug(
+ 'nothing to update for host %s', host.name
+ )
+ return host
+
+
+@utils.supported_filters(
+ optional_support_keys=(
+ UPDATED_CLUSTERHOST_FIELDS +
+ PATCHED_CLUSTERHOST_FIELDS
+ ),
+ ignore_support_keys=UPDATED_HOST_FIELDS
+)
+@utils.input_validates_with_args(
+ roles=_clusterhost_roles_validates,
+ patched_roles=_clusterhost_roles_validates
+)
+def _update_clusterhost_only(
+ clusterhost, session=None, user=None, **kwargs
+):
+ """Update clusterhost only."""
+ check_cluster_editable(clusterhost.cluster, user=user)
+ return utils.update_db_object(
+ session, clusterhost, **kwargs
+ )
+
+
+@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS)
+def _update_clusterhost(clusterhost, session=None, user=None, **kwargs):
+ """Update clusterhost and underlying host if necessary."""
+ _update_host_if_necessary(
+ clusterhost, session=session, user=user, **kwargs
+ )
+ return _update_clusterhost_only(
+ clusterhost, session=session, user=user, **kwargs
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=(UPDATED_HOST_FIELDS + UPDATED_CLUSTERHOST_FIELDS),
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_CLUSTER_HOSTS
+)
+def update_cluster_host(
+ cluster_id, host_id, user=None,
+ session=None, **kwargs
+):
+ """Update clusterhost by cluster id and host id."""
+ logging.info('updating kwargs: %s', kwargs)
+ clusterhost = _get_cluster_host(
+ cluster_id, host_id, session=session
+ )
+ return _update_clusterhost(
+ clusterhost, session=session, user=user, **kwargs
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=(UPDATED_HOST_FIELDS + UPDATED_CLUSTERHOST_FIELDS),
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_CLUSTER_HOSTS
+)
+def update_clusterhost(
+ clusterhost_id, user=None,
+ session=None, **kwargs
+):
+ """Update clusterhost by clusterhost id."""
+ clusterhost = _get_clusterhost(
+ clusterhost_id, session=session
+ )
+ return _update_clusterhost(
+ clusterhost, session=session, user=user, **kwargs
+ )
+
+
+# replace roles to patched_roles in kwargs.
+# It tells db roles field will be patched.
+@utils.replace_filters(
+ roles='patched_roles'
+)
+@utils.supported_filters(
+ optional_support_keys=PATCHED_CLUSTERHOST_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_CLUSTER_HOSTS
+)
+def patch_cluster_host(
+ cluster_id, host_id, user=None,
+ session=None, **kwargs
+):
+ """Patch clusterhost by cluster id and host id."""
+ logging.info("kwargs are %s", kwargs)
+ clusterhost = _get_cluster_host(
+ cluster_id, host_id, session=session
+ )
+ updated_clusterhost = _update_clusterhost(
+ clusterhost, session=session, user=user, **kwargs
+ )
+ return updated_clusterhost
+
+
+# replace roles to patched_roles in kwargs.
+# It tells db roles field will be patched.
+@utils.replace_filters(
+ roles='patched_roles'
+)
+@utils.supported_filters(
+ optional_support_keys=PATCHED_CLUSTERHOST_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_CLUSTER_HOSTS
+)
+def patch_clusterhost(
+ clusterhost_id, user=None, session=None,
+ **kwargs
+):
+ """Patch clusterhost by clusterhost id."""
+ clusterhost = _get_clusterhost(
+ clusterhost_id, session=session
+ )
+ return _update_clusterhost(
+ clusterhost, session=session, user=user, **kwargs
+ )
+
+
+@user_api.check_user_permission(
+ permission.PERMISSION_DEL_CLUSTER_HOST
+)
+@utils.wrap_to_dict(
+ RESP_CLUSTERHOST_FIELDS + ['status', 'host'],
+ host=RESP_CLUSTERHOST_FIELDS
+)
+def _del_cluster_host(
+ clusterhost,
+ force=False, from_database_only=False,
+ delete_underlying_host=False, user=None,
+ session=None, **kwargs
+):
+ """delete clusterhost.
+
+ If force, the cluster host will be deleted anyway.
+ If from_database_only, the cluster host recored will only be
+ deleted from database. Otherwise a celery task sent to do
+ clean deletion.
+ If delete_underlying_host, the underlying host will also be deleted.
+ The backend will call _del_cluster_host again when the clusterhost is
+ deleted from os installer/package installer with from_database_only
+ set.
+ """
+ # force set clusterhost state to ERROR when we want to delete the
+ # clusterhost anyway even the clusterhost is in installing or already
+ # installed. It let the api know the deleting is in doing when backend
+ # is doing the real deleting. In future we may import a new state like
+ # INDELETE to indicate the deleting is processing.
+ # We need discuss about if we can delete a clusterhost when it is already
+ # installed by api.
+ if clusterhost.state.state != 'UNINITIALIZED' and force:
+ clusterhost.state.state = 'ERROR'
+ if not force:
+ check_cluster_editable(
+ clusterhost.cluster, user=user,
+ check_in_installing=True
+ )
+ # delete underlying host if delete_underlying_host is set.
+ if delete_underlying_host:
+ host = clusterhost.host
+ if host.state.state != 'UNINITIALIZED' and force:
+ host.state.state = 'ERROR'
+ # only delete the host when user have the permission to delete it.
+ import compass.db.api.host as host_api
+ if host_api.is_host_editable(
+ host, user=user,
+ check_in_installing=True
+ ):
+ # if there is no need to do the deletion by backend or
+ # from_database_only is set, we only delete the record
+ # in database.
+ if host.state.state == 'UNINITIALIZED' or from_database_only:
+ utils.del_db_object(
+ session, host
+ )
+
+ # if there is no need to do the deletion by backend or
+ # from_database_only is set, we only delete the record in database.
+ if clusterhost.state.state == 'UNINITIALIZED' or from_database_only:
+ return utils.del_db_object(
+ session, clusterhost
+ )
+ else:
+ logging.info(
+ 'send del cluster %s host %s task to celery',
+ clusterhost.cluster_id, clusterhost.host_id
+ )
+ from compass.tasks import client as celery_client
+ celery_client.celery.send_task(
+ 'compass.tasks.delete_cluster_host',
+ (
+ user.email, clusterhost.cluster_id, clusterhost.host_id,
+ delete_underlying_host
+ ),
+ queue=user.email,
+ exchange=user.email,
+ routing_key=user.email
+ )
+ return {
+ 'status': 'delete action sent',
+ 'host': clusterhost,
+ }
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+def del_cluster_host(
+ cluster_id, host_id,
+ force=False, from_database_only=False,
+ delete_underlying_host=False, user=None,
+ session=None, **kwargs
+):
+ """Delete clusterhost by cluster id and host id."""
+ clusterhost = _get_cluster_host(
+ cluster_id, host_id, session=session
+ )
+ return _del_cluster_host(
+ clusterhost, force=force, from_database_only=from_database_only,
+ delete_underlying_host=delete_underlying_host, user=user,
+ session=session, **kwargs
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+def del_clusterhost(
+ clusterhost_id,
+ force=False, from_database_only=False,
+ delete_underlying_host=False, user=None,
+ session=None, **kwargs
+):
+ """Delete clusterhost by clusterhost id."""
+ clusterhost = _get_clusterhost(
+ clusterhost_id, session=session
+ )
+ return _del_cluster_host(
+ clusterhost, force=force, from_database_only=from_database_only,
+ delete_underlying_host=delete_underlying_host, user=user,
+ session=session, **kwargs
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_CLUSTERHOST_CONFIG
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS)
+def get_cluster_host_config(
+ cluster_id, host_id, user=None,
+ session=None, **kwargs
+):
+ """Get clusterhost config by cluster id and host id."""
+ return _get_cluster_host(
+ cluster_id, host_id, session=session
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_CLUSTERHOST_CONFIG
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS)
+def get_cluster_host_deployed_config(
+ cluster_id, host_id, user=None, session=None, **kwargs
+):
+ """Get clusterhost deployed config by cluster id and host id."""
+ return _get_cluster_host(
+ cluster_id, host_id, session=session
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_CLUSTERHOST_CONFIG
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS)
+def get_clusterhost_config(clusterhost_id, user=None, session=None, **kwargs):
+ """Get clusterhost config by clusterhost id."""
+ return _get_clusterhost(
+ clusterhost_id, session=session
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_CLUSTERHOST_CONFIG
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS)
+def get_clusterhost_deployed_config(
+ clusterhost_id, user=None,
+ session=None, **kwargs
+):
+ """Get clusterhost deployed config by clusterhost id."""
+ return _get_clusterhost(
+ clusterhost_id, session=session
+ )
+
+
+def _clusterhost_os_config_validates(
+ config, clusterhost, session=None, user=None, **kwargs
+):
+ """Validate clusterhost's underlying host os config."""
+ from compass.db.api import host as host_api
+ host = clusterhost.host
+ host_api.check_host_editable(host, user=user)
+ metadata_api.validate_os_config(
+ config, host.os_id
+ )
+
+
+def _clusterhost_package_config_validates(
+ config, clusterhost, session=None, user=None, **kwargs
+):
+ """Validate clusterhost's cluster package config."""
+ cluster = clusterhost.cluster
+ check_cluster_editable(cluster, user=user)
+ metadata_api.validate_flavor_config(
+ config, cluster.flavor_id
+ )
+
+
+def _filter_clusterhost_host_editable(
+ config, clusterhost, session=None, user=None, **kwargs
+):
+ """Filter fields if the underlying host is not editable."""
+ from compass.db.api import host as host_api
+ host = clusterhost.host
+ return host_api.is_host_editable(host, user=user)
+
+
+@utils.input_filters(
+ put_os_config=_filter_clusterhost_host_editable,
+ patched_os_config=_filter_clusterhost_host_editable
+)
+@utils.input_validates_with_args(
+ put_os_config=_clusterhost_os_config_validates,
+ put_package_config=_clusterhost_package_config_validates
+)
+@utils.output_validates_with_args(
+ os_config=_clusterhost_os_config_validates,
+ package_config=_clusterhost_package_config_validates
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS)
+def _update_clusterhost_config(clusterhost, session=None, user=None, **kwargs):
+ """Update clusterhost config."""
+ return utils.update_db_object(
+ session, clusterhost, **kwargs
+ )
+
+
+def _clusterhost_host_validated(
+ config, clusterhost, session=None, user=None, **kwargs
+):
+ """Check clusterhost's underlying host is validated."""
+ from compass.db.api import host as host_api
+ host = clusterhost.host
+ host_api.check_host_editable(host, user=user)
+ host_api.check_host_validated(host)
+
+
+def _clusterhost_cluster_validated(
+ config, clusterhost, session=None, user=None, **kwargs
+):
+ """Check clusterhost's cluster is validated."""
+ cluster = clusterhost.cluster
+ check_cluster_editable(cluster, user=user)
+ check_clusterhost_validated(clusterhost)
+
+
+@utils.input_filters(
+ deployed_os_config=_filter_clusterhost_host_editable,
+)
+@utils.input_validates_with_args(
+ deployed_os_config=_clusterhost_host_validated,
+ deployed_package_config=_clusterhost_cluster_validated
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS)
+def _update_clusterhost_deployed_config(
+ clusterhost, session=None, user=None, **kwargs
+):
+ """Update clusterhost deployed config."""
+ return utils.update_db_object(
+ session, clusterhost, **kwargs
+ )
+
+
+# replace os_config to put_os_config and
+# package_config to put_package_config in kwargs.
+# It tells db these fields will be updated not patched.
+@utils.replace_filters(
+ os_config='put_os_config',
+ package_config='put_package_config'
+)
+@utils.supported_filters(
+ optional_support_keys=UPDATED_CLUSTERHOST_CONFIG_FIELDS,
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_CLUSTERHOST_CONFIG
+)
+def update_cluster_host_config(
+ cluster_id, host_id, user=None, session=None, **kwargs
+):
+ """Update clusterhost config by cluster id and host id."""
+ clusterhost = _get_cluster_host(
+ cluster_id, host_id, session=session
+ )
+ return _update_clusterhost_config(
+ clusterhost, user=user, session=session, **kwargs
+ )
+
+
+# replace os_config to deployed_os_config and
+# package_config to deployed_package_config in kwargs.
+@utils.replace_filters(
+ os_config='deployed_os_config',
+ package_config='deployed_package_config'
+)
+@utils.supported_filters(
+ optional_support_keys=UPDATED_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_CLUSTERHOST_CONFIG
+)
+def update_cluster_host_deployed_config(
+ cluster_id, host_id, user=None, session=None, **kwargs
+):
+ """Update clusterhost deployed config by cluster id and host id."""
+ clusterhost = _get_cluster_host(
+ cluster_id, host_id, session=session
+ )
+ return _update_clusterhost_deployed_config(
+ clusterhost, session=session, user=user, **kwargs
+ )
+
+
+# replace os_config to put_os_config and
+# package_config to put_package_config in kwargs.
+# It tells db these fields will be updated not patched.
+@utils.replace_filters(
+ os_config='put_os_config',
+ package_config='put_package_config'
+)
+@utils.supported_filters(
+ optional_support_keys=UPDATED_CLUSTERHOST_CONFIG_FIELDS,
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_CLUSTERHOST_CONFIG
+)
+def update_clusterhost_config(
+ clusterhost_id, user=None, session=None, **kwargs
+):
+ """Update clusterhost config by clusterhost id."""
+ clusterhost = _get_clusterhost(
+ clusterhost_id, session=session
+ )
+ return _update_clusterhost_config(
+ clusterhost, session=session, user=user, **kwargs
+ )
+
+
+# replace os_config to deployed_os_config and
+# package_config to deployed_package_config in kwargs.
+@utils.replace_filters(
+ os_config='deployed_os_config',
+ package_config='deployed_package_config'
+)
+@utils.supported_filters(
+ optional_support_keys=UPDATED_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_CLUSTERHOST_CONFIG
+)
+def update_clusterhost_deployed_config(
+ clusterhost_id, user=None, session=None, **kwargs
+):
+ """Update clusterhost deployed config by clusterhost id."""
+ clusterhost = _get_clusterhost(
+ clusterhost_id, session=session
+ )
+ return _update_clusterhost_deployed_config(
+ clusterhost, session=session, user=user, **kwargs
+ )
+
+
+# replace os_config to patched_os_config and
+# package_config to patched_package_config in kwargs
+# It tells db these fields will be patched not updated.
+@utils.replace_filters(
+ os_config='patched_os_config',
+ package_config='patched_package_config'
+)
+@utils.supported_filters(
+ optional_support_keys=PATCHED_CLUSTERHOST_CONFIG_FIELDS,
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_CLUSTERHOST_CONFIG
+)
+def patch_cluster_host_config(
+ cluster_id, host_id, user=None, session=None, **kwargs
+):
+ """patch clusterhost config by cluster id and host id."""
+ clusterhost = _get_cluster_host(
+ cluster_id, host_id, session=session
+ )
+ return _update_clusterhost_config(
+ clusterhost, session=session, user=user, **kwargs
+ )
+
+
+# replace os_config to patched_os_config and
+# package_config to patched_package_config in kwargs
+# It tells db these fields will be patched not updated.
+@utils.replace_filters(
+ os_config='patched_os_config',
+ package_config='patched_package_config'
+)
+@utils.supported_filters(
+ optional_support_keys=PATCHED_CLUSTERHOST_CONFIG_FIELDS,
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_CLUSTERHOST_CONFIG
+)
+def patch_clusterhost_config(
+ clusterhost_id, user=None, session=None, **kwargs
+):
+ """patch clusterhost config by clusterhost id."""
+ clusterhost = _get_clusterhost(
+ clusterhost_id, session=session
+ )
+ return _update_clusterhost_config(
+ clusterhost, session=session, user=user, **kwargs
+ )
+
+
+def _clusterhost_host_editable(
+ config, clusterhost, session=None, user=None, **kwargs
+):
+ """Check clusterhost underlying host is editable."""
+ from compass.db.api import host as host_api
+ host_api.check_host_editable(clusterhost.host, user=user)
+
+
+def _clusterhost_cluster_editable(
+ config, clusterhost, session=None, user=None, **kwargs
+):
+ """Check clusterhost's cluster is editable."""
+ check_cluster_editable(clusterhost.cluster, user=user)
+
+
+@utils.supported_filters(
+ optional_support_keys=['os_config', 'package_config']
+)
+@utils.input_filters(
+ os_config=_filter_clusterhost_host_editable,
+)
+@utils.output_validates_with_args(
+ package_config=_clusterhost_cluster_editable
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS)
+def _delete_clusterhost_config(
+ clusterhost, session=None, user=None, **kwargs
+):
+ """delete clusterhost config."""
+ return utils.update_db_object(
+ session, clusterhost, config_validated=False,
+ **kwargs
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEL_CLUSTERHOST_CONFIG
+)
+def delete_cluster_host_config(
+ cluster_id, host_id, user=None, session=None
+):
+ """Delete a clusterhost config by cluster id and host id."""
+ clusterhost = _get_cluster_host(
+ cluster_id, host_id, session=session
+ )
+ return _delete_clusterhost_config(
+ clusterhost, session=session, user=user,
+ os_config={}, package_config={}
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEL_CLUSTERHOST_CONFIG
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS)
+def delete_clusterhost_config(clusterhost_id, user=None, session=None):
+ """Delet a clusterhost config by clusterhost id."""
+ clusterhost = _get_clusterhost(
+ clusterhost_id, session=session
+ )
+ return _delete_clusterhost_config(
+ clusterhost, session=session, user=user,
+ os_config={}, package_config={}
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=['add_hosts', 'remove_hosts', 'set_hosts']
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_CLUSTER_HOSTS
+)
+@utils.wrap_to_dict(
+ ['hosts'],
+ hosts=RESP_CLUSTERHOST_FIELDS
+)
+def update_cluster_hosts(
+ cluster_id, add_hosts={}, set_hosts=None,
+ remove_hosts={}, user=None, session=None
+):
+ """Update cluster hosts."""
+ cluster = _get_cluster(cluster_id, session=session)
+ if remove_hosts:
+ _remove_clusterhosts(
+ cluster, session=session, user=user, **remove_hosts
+ )
+ if add_hosts:
+ _add_clusterhosts(
+ cluster, session=session, user=user, **add_hosts
+ )
+ if set_hosts is not None:
+ _set_clusterhosts(
+ cluster, session=session, user=user, **set_hosts
+ )
+
+ return {
+ 'hosts': list_cluster_hosts(cluster_id, session=session)
+ }
+
+
+def validate_clusterhost(clusterhost, session=None):
+ """validate clusterhost."""
+ roles = clusterhost.roles
+ if not roles:
+ if clusterhost.cluster.flavor_name:
+ raise exception.InvalidParameter(
+ 'empty roles for clusterhost %s' % clusterhost.name
+ )
+
+
+def validate_cluster(cluster, session=None):
+ """Validate cluster."""
+ if not cluster.clusterhosts:
+ raise exception.InvalidParameter(
+ 'cluster %s does not have any hosts' % cluster.name
+ )
+ if cluster.flavor_name:
+ cluster_roles = cluster.flavor['roles']
+ else:
+ cluster_roles = []
+ necessary_roles = set([
+ role['name'] for role in cluster_roles if not role.get('optional')
+ ])
+ clusterhost_roles = set([])
+ interface_subnets = {}
+ for clusterhost in cluster.clusterhosts:
+ roles = clusterhost.roles
+ for role in roles:
+ clusterhost_roles.add(role['name'])
+ host = clusterhost.host
+ for host_network in host.host_networks:
+ interface_subnets.setdefault(
+ host_network.interface, set([])
+ ).add(host_network.subnet.subnet)
+ missing_roles = necessary_roles - clusterhost_roles
+ if missing_roles:
+ raise exception.InvalidParameter(
+ 'cluster %s have some roles %s not assigned to any host' % (
+ cluster.name, list(missing_roles)
+ )
+ )
+ for interface, subnets in interface_subnets.items():
+ if len(subnets) > 1:
+ raise exception.InvalidParameter(
+ 'cluster %s multi subnets %s in interface %s' % (
+ cluster.name, list(subnets), interface
+ )
+ )
+
+
+@utils.supported_filters(optional_support_keys=['review'])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_REVIEW_CLUSTER
+)
+@utils.wrap_to_dict(
+ RESP_REVIEW_FIELDS,
+ cluster=RESP_CONFIG_FIELDS,
+ hosts=RESP_CLUSTERHOST_CONFIG_FIELDS
+)
+def review_cluster(cluster_id, review={}, user=None, session=None, **kwargs):
+ """review cluster.
+
+ Args:
+ cluster_id: the cluster id.
+ review: dict contains hosts to be reviewed. either contains key
+ hosts or clusterhosts. where hosts is a list of host id,
+ clusterhosts is a list of clusterhost id.
+ """
+ from compass.db.api import host as host_api
+ cluster = _get_cluster(cluster_id, session=session)
+ check_cluster_editable(cluster, user=user)
+ host_ids = review.get('hosts', [])
+ clusterhost_ids = review.get('clusterhosts', [])
+ clusterhosts = []
+ # Get clusterhosts need to be reviewed.
+ for clusterhost in cluster.clusterhosts:
+ if (
+ clusterhost.clusterhost_id in clusterhost_ids or
+ clusterhost.host_id in host_ids
+ ):
+ clusterhosts.append(clusterhost)
+
+ os_config = copy.deepcopy(cluster.os_config)
+ os_config = metadata_api.autofill_os_config(
+ os_config, cluster.os_id, cluster=cluster
+ )
+ metadata_api.validate_os_config(
+ os_config, cluster.os_id, True
+ )
+ for clusterhost in clusterhosts:
+ host = clusterhost.host
+ # ignore underlying host os config validation
+ # since the host is not editable
+ if not host_api.is_host_editable(
+ host, user=user, check_in_installing=False
+ ):
+ logging.info(
+ 'ignore update host %s config '
+ 'since it is not editable' % host.name
+ )
+ continue
+ host_os_config = copy.deepcopy(host.os_config)
+ host_os_config = metadata_api.autofill_os_config(
+ host_os_config, host.os_id,
+ host=host
+ )
+ deployed_os_config = util.merge_dict(
+ os_config, host_os_config
+ )
+ metadata_api.validate_os_config(
+ deployed_os_config, host.os_id, True
+ )
+ host_api.validate_host(host)
+ utils.update_db_object(
+ session, host, os_config=host_os_config, config_validated=True
+ )
+
+ package_config = copy.deepcopy(cluster.package_config)
+ if cluster.flavor_name:
+ package_config = metadata_api.autofill_flavor_config(
+ package_config, cluster.flavor_id,
+ cluster=cluster
+ )
+ metadata_api.validate_flavor_config(
+ package_config, cluster.flavor_id, True
+ )
+ for clusterhost in clusterhosts:
+ clusterhost_package_config = copy.deepcopy(
+ clusterhost.package_config
+ )
+ clusterhost_package_config = (
+ metadata_api.autofill_flavor_config(
+ clusterhost_package_config,
+ cluster.flavor_id,
+ clusterhost=clusterhost
+ )
+ )
+ deployed_package_config = util.merge_dict(
+ package_config, clusterhost_package_config
+ )
+ metadata_api.validate_flavor_config(
+ deployed_package_config,
+ cluster.flavor_id, True
+ )
+ validate_clusterhost(clusterhost, session=session)
+ utils.update_db_object(
+ session, clusterhost,
+ package_config=clusterhost_package_config,
+ config_validated=True
+ )
+
+ validate_cluster(cluster, session=session)
+ utils.update_db_object(
+ session, cluster, os_config=os_config, package_config=package_config,
+ config_validated=True
+ )
+ return {
+ 'cluster': cluster,
+ 'hosts': clusterhosts
+ }
+
+
+@utils.supported_filters(optional_support_keys=['deploy'])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEPLOY_CLUSTER
+)
+@utils.wrap_to_dict(
+ RESP_DEPLOY_FIELDS,
+ cluster=RESP_CONFIG_FIELDS,
+ hosts=RESP_CLUSTERHOST_FIELDS
+)
+def deploy_cluster(
+ cluster_id, deploy={}, user=None, session=None, **kwargs
+):
+ """deploy cluster.
+
+ Args:
+ cluster_id: cluster id.
+ deploy: dict contains key either hosts or clusterhosts.
+ deploy['hosts'] is a list of host id,
+ deploy['clusterhosts'] is a list of clusterhost id.
+ """
+ from compass.db.api import host as host_api
+ from compass.tasks import client as celery_client
+ cluster = _get_cluster(cluster_id, session=session)
+ host_ids = deploy.get('hosts', [])
+ clusterhost_ids = deploy.get('clusterhosts', [])
+ clusterhosts = []
+ # get clusterhost to deploy.
+ for clusterhost in cluster.clusterhosts:
+ if (
+ clusterhost.clusterhost_id in clusterhost_ids or
+ clusterhost.host_id in host_ids
+ ):
+ clusterhosts.append(clusterhost)
+ check_cluster_editable(cluster, user=user)
+ check_cluster_validated(cluster)
+ utils.update_db_object(session, cluster.state, state='INITIALIZED')
+ for clusterhost in clusterhosts:
+ host = clusterhost.host
+ # ignore checking if underlying host is validated if
+ # the host is not editable.
+ if host_api.is_host_editable(host, user=user):
+ host_api.check_host_validated(host)
+ utils.update_db_object(session, host.state, state='INITIALIZED')
+ if cluster.flavor_name:
+ check_clusterhost_validated(clusterhost)
+ utils.update_db_object(
+ session, clusterhost.state, state='INITIALIZED'
+ )
+
+ celery_client.celery.send_task(
+ 'compass.tasks.deploy_cluster',
+ (
+ user.email, cluster_id,
+ [clusterhost.host_id for clusterhost in clusterhosts]
+ ),
+ queue=user.email,
+ exchange=user.email,
+ routing_key=user.email
+ )
+ return {
+ 'status': 'deploy action sent',
+ 'cluster': cluster,
+ 'hosts': clusterhosts
+ }
+
+
+@utils.supported_filters(optional_support_keys=['redeploy'])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEPLOY_CLUSTER
+)
+@utils.wrap_to_dict(
+ RESP_DEPLOY_FIELDS,
+ cluster=RESP_CONFIG_FIELDS,
+ hosts=RESP_CLUSTERHOST_FIELDS
+)
+def redeploy_cluster(
+ cluster_id, deploy={}, user=None, session=None, **kwargs
+):
+ """redeploy cluster.
+
+ Args:
+ cluster_id: cluster id.
+ """
+ from compass.db.api import host as host_api
+ from compass.tasks import client as celery_client
+ cluster = _get_cluster(cluster_id, session=session)
+
+ check_cluster_editable(cluster, user=user)
+ check_cluster_validated(cluster)
+ utils.update_db_object(
+ session, cluster.state,
+ state='INITIALIZED',
+ percentage=0,
+ ready=False
+ )
+ for clusterhost in cluster.clusterhosts:
+ host = clusterhost.host
+ # ignore checking if underlying host is validated if
+ # the host is not editable.
+ host_api.check_host_validated(host)
+ utils.update_db_object(
+ session, host.state,
+ state='INITIALIZED',
+ percentage=0,
+ ready=False
+ )
+ if cluster.flavor_name:
+ check_clusterhost_validated(clusterhost)
+ utils.update_db_object(
+ session,
+ clusterhost.state,
+ state='INITIALIZED',
+ percentage=0,
+ ready=False
+ )
+
+ celery_client.celery.send_task(
+ 'compass.tasks.redeploy_cluster',
+ (
+ user.email, cluster_id
+ ),
+ queue=user.email,
+ exchange=user.email,
+ routing_key=user.email
+ )
+ return {
+ 'status': 'redeploy action sent',
+ 'cluster': cluster
+ }
+
+
+@utils.supported_filters(optional_support_keys=['apply_patch'])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEPLOY_CLUSTER
+)
+@utils.wrap_to_dict(
+ RESP_DEPLOY_FIELDS,
+ cluster=RESP_CONFIG_FIELDS,
+ hosts=RESP_CLUSTERHOST_FIELDS
+)
+def patch_cluster(cluster_id, user=None, session=None, **kwargs):
+
+ from compass.tasks import client as celery_client
+
+ cluster = _get_cluster(cluster_id, session=session)
+ celery_client.celery.send_task(
+ 'compass.tasks.patch_cluster',
+ (
+ user.email, cluster_id,
+ ),
+ queue=user.email,
+ exchange=user.email,
+ routing_key=user.email
+ )
+ return {
+ 'status': 'patch action sent',
+ 'cluster': cluster
+ }
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_GET_CLUSTER_STATE
+)
+@utils.wrap_to_dict(RESP_STATE_FIELDS)
+def get_cluster_state(cluster_id, user=None, session=None, **kwargs):
+ """Get cluster state info."""
+ return _get_cluster(cluster_id, session=session).state_dict()
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_GET_CLUSTERHOST_STATE
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS)
+def get_cluster_host_state(
+ cluster_id, host_id, user=None, session=None, **kwargs
+):
+ """Get clusterhost state merged with underlying host state."""
+ return _get_cluster_host(
+ cluster_id, host_id, session=session
+ ).state_dict()
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_GET_CLUSTERHOST_STATE
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS)
+def get_cluster_host_self_state(
+ cluster_id, host_id, user=None, session=None, **kwargs
+):
+ """Get clusterhost itself state."""
+ return _get_cluster_host(
+ cluster_id, host_id, session=session
+ ).state
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_GET_CLUSTERHOST_STATE
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS)
+def get_clusterhost_state(
+ clusterhost_id, user=None, session=None, **kwargs
+):
+ """Get clusterhost state merged with underlying host state."""
+ return _get_clusterhost(
+ clusterhost_id, session=session
+ ).state_dict()
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_GET_CLUSTERHOST_STATE
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS)
+def get_clusterhost_self_state(
+ clusterhost_id, user=None, session=None, **kwargs
+):
+ """Get clusterhost itself state."""
+ return _get_clusterhost(
+ clusterhost_id, session=session
+ ).state
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_CLUSTERHOST_STATE_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_CLUSTERHOST_STATE
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS)
+def update_cluster_host_state(
+ cluster_id, host_id, user=None, session=None, **kwargs
+):
+ """Update a clusterhost itself state."""
+ clusterhost = _get_cluster_host(
+ cluster_id, host_id, session=session
+ )
+ # Modify(harry): without progress_update.py to update cluster state
+ # update cluster state here
+ cluster = _get_cluster(clusterhost.cluster_id, session=session)
+ utils.update_db_object(session, clusterhost.state, **kwargs)
+ utils.update_db_object(session, cluster.state, **kwargs)
+ return clusterhost.state_dict()
+
+
+def _update_clusterhost_state(
+ clusterhost, from_database_only=False,
+ session=None, user=None, **kwargs
+):
+ """Update clusterhost state.
+
+ If from_database_only, the state will only be updated in database.
+ Otherwise a task sent to celery and os installer/package installer
+ will also update its state if needed.
+ """
+ if 'ready' in kwargs and kwargs['ready'] and not clusterhost.state.ready:
+ ready_triggered = True
+ else:
+ ready_triggered = False
+ cluster_ready = False
+ host = clusterhost.host
+ cluster = clusterhost.cluster
+ host_ready = not host.state.ready
+ if ready_triggered:
+ cluster_ready = True
+ for clusterhost_in_cluster in cluster.clusterhosts:
+ if (
+ clusterhost_in_cluster.clusterhost_id
+ == clusterhost.clusterhost_id
+ ):
+ continue
+ if not clusterhost_in_cluster.state.ready:
+ cluster_ready = False
+
+ logging.info(
+ 'clusterhost %s ready: %s',
+ clusterhost.name, ready_triggered
+ )
+ logging.info('cluster ready: %s', cluster_ready)
+ logging.info('host ready: %s', host_ready)
+ if not ready_triggered or from_database_only:
+ logging.info('%s state is set to %s', clusterhost.name, kwargs)
+ utils.update_db_object(session, clusterhost.state, **kwargs)
+ if not clusterhost.state.ready:
+ logging.info('%s state ready is set to False', cluster.name)
+ utils.update_db_object(session, cluster.state, ready=False)
+ status = '%s state is updated' % clusterhost.name
+ else:
+ if not user:
+ user_id = cluster.creator_id
+ user_dict = user_api.get_user(user_id, session=session)
+ user_email = user_dict['email']
+ else:
+ user_email = user.email
+ from compass.tasks import client as celery_client
+ celery_client.celery.send_task(
+ 'compass.tasks.package_installed',
+ (
+ clusterhost.cluster_id, clusterhost.host_id,
+ cluster_ready, host_ready
+ ),
+ queue=user_email,
+ exchange=user_email,
+ routing_key=user_email
+ )
+ status = '%s: cluster ready %s host ready %s' % (
+ clusterhost.name, cluster_ready, host_ready
+ )
+ logging.info('action status: %s', status)
+ return {
+ 'status': status,
+ 'clusterhost': clusterhost.state_dict()
+ }
+
+
+@util.deprecated
+@utils.supported_filters(
+ optional_support_keys=UPDATED_CLUSTERHOST_STATE_INTERNAL_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_CLUSTERHOST_STATE
+)
+@utils.wrap_to_dict(['status', 'clusterhost'])
+def update_cluster_host_state_internal(
+ cluster_id, host_id, from_database_only=False,
+ user=None, session=None, **kwargs
+):
+ """Update a clusterhost state by installation process."""
+ # TODO(xicheng): it should be merged into update_cluster_host_state
+ clusterhost = _get_cluster_host(
+ cluster_id, host_id, session=session
+ )
+ return _update_clusterhost_state(
+ clusterhost, from_database_only=from_database_only,
+ session=session, users=user, **kwargs
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_CLUSTERHOST_STATE_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_CLUSTERHOST_STATE
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS)
+def update_clusterhost_state(
+ clusterhost_id, user=None, session=None, **kwargs
+):
+ """Update a clusterhost itself state."""
+ clusterhost = _get_clusterhost(
+ clusterhost_id, session=session
+ )
+ # Modify(harry): without progress_update.py to update cluster state
+ # update cluster state here
+ cluster = _get_cluster(clusterhost.cluster_id, session=session)
+ utils.update_db_object(session, clusterhost.state, **kwargs)
+ utils.update_db_object(session, cluster.state, **kwargs)
+ return clusterhost.state_dict()
+
+
+@util.deprecated
+@utils.supported_filters(
+ optional_support_keys=UPDATED_CLUSTERHOST_STATE_INTERNAL_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_CLUSTERHOST_STATE
+)
+@utils.wrap_to_dict(['status', 'clusterhost'])
+def update_clusterhost_state_internal(
+ clusterhost_id, from_database_only=False,
+ user=None, session=None, **kwargs
+):
+ """Update a clusterhost state by installation process."""
+ # TODO(xicheng): it should be merged into update_clusterhost_state
+ clusterhost = _get_clusterhost(clusterhost_id, session=session)
+ return _update_clusterhost_state(
+ clusterhost, from_database_only=from_database_only,
+ session=session, user=user, **kwargs
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_CLUSTER_STATE_FIELDS,
+ ignore_support_keys=(IGNORE_FIELDS + IGNORE_UPDATED_CLUSTER_STATE_FIELDS)
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_CLUSTER_STATE
+)
+@utils.wrap_to_dict(RESP_STATE_FIELDS)
+def update_cluster_state(
+ cluster_id, user=None, session=None, **kwargs
+):
+ """Update a cluster state."""
+ cluster = _get_cluster(
+ cluster_id, session=session
+ )
+ utils.update_db_object(session, cluster.state, **kwargs)
+ return cluster.state_dict()
+
+
+@util.deprecated
+@utils.supported_filters(
+ optional_support_keys=UPDATED_CLUSTER_STATE_INTERNAL_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_CLUSTER_STATE
+)
+@utils.wrap_to_dict(['status', 'cluster'])
+def update_cluster_state_internal(
+ cluster_id, from_database_only=False,
+ user=None, session=None, **kwargs
+):
+ """Update a cluster state by installation process.
+
+ If from_database_only, the state will only be updated in database.
+ Otherwise a task sent to do state update in os installer and
+ package installer.
+ """
+ # TODO(xicheng): it should be merged into update_cluster_state
+ cluster = _get_cluster(cluster_id, session=session)
+ if 'ready' in kwargs and kwargs['ready'] and not cluster.state.ready:
+ ready_triggered = True
+ else:
+ ready_triggered = False
+ clusterhost_ready = {}
+ if ready_triggered:
+ for clusterhost in cluster.clusterhosts:
+ clusterhost_ready[clusterhost.host_id] = (
+ not clusterhost.state.ready
+ )
+
+ logging.info('cluster %s ready: %s', cluster_id, ready_triggered)
+ logging.info('clusterhost ready: %s', clusterhost_ready)
+
+ if not ready_triggered or from_database_only:
+ logging.info('%s state is set to %s', cluster.name, kwargs)
+ utils.update_db_object(session, cluster.state, **kwargs)
+ if not cluster.state.ready:
+ for clusterhost in cluster.clusterhosts:
+ logging.info('%s state ready is to False', clusterhost.name)
+ utils.update_db_object(
+ session, clusterhost.state, ready=False
+ )
+ status = '%s state is updated' % cluster.name
+ else:
+ if not user:
+ user_id = cluster.creator_id
+ user_dict = user_api.get_user(user_id, session=session)
+ user_email = user_dict['email']
+ else:
+ user_email = user.email
+ from compass.tasks import client as celery_client
+ celery_client.celery.send_task(
+ 'compass.tasks.cluster_installed',
+ (clusterhost.cluster_id, clusterhost_ready),
+ queue=user_email,
+ exchange=user_email,
+ routing_key=user_email
+ )
+ status = '%s installed action set clusterhost ready %s' % (
+ cluster.name, clusterhost_ready
+ )
+ logging.info('action status: %s', status)
+ return {
+ 'status': status,
+ 'cluster': cluster.state_dict()
+ }
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS)
+def get_cluster_host_log_histories(
+ cluster_id, host_id, user=None, session=None, **kwargs
+):
+ """Get clusterhost log history by cluster id and host id."""
+ return _get_cluster_host(
+ cluster_id, host_id, session=session
+ ).log_histories
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS)
+def get_clusterhost_log_histories(
+ clusterhost_id, user=None,
+ session=None, **kwargs
+):
+ """Get clusterhost log history by clusterhost id."""
+ return _get_clusterhost(
+ clusterhost_id, session=session
+ ).log_histories
+
+
+def _get_cluster_host_log_history(
+ cluster_id, host_id, filename, session=None, **kwargs
+):
+ """Get clusterhost log history by cluster id, host id and filename."""
+ clusterhost = _get_cluster_host(cluster_id, host_id, session=session)
+ return utils.get_db_object(
+ session, models.ClusterHostLogHistory,
+ clusterhost_id=clusterhost.clusterhost_id, filename=filename,
+ **kwargs
+ )
+
+
+def _get_clusterhost_log_history(
+ clusterhost_id, filename, session=None, **kwargs
+):
+ """Get clusterhost log history by clusterhost id and filename."""
+ clusterhost = _get_clusterhost(clusterhost_id, session=session)
+ return utils.get_db_object(
+ session, models.ClusterHostLogHistory,
+ clusterhost_id=clusterhost.clusterhost_id, filename=filename,
+ **kwargs
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS)
+def get_cluster_host_log_history(
+ cluster_id, host_id, filename, user=None, session=None, **kwargs
+):
+ """Get clusterhost log history by cluster id, host id and filename."""
+ return _get_cluster_host_log_history(
+ cluster_id, host_id, filename, session=session
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS)
+def get_clusterhost_log_history(
+ clusterhost_id, filename, user=None, session=None, **kwargs
+):
+ """Get host log history by clusterhost id and filename."""
+ return _get_clusterhost_log_history(
+ clusterhost_id, filename, session=session
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS)
+def update_cluster_host_log_history(
+ cluster_id, host_id, filename, user=None, session=None, **kwargs
+):
+ """Update a host log history by cluster id, host id and filename."""
+ cluster_host_log_history = _get_cluster_host_log_history(
+ cluster_id, host_id, filename, session=session
+ )
+ return utils.update_db_object(
+ session, cluster_host_log_history, **kwargs
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS)
+def update_clusterhost_log_history(
+ clusterhost_id, filename, user=None, session=None, **kwargs
+):
+ """Update a host log history by clusterhost id and filename."""
+ clusterhost_log_history = _get_clusterhost_log_history(
+ clusterhost_id, filename, session=session
+ )
+ return utils.update_db_object(session, clusterhost_log_history, **kwargs)
+
+
+@utils.supported_filters(
+ ADDED_CLUSTERHOST_LOG_FIELDS,
+ optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS)
+def add_clusterhost_log_history(
+ clusterhost_id, exception_when_existing=False,
+ filename=None, user=None, session=None, **kwargs
+):
+ """add a host log history by clusterhost id and filename."""
+ clusterhost = _get_clusterhost(clusterhost_id, session=session)
+ return utils.add_db_object(
+ session, models.ClusterHostLogHistory,
+ exception_when_existing,
+ clusterhost.clusterhost_id, filename, **kwargs
+ )
+
+
+@utils.supported_filters(
+ ADDED_CLUSTERHOST_LOG_FIELDS,
+ optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS)
+def add_cluster_host_log_history(
+ cluster_id, host_id, exception_when_existing=False,
+ filename=None, user=None, session=None, **kwargs
+):
+ """add a host log history by cluster id, host id and filename."""
+ clusterhost = _get_cluster_host(
+ cluster_id, host_id, session=session
+ )
+ return utils.add_db_object(
+ session, models.ClusterHostLogHistory, exception_when_existing,
+ clusterhost.clusterhost_id, filename, **kwargs
+ )
diff --git a/compass-tasks/db/api/database.py b/compass-tasks/db/api/database.py
new file mode 100644
index 0000000..49769d7
--- /dev/null
+++ b/compass-tasks/db/api/database.py
@@ -0,0 +1,264 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provider interface to manipulate database."""
+import functools
+import logging
+import netaddr
+
+from contextlib import contextmanager
+from sqlalchemy import create_engine
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.exc import OperationalError
+from sqlalchemy.orm import scoped_session
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.pool import NullPool
+from sqlalchemy.pool import QueuePool
+from sqlalchemy.pool import SingletonThreadPool
+from sqlalchemy.pool import StaticPool
+from threading import local
+
+from compass.db import exception
+from compass.db import models
+from compass.utils import logsetting
+from compass.utils import setting_wrapper as setting
+
+
+ENGINE = None
+SESSION = sessionmaker(autocommit=False, autoflush=False)
+SCOPED_SESSION = None
+SESSION_HOLDER = local()
+
+POOL_MAPPING = {
+ 'instant': NullPool,
+ 'static': StaticPool,
+ 'queued': QueuePool,
+ 'thread_single': SingletonThreadPool
+}
+
+
+def init(database_url=None):
+ """Initialize database.
+
+ Adjust sqlalchemy logging if necessary.
+
+ :param database_url: string, database url.
+ """
+ global ENGINE
+ global SCOPED_SESSION
+ if not database_url:
+ database_url = setting.SQLALCHEMY_DATABASE_URI
+ logging.info('init database %s', database_url)
+ root_logger = logging.getLogger()
+ fine_debug = root_logger.isEnabledFor(logsetting.LOGLEVEL_MAPPING['fine'])
+ if fine_debug:
+ logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
+ finest_debug = root_logger.isEnabledFor(
+ logsetting.LOGLEVEL_MAPPING['finest']
+ )
+ if finest_debug:
+ logging.getLogger('sqlalchemy.dialects').setLevel(logging.INFO)
+ logging.getLogger('sqlalchemy.pool').setLevel(logging.INFO)
+ logging.getLogger('sqlalchemy.orm').setLevel(logging.INFO)
+ poolclass = POOL_MAPPING[setting.SQLALCHEMY_DATABASE_POOL_TYPE]
+ ENGINE = create_engine(
+ database_url, convert_unicode=True,
+ poolclass=poolclass
+ )
+ SESSION.configure(bind=ENGINE)
+ SCOPED_SESSION = scoped_session(SESSION)
+ models.BASE.query = SCOPED_SESSION.query_property()
+
+
+def in_session():
+ """check if in database session scope."""
+ bool(hasattr(SESSION_HOLDER, 'session'))
+
+
+@contextmanager
+def session(exception_when_in_session=True):
+ """database session scope.
+
+ To operate database, it should be called in database session.
+ If not exception_when_in_session, the with session statement support
+ nested session and only the out most session commit/rollback the
+ transaction.
+ """
+ if not ENGINE:
+ init()
+
+ nested_session = False
+ if hasattr(SESSION_HOLDER, 'session'):
+ if exception_when_in_session:
+ logging.error('we are already in session')
+ raise exception.DatabaseException('session already exist')
+ else:
+ new_session = SESSION_HOLDER.session
+ nested_session = True
+ logging.log(
+ logsetting.getLevelByName('fine'),
+ 'reuse session %s', nested_session
+ )
+ else:
+ new_session = SCOPED_SESSION()
+ setattr(SESSION_HOLDER, 'session', new_session)
+ logging.log(
+ logsetting.getLevelByName('fine'),
+ 'enter session %s', new_session
+ )
+ try:
+ yield new_session
+ if not nested_session:
+ new_session.commit()
+ except Exception as error:
+ if not nested_session:
+ new_session.rollback()
+ logging.error('failed to commit session')
+ logging.exception(error)
+ if isinstance(error, IntegrityError):
+ for item in error.statement.split():
+ if item.islower():
+ object = item
+ break
+ raise exception.DuplicatedRecord(
+ '%s in %s' % (error.orig, object)
+ )
+ elif isinstance(error, OperationalError):
+ raise exception.DatabaseException(
+ 'operation error in database'
+ )
+ elif isinstance(error, exception.DatabaseException):
+ raise error
+ else:
+ raise exception.DatabaseException(str(error))
+ finally:
+ if not nested_session:
+ new_session.close()
+ SCOPED_SESSION.remove()
+ delattr(SESSION_HOLDER, 'session')
+ logging.log(
+ logsetting.getLevelByName('fine'),
+ 'exit session %s', new_session
+ )
+
+
+def current_session():
+ """Get the current session scope when it is called.
+
+ :return: database session.
+ :raises: DatabaseException when it is not in session.
+ """
+ try:
+ return SESSION_HOLDER.session
+ except Exception as error:
+ logging.error('It is not in the session scope')
+ logging.exception(error)
+ if isinstance(error, exception.DatabaseException):
+ raise error
+ else:
+ raise exception.DatabaseException(str(error))
+
+
+def run_in_session(exception_when_in_session=True):
+ """Decorator to make sure the decorated function run in session.
+
+ When not exception_when_in_session, the run_in_session can be
+ decorated several times.
+ """
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ try:
+ my_session = kwargs.get('session')
+ if my_session is not None:
+ return func(*args, **kwargs)
+ else:
+ with session(
+ exception_when_in_session=exception_when_in_session
+ ) as my_session:
+ kwargs['session'] = my_session
+ return func(*args, **kwargs)
+ except Exception as error:
+ logging.error(
+ 'got exception with func %s args %s kwargs %s',
+ func, args, kwargs
+ )
+ logging.exception(error)
+ raise error
+ return wrapper
+ return decorator
+
+
+def _setup_user_table(user_session):
+ """Initialize user table with default user."""
+ logging.info('setup user table')
+ from compass.db.api import user
+ user.add_user(
+ session=user_session,
+ email=setting.COMPASS_ADMIN_EMAIL,
+ password=setting.COMPASS_ADMIN_PASSWORD,
+ is_admin=True
+ )
+
+
+def _setup_permission_table(permission_session):
+ """Initialize permission table."""
+ logging.info('setup permission table.')
+ from compass.db.api import permission
+ permission.add_permissions_internal(
+ session=permission_session
+ )
+
+
+def _setup_switch_table(switch_session):
+ """Initialize switch table."""
+ # TODO(xicheng): deprecate setup default switch.
+ logging.info('setup switch table')
+ from compass.db.api import switch
+ switch.add_switch(
+ True, setting.DEFAULT_SWITCH_IP,
+ session=switch_session,
+ machine_filters=['allow ports all']
+ )
+
+
+def _update_others(other_session):
+ """Update other tables."""
+ logging.info('update other tables')
+ from compass.db.api import utils
+ from compass.db import models
+ utils.update_db_objects(
+ other_session, models.Cluster
+ )
+ utils.update_db_objects(
+ other_session, models.Host
+ )
+ utils.update_db_objects(
+ other_session, models.ClusterHost
+ )
+
+
+@run_in_session()
+def create_db(session=None):
+ """Create database."""
+ models.BASE.metadata.create_all(bind=ENGINE)
+ _setup_permission_table(session)
+ _setup_user_table(session)
+ _setup_switch_table(session)
+ _update_others(session)
+
+
+def drop_db():
+ """Drop database."""
+ models.BASE.metadata.drop_all(bind=ENGINE)
diff --git a/compass-tasks/db/api/health_check_report.py b/compass-tasks/db/api/health_check_report.py
new file mode 100644
index 0000000..aaea7a7
--- /dev/null
+++ b/compass-tasks/db/api/health_check_report.py
@@ -0,0 +1,190 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Cluster health check report."""
+import logging
+
+from compass.db.api import cluster as cluster_api
+from compass.db.api import database
+from compass.db.api import host as host_api
+from compass.db.api import permission
+from compass.db.api import user as user_api
+from compass.db.api import utils
+from compass.db import exception
+from compass.db import models
+
+
+REQUIRED_INSERT_FIELDS = ['name']
+OPTIONAL_INSERT_FIELDS = [
+ 'display_name', 'report', 'category', 'state', 'error_message'
+]
+UPDATE_FIELDS = ['report', 'state', 'error_message']
+RESP_FIELDS = [
+ 'cluster_id', 'name', 'display_name', 'report',
+ 'category', 'state', 'error_message'
+]
+RESP_ACTION_FIELDS = ['cluster_id', 'status']
+
+
+@utils.supported_filters(REQUIRED_INSERT_FIELDS, OPTIONAL_INSERT_FIELDS)
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_FIELDS)
+def add_report_record(cluster_id, name=None, report={},
+ state='verifying', session=None, **kwargs):
+ """Create a health check report record."""
+ # Replace any white space into '-'
+ words = name.split()
+ name = '-'.join(words)
+ cluster = cluster_api.get_cluster_internal(cluster_id, session=session)
+ return utils.add_db_object(
+ session, models.HealthCheckReport, True, cluster.id, name,
+ report=report, state=state, **kwargs
+ )
+
+
+def _get_report(cluster_id, name, session=None):
+ cluster = cluster_api.get_cluster_internal(cluster_id, session=session)
+ return utils.get_db_object(
+ session, models.HealthCheckReport, cluster_id=cluster.id, name=name
+ )
+
+
+@utils.supported_filters(UPDATE_FIELDS)
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_FIELDS)
+def update_report(cluster_id, name, session=None, **kwargs):
+ """Update health check report."""
+ report = _get_report(cluster_id, name, session=session)
+ if report.state == 'finished':
+ err_msg = 'Report cannot be updated if state is in "finished"'
+ raise exception.Forbidden(err_msg)
+
+ return utils.update_db_object(session, report, **kwargs)
+
+
+@utils.supported_filters(UPDATE_FIELDS)
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_FIELDS)
+def update_multi_reports(cluster_id, session=None, **kwargs):
+ """Bulk update reports."""
+ # TODO(grace): rename the fuction if needed to reflect the fact.
+ return set_error(cluster_id, session=session, **kwargs)
+
+
+def set_error(cluster_id, report={}, session=None,
+ state='error', error_message=None):
+ cluster = cluster_api.get_cluster_internal(cluster_id, session=session)
+ logging.debug(
+ "updates all reports as %s in cluster %s",
+ state, cluster_id
+ )
+ return utils.update_db_objects(
+ session, models.HealthCheckReport,
+ updates={
+ 'report': {},
+ 'state': 'error',
+ 'error_message': error_message
+ }, cluster_id=cluster.id
+ )
+
+
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_HEALTH_REPORT
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def list_health_reports(cluster_id, user=None, session=None):
+ """List all reports in the specified cluster."""
+ cluster = cluster_api.get_cluster_internal(cluster_id, session=session)
+ return utils.list_db_objects(
+ session, models.HealthCheckReport, cluster_id=cluster.id
+ )
+
+
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_GET_HEALTH_REPORT
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def get_health_report(cluster_id, name, user=None, session=None):
+ return _get_report(
+ cluster_id, name, session=session
+ )
+
+
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DELETE_REPORT
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def delete_reports(cluster_id, name=None, user=None, session=None):
+ # TODO(grace): better to separate this function into two.
+ # One is to delete a report of a cluster, the other to delete all
+ # reports under a cluster.
+ if name:
+ report = _get_report(cluster_id, name, session=session)
+ return utils.del_db_object(session, report)
+ else:
+ cluster = cluster_api.get_cluster_internal(
+ cluster_id, session=session
+ )
+ return utils.del_db_objects(
+ session, models.HealthCheckReport, cluster_id=cluster.id
+ )
+
+
+@utils.supported_filters(optional_support_keys=['check_health'])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_CHECK_CLUSTER_HEALTH
+)
+@utils.wrap_to_dict(RESP_ACTION_FIELDS)
+def start_check_cluster_health(cluster_id, send_report_url,
+ user=None, session=None, check_health={}):
+ """Start to check cluster health."""
+ cluster = cluster_api.get_cluster_internal(cluster_id, session=session)
+
+ if cluster.state.state != 'SUCCESSFUL':
+ logging.debug("state is %s" % cluster.state.state)
+ err_msg = "Healthcheck starts only after cluster finished deployment!"
+ raise exception.Forbidden(err_msg)
+
+ reports = utils.list_db_objects(
+ session, models.HealthCheckReport,
+ cluster_id=cluster.id, state='verifying'
+ )
+ if reports:
+ err_msg = 'Healthcheck in progress, please wait for it to complete!'
+ raise exception.Forbidden(err_msg)
+
+ # Clear all preivous report
+ # TODO(grace): the delete should be moved into celery task.
+ # We should consider the case that celery task is down.
+ utils.del_db_objects(
+ session, models.HealthCheckReport, cluster_id=cluster.id
+ )
+
+ from compass.tasks import client as celery_client
+ celery_client.celery.send_task(
+ 'compass.tasks.cluster_health',
+ (cluster.id, send_report_url, user.email),
+ queue=user.email,
+ exchange=user.email,
+ routing_key=user.email
+ )
+ return {
+ "cluster_id": cluster.id,
+ "status": "start to check cluster health."
+ }
diff --git a/compass-tasks/db/api/host.py b/compass-tasks/db/api/host.py
new file mode 100644
index 0000000..15e0bb6
--- /dev/null
+++ b/compass-tasks/db/api/host.py
@@ -0,0 +1,1120 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Host database operations."""
+import functools
+import logging
+import netaddr
+import re
+
+from compass.db.api import database
+from compass.db.api import metadata_holder as metadata_api
+from compass.db.api import permission
+from compass.db.api import user as user_api
+from compass.db.api import utils
+from compass.db import exception
+from compass.db import models
+from compass.utils import util
+
+
+SUPPORTED_FIELDS = ['name', 'os_name', 'owner', 'mac', 'id']
+SUPPORTED_MACHINE_HOST_FIELDS = [
+ 'mac', 'tag', 'location', 'os_name', 'os_id'
+]
+SUPPORTED_NETOWORK_FIELDS = [
+ 'interface', 'ip', 'is_mgmt', 'is_promiscuous'
+]
+RESP_FIELDS = [
+ 'id', 'name', 'hostname', 'os_name', 'owner', 'mac',
+ 'switch_ip', 'port', 'switches', 'os_installer', 'os_id', 'ip',
+ 'reinstall_os', 'os_installed', 'tag', 'location', 'networks',
+ 'created_at', 'updated_at'
+]
+RESP_CLUSTER_FIELDS = [
+ 'id', 'name', 'os_name', 'reinstall_distributed_system',
+ 'owner', 'adapter_name', 'flavor_name',
+ 'distributed_system_installed', 'created_at', 'updated_at'
+]
+RESP_NETWORK_FIELDS = [
+ 'id', 'ip', 'interface', 'netmask', 'is_mgmt', 'is_promiscuous',
+ 'created_at', 'updated_at'
+]
+RESP_CONFIG_FIELDS = [
+ 'os_config',
+ 'config_setp',
+ 'config_validated',
+ 'networks',
+ 'created_at',
+ 'updated_at'
+]
+RESP_DEPLOYED_CONFIG_FIELDS = [
+ 'deployed_os_config'
+]
+RESP_DEPLOY_FIELDS = [
+ 'status', 'host'
+]
+UPDATED_FIELDS = ['name', 'reinstall_os']
+UPDATED_CONFIG_FIELDS = [
+ 'put_os_config'
+]
+PATCHED_CONFIG_FIELDS = [
+ 'patched_os_config'
+]
+UPDATED_DEPLOYED_CONFIG_FIELDS = [
+ 'deployed_os_config'
+]
+ADDED_NETWORK_FIELDS = [
+ 'interface', 'ip', 'subnet_id'
+]
+OPTIONAL_ADDED_NETWORK_FIELDS = ['is_mgmt', 'is_promiscuous']
+UPDATED_NETWORK_FIELDS = [
+ 'interface', 'ip', 'subnet_id', 'subnet', 'is_mgmt',
+ 'is_promiscuous'
+]
+IGNORE_FIELDS = [
+ 'id', 'created_at', 'updated_at'
+]
+RESP_STATE_FIELDS = [
+ 'id', 'state', 'percentage', 'message', 'severity', 'ready'
+]
+UPDATED_STATE_FIELDS = [
+ 'state', 'percentage', 'message', 'severity'
+]
+UPDATED_STATE_INTERNAL_FIELDS = [
+ 'ready'
+]
+RESP_LOG_FIELDS = [
+ 'id', 'filename', 'position', 'partial_line', 'percentage',
+ 'message', 'severity', 'line_matcher_name'
+]
+ADDED_LOG_FIELDS = [
+ 'filename'
+]
+UPDATED_LOG_FIELDS = [
+ 'position', 'partial_line', 'percentage',
+ 'message', 'severity', 'line_matcher_name'
+]
+
+
+@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_HOSTS
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def list_hosts(user=None, session=None, **filters):
+ """List hosts."""
+ return utils.list_db_objects(
+ session, models.Host, **filters
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=SUPPORTED_MACHINE_HOST_FIELDS)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_HOSTS
+)
+@utils.output_filters(
+ missing_ok=True,
+ tag=utils.general_filter_callback,
+ location=utils.general_filter_callback,
+ os_name=utils.general_filter_callback,
+ os_id=utils.general_filter_callback
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def list_machines_or_hosts(user=None, session=None, **filters):
+ """List machines or hosts if possible."""
+ machines = utils.list_db_objects(
+ session, models.Machine, **filters
+ )
+ machines_or_hosts = []
+ for machine in machines:
+ host = machine.host
+ if host:
+ machines_or_hosts.append(host)
+ else:
+ machines_or_hosts.append(machine)
+ return machines_or_hosts
+
+
+def _get_host(host_id, session=None, **kwargs):
+ """Get host by id."""
+ if isinstance(host_id, (int, long)):
+ return utils.get_db_object(
+ session, models.Host,
+ id=host_id, **kwargs
+ )
+ else:
+ raise exception.InvalidParameter(
+ 'host id %s type is not int compatible' % host_id
+ )
+
+
+def get_host_internal(host_id, session=None, **kwargs):
+ """Helper function to get host.
+
+ Used by other files under db/api.
+ """
+ return _get_host(host_id, session=session, **kwargs)
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_HOSTS
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def get_host(
+ host_id, exception_when_missing=True,
+ user=None, session=None, **kwargs
+):
+ """get host info."""
+ return _get_host(
+ host_id,
+ exception_when_missing=exception_when_missing,
+ session=session
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_HOSTS
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def get_machine_or_host(
+ host_id, exception_when_missing=True,
+ user=None, session=None, **kwargs
+):
+ """get machine or host if possible."""
+ from compass.db.api import machine as machine_api
+ machine = machine_api.get_machine_internal(
+ host_id,
+ exception_when_missing=exception_when_missing,
+ session=session
+ )
+ if machine.host:
+ return machine.host
+ else:
+ return machine
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_HOST_CLUSTERS
+)
+@utils.wrap_to_dict(RESP_CLUSTER_FIELDS)
+def get_host_clusters(host_id, user=None, session=None, **kwargs):
+ """get host clusters."""
+ host = _get_host(host_id, session=session)
+ return [clusterhost.cluster for clusterhost in host.clusterhosts]
+
+
+def check_host_validated(host):
+ """Check host is validated."""
+ if not host.config_validated:
+ raise exception.Forbidden(
+ 'host %s is not validated' % host.name
+ )
+
+
+def check_host_editable(
+ host, user=None,
+ check_in_installing=False
+):
+ """Check host is editable.
+
+ If we try to set reinstall_os or check the host is not in installing
+ state, we should set check_in_installing to True.
+ Otherwise we will check the host is not in installing or installed.
+ We also make sure the user is admin or the owner of the host to avoid
+ unauthorized user to update host attributes.
+ """
+ if check_in_installing:
+ if host.state.state == 'INSTALLING':
+ raise exception.Forbidden(
+ 'host %s is not editable '
+ 'when state is in installing' % host.name
+ )
+ elif not host.reinstall_os:
+ raise exception.Forbidden(
+ 'host %s is not editable '
+ 'when not to be reinstalled' % host.name
+ )
+ if user and not user.is_admin and host.creator_id != user.id:
+ raise exception.Forbidden(
+ 'host %s is not editable '
+ 'when user is not admin or the owner of the host' % host.name
+ )
+
+
+def is_host_editable(
+ host, user=None,
+ check_in_installing=False
+):
+ """Get if host is editable."""
+ try:
+ check_host_editable(
+ host, user=user,
+ check_in_installing=check_in_installing
+ )
+ return True
+ except exception.Forbidden:
+ return False
+
+
+def validate_host(host):
+ """Validate host.
+
+ Makesure hostname is not empty, there is only one mgmt network,
+ The mgmt network is not in promiscuous mode.
+ """
+ if not host.hostname:
+ raise exception.Invalidparameter(
+ 'host %s does not set hostname' % host.name
+ )
+ if not host.host_networks:
+ raise exception.InvalidParameter(
+ 'host %s does not have any network' % host.name
+ )
+ mgmt_interface_set = False
+ for host_network in host.host_networks:
+ if host_network.is_mgmt:
+ if mgmt_interface_set:
+ raise exception.InvalidParameter(
+ 'host %s multi interfaces set mgmt ' % host.name
+ )
+ if host_network.is_promiscuous:
+ raise exception.InvalidParameter(
+ 'host %s interface %s is mgmt but promiscuous' % (
+ host.name, host_network.interface
+ )
+ )
+ mgmt_interface_set = True
+ if not mgmt_interface_set:
+ raise exception.InvalidParameter(
+ 'host %s has no mgmt interface' % host.name
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(name=utils.check_name)
+@utils.wrap_to_dict(RESP_FIELDS)
+def _update_host(host_id, session=None, user=None, **kwargs):
+ """Update a host internal."""
+ host = _get_host(host_id, session=session)
+ if host.state.state == "SUCCESSFUL" and not host.reinstall_os:
+ logging.info("ignoring successful host: %s", host_id)
+ return {}
+ check_host_editable(
+ host, user=user,
+ check_in_installing=kwargs.get('reinstall_os', False)
+ )
+ return utils.update_db_object(session, host, **kwargs)
+
+
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_HOST
+)
+def update_host(host_id, user=None, session=None, **kwargs):
+ """Update a host."""
+ return _update_host(host_id, session=session, user=user, **kwargs)
+
+
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_HOST
+)
+def update_hosts(data=[], user=None, session=None):
+ """Update hosts."""
+ # TODO(xicheng): this batch function is not similar as others.
+ # try to make it similar output as others and batch update should
+ # tolerate partial failure.
+ hosts = []
+ for host_data in data:
+ hosts.append(_update_host(session=session, user=user, **host_data))
+ return hosts
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEL_HOST
+)
+@utils.wrap_to_dict(
+ RESP_FIELDS + ['status', 'host'],
+ host=RESP_FIELDS
+)
+def del_host(
+ host_id, force=False, from_database_only=False,
+ user=None, session=None, **kwargs
+):
+ """Delete a host.
+
+ If force, we delete the host anyway.
+ If from_database_only, we only delete the host record in databaes.
+ Otherwise we send to del host task to celery to delete the host
+ record in os installer and package installer, clean installation logs
+ and at last clean database record.
+ The backend will call this function again after it deletes the record
+ in os installer and package installer with from_database_only set.
+ """
+ from compass.db.api import cluster as cluster_api
+ host = _get_host(host_id, session=session)
+ # force set host state to ERROR when we want to delete the
+ # host anyway even the host is in installing or already
+ # installed. It let the api know the deleting is in doing when backend
+ # is doing the real deleting. In future we may import a new state like
+ # INDELETE to indicate the deleting is processing.
+ # We need discuss about if we can delete a host when it is already
+ # installed by api.
+ if host.state.state != 'UNINITIALIZED' and force:
+ host.state.state = 'ERROR'
+ check_host_editable(
+ host, user=user,
+ check_in_installing=True
+ )
+ cluster_ids = []
+ for clusterhost in host.clusterhosts:
+ if clusterhost.state.state != 'UNINITIALIZED' and force:
+ clusterhost.state.state = 'ERROR'
+ # TODO(grace): here we check all clusters which use this host editable.
+ # Because in backend we do not have functions to delete host without
+ # reference its cluster. After deleting pure host supported in backend,
+ # we should change code here to is_cluster_editable.
+ # Here delete a host may fail even we set force flag.
+ cluster_api.check_cluster_editable(
+ clusterhost.cluster, user=user,
+ check_in_installing=True
+ )
+ cluster_ids.append(clusterhost.cluster_id)
+
+ # Delete host record directly if there is no need to delete it
+ # in backend or from_database_only is set.
+ if host.state.state == 'UNINITIALIZED' or from_database_only:
+ return utils.del_db_object(session, host)
+ else:
+ logging.info(
+ 'send del host %s task to celery', host_id
+ )
+ if not user:
+ user_id = host.creator_id
+ user_dict = user_api.get_user(user_id, session=session)
+ user_email = user_dict['email']
+ else:
+ user_email = user.email
+ from compass.tasks import client as celery_client
+ celery_client.celery.send_task(
+ 'compass.tasks.delete_host',
+ (
+ user.email, host.id, cluster_ids
+ ),
+ queue=user_email,
+ exchange=user_email,
+ routing_key=user_email
+ )
+ return {
+ 'status': 'delete action sent',
+ 'host': host,
+ }
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_HOST_CONFIG
+)
+@utils.wrap_to_dict(RESP_CONFIG_FIELDS)
+def get_host_config(host_id, user=None, session=None, **kwargs):
+ """Get host config."""
+ return _get_host(host_id, session=session)
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_HOST_CONFIG
+)
+@utils.wrap_to_dict(RESP_DEPLOYED_CONFIG_FIELDS)
+def get_host_deployed_config(host_id, user=None, session=None, **kwargs):
+ """Get host deployed config."""
+ return _get_host(host_id, session=session)
+
+
+# replace os_config to deployed_os_config in kwargs.
+@utils.replace_filters(
+ os_config='deployed_os_config'
+)
+@utils.supported_filters(
+ UPDATED_DEPLOYED_CONFIG_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_HOST_CONFIG
+)
+@utils.wrap_to_dict(RESP_CONFIG_FIELDS)
+def update_host_deployed_config(host_id, user=None, session=None, **kwargs):
+ """Update host deployed config."""
+ host = _get_host(host_id, session=session)
+ check_host_editable(host, user=user)
+ check_host_validated(host)
+ return utils.update_db_object(session, host, **kwargs)
+
+
+def _host_os_config_validates(
+ config, host, session=None, user=None, **kwargs
+):
+ """Check host os config's validation."""
+ metadata_api.validate_os_config(
+ config, host.os_id
+ )
+
+
+@utils.input_validates_with_args(
+ put_os_config=_host_os_config_validates
+)
+@utils.output_validates_with_args(
+ os_config=_host_os_config_validates
+)
+@utils.wrap_to_dict(RESP_CONFIG_FIELDS)
+def _update_host_config(host, session=None, user=None, **kwargs):
+ """Update host config."""
+ check_host_editable(host, user=user)
+ return utils.update_db_object(session, host, **kwargs)
+
+
+# replace os_config to put_os_config in kwargs.
+# It tells db the os_config will be updated not patched.
+@utils.replace_filters(
+ os_config='put_os_config'
+)
+@utils.supported_filters(
+ UPDATED_CONFIG_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_HOST_CONFIG
+)
+def update_host_config(host_id, user=None, session=None, **kwargs):
+ """Update host config."""
+ host = _get_host(host_id, session=session)
+ return _update_host_config(
+ host, session=session, user=user, **kwargs
+ )
+
+
+# replace os_config to patched_os_config in kwargs.
+# It tells db os_config will be patched not be updated.
+@utils.replace_filters(
+ os_config='patched_os_config'
+)
+@utils.supported_filters(
+ PATCHED_CONFIG_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_HOST_CONFIG
+)
+def patch_host_config(host_id, user=None, session=None, **kwargs):
+ """Patch host config."""
+ host = _get_host(host_id, session=session)
+ return _update_host_config(
+ host, session=session, user=user, **kwargs
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEL_HOST_CONFIG
+)
+@utils.wrap_to_dict(RESP_CONFIG_FIELDS)
+def del_host_config(host_id, user=None, session=None):
+ """delete a host config."""
+ host = _get_host(host_id, session=session)
+ check_host_editable(host, user=user)
+ return utils.update_db_object(
+ session, host, os_config={}, config_validated=False
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=SUPPORTED_NETOWORK_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_HOST_NETWORKS
+)
+@utils.wrap_to_dict(RESP_NETWORK_FIELDS)
+def list_host_networks(host_id, user=None, session=None, **filters):
+ """Get host networks for a host."""
+ host = _get_host(host_id, session=session)
+ return utils.list_db_objects(
+ session, models.HostNetwork,
+ host_id=host.id, **filters
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=SUPPORTED_NETOWORK_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_HOST_NETWORKS
+)
+@utils.wrap_to_dict(RESP_NETWORK_FIELDS)
+def list_hostnetworks(user=None, session=None, **filters):
+ """Get host networks."""
+ return utils.list_db_objects(
+ session, models.HostNetwork, **filters
+ )
+
+
+def _get_hostnetwork(host_network_id, session=None, **kwargs):
+ """Get hostnetwork by hostnetwork id."""
+ if isinstance(host_network_id, (int, long)):
+ return utils.get_db_object(
+ session, models.HostNetwork,
+ id=host_network_id, **kwargs
+ )
+ raise exception.InvalidParameter(
+ 'host network id %s type is not int compatible' % host_network_id
+ )
+
+
+def _get_host_network(host_id, host_network_id, session=None, **kwargs):
+ """Get hostnetwork by host id and hostnetwork id."""
+ host = _get_host(host_id, session=session)
+ host_network = _get_hostnetwork(host_network_id, session=session, **kwargs)
+ if host_network.host_id != host.id:
+ raise exception.RecordNotExists(
+ 'host %s does not own host network %s' % (
+ host.id, host_network.id
+ )
+ )
+ return host_network
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_HOST_NETWORKS
+)
+@utils.wrap_to_dict(RESP_NETWORK_FIELDS)
+def get_host_network(
+ host_id, host_network_id,
+ user=None, session=None, **kwargs
+):
+ """Get host network."""
+ return _get_host_network(
+ host_id, host_network_id, session=session
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_HOST_NETWORKS
+)
+@utils.wrap_to_dict(RESP_NETWORK_FIELDS)
+def get_hostnetwork(host_network_id, user=None, session=None, **kwargs):
+ """Get host network."""
+ return _get_hostnetwork(host_network_id, session=session)
+
+
+@utils.supported_filters(
+ ADDED_NETWORK_FIELDS,
+ optional_support_keys=OPTIONAL_ADDED_NETWORK_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(
+ ip=utils.check_ip
+)
+@utils.wrap_to_dict(RESP_NETWORK_FIELDS)
+def _add_host_network(
+ host_id, exception_when_existing=True,
+ session=None, user=None, interface=None, ip=None, **kwargs
+):
+ """Add hostnetwork to a host."""
+ host = _get_host(host_id, session=session)
+ check_host_editable(host, user=user)
+ user_id = user.id
+ return utils.add_db_object(
+ session, models.HostNetwork,
+ exception_when_existing,
+ host.id, interface, user_id, ip=ip, **kwargs
+ )
+
+
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_HOST_NETWORK
+)
+def add_host_network(
+ host_id, exception_when_existing=True,
+ interface=None, user=None, session=None, **kwargs
+):
+ """Create a hostnetwork to a host."""
+ return _add_host_network(
+ host_id,
+ exception_when_existing,
+ interface=interface, session=session, user=user, **kwargs
+ )
+
+
+def _get_hostnetwork_by_ip(
+ ip, session=None, **kwargs
+):
+ ip_int = long(netaddr.IPAddress(ip))
+ return utils.get_db_object(
+ session, models.HostNetwork,
+ ip_int=ip_int, **kwargs
+ )
+
+
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_HOST_NETWORK
+)
+def add_host_networks(
+ exception_when_existing=False,
+ data=[], user=None, session=None
+):
+ """Create host networks."""
+ hosts = []
+ failed_hosts = []
+ for host_data in data:
+ host_id = host_data['host_id']
+ host = _get_host(host_id, session=session)
+ networks = host_data['networks']
+ host_networks = []
+ failed_host_networks = []
+ for network in networks:
+ host_network = _get_hostnetwork_by_ip(
+ network['ip'], session=session,
+ exception_when_missing=False
+ )
+ if (
+ host_network and not (
+ host_network.host_id == host.id and
+ host_network.interface == network['interface']
+ )
+ ):
+ logging.error('ip %s exists in host network %s' % (
+ network['ip'], host_network.id
+ ))
+ failed_host_networks.append(network)
+ else:
+ host_networks.append(_add_host_network(
+ host.id, exception_when_existing,
+ session=session, user=user, **network
+ ))
+ if host_networks:
+ hosts.append({'host_id': host.id, 'networks': host_networks})
+ if failed_host_networks:
+ failed_hosts.append({
+ 'host_id': host.id, 'networks': failed_host_networks
+ })
+ return {
+ 'hosts': hosts,
+ 'failed_hosts': failed_hosts
+ }
+
+
+@utils.wrap_to_dict(RESP_NETWORK_FIELDS)
+def _update_host_network(
+ host_network, session=None, user=None, **kwargs
+):
+ """Update host network."""
+ check_host_editable(host_network.host, user=user)
+ return utils.update_db_object(session, host_network, **kwargs)
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_NETWORK_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(
+ ip=utils.check_ip
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_HOST_NETWORK
+)
+def update_host_network(
+ host_id, host_network_id, user=None, session=None, **kwargs
+):
+ """Update a host network by host id and host network id."""
+ host = _get_host(
+ host_id, session=session
+ )
+ if host.state.state == "SUCCESSFUL" and not host.reinstall_os:
+ logging.info("ignoring updating request for successful hosts")
+ return {}
+
+ host_network = _get_host_network(
+ host_id, host_network_id, session=session
+ )
+ return _update_host_network(
+ host_network, session=session, user=user, **kwargs
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_NETWORK_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(
+ ip=utils.check_ip
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_HOST_NETWORK
+)
+def update_hostnetwork(host_network_id, user=None, session=None, **kwargs):
+ """Update a host network by host network id."""
+ host_network = _get_hostnetwork(
+ host_network_id, session=session
+ )
+ return _update_host_network(
+ host_network, session=session, user=user, **kwargs
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEL_HOST_NETWORK
+)
+@utils.wrap_to_dict(RESP_NETWORK_FIELDS)
+def del_host_network(
+ host_id, host_network_id, user=None,
+ session=None, **kwargs
+):
+ """Delete a host network by host id and host network id."""
+ host_network = _get_host_network(
+ host_id, host_network_id, session=session
+ )
+ check_host_editable(host_network.host, user=user)
+ return utils.del_db_object(session, host_network)
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEL_HOST_NETWORK
+)
+@utils.wrap_to_dict(RESP_NETWORK_FIELDS)
+def del_hostnetwork(host_network_id, user=None, session=None, **kwargs):
+ """Delete a host network by host network id."""
+ host_network = _get_hostnetwork(
+ host_network_id, session=session
+ )
+ check_host_editable(host_network.host, user=user)
+ return utils.del_db_object(session, host_network)
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_GET_HOST_STATE
+)
+@utils.wrap_to_dict(RESP_STATE_FIELDS)
+def get_host_state(host_id, user=None, session=None, **kwargs):
+ """Get host state info."""
+ return _get_host(host_id, session=session).state
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_STATE_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_HOST_STATE
+)
+@utils.wrap_to_dict(RESP_STATE_FIELDS)
+def update_host_state(host_id, user=None, session=None, **kwargs):
+ """Update a host state."""
+ host = _get_host(host_id, session=session)
+ utils.update_db_object(session, host.state, **kwargs)
+ return host.state
+
+
+@util.deprecated
+@utils.supported_filters(
+ optional_support_keys=UPDATED_STATE_INTERNAL_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_HOST_STATE
+)
+@utils.wrap_to_dict(['status', 'host'])
+def update_host_state_internal(
+ host_id, from_database_only=False,
+ user=None, session=None, **kwargs
+):
+ """Update a host state.
+
+ This function is called when host os is installed.
+ If from_database_only, the state is updated in database.
+ Otherwise a celery task sent to os installer and package installer
+ to do some future actions.
+ """
+ # TODO(xicheng): should be merged into update_host_state
+ host = _get_host(host_id, session=session)
+ logging.info("======host state: %s", host.state)
+ if 'ready' in kwargs and kwargs['ready'] and not host.state.ready:
+ ready_triggered = True
+ else:
+ ready_triggered = False
+ clusterhosts_ready = {}
+ clusters_os_ready = {}
+ if ready_triggered:
+ for clusterhost in host.clusterhosts:
+ cluster = clusterhost.cluster
+ if cluster.flavor_name:
+ clusterhosts_ready[cluster.id] = False
+ else:
+ clusterhosts_ready[cluster.id] = True
+ all_os_ready = True
+ for clusterhost_in_cluster in cluster.clusterhosts:
+ host_in_cluster = clusterhost_in_cluster.host
+ if host_in_cluster.id == host.id:
+ continue
+ if not host_in_cluster.state.ready:
+ all_os_ready = False
+ clusters_os_ready[cluster.id] = all_os_ready
+ logging.debug('host %s ready: %s', host_id, ready_triggered)
+ logging.debug("clusterhosts_ready is: %s", clusterhosts_ready)
+ logging.debug("clusters_os_ready is %s", clusters_os_ready)
+
+ if not ready_triggered or from_database_only:
+ logging.debug('%s state is set to %s', host.name, kwargs)
+ utils.update_db_object(session, host.state, **kwargs)
+ if not host.state.ready:
+ for clusterhost in host.clusterhosts:
+ utils.update_db_object(
+ session, clusterhost.state, ready=False
+ )
+ utils.update_db_object(
+ session, clusterhost.cluster.state, ready=False
+ )
+ status = '%s state is updated' % host.name
+ else:
+ if not user:
+ user_id = host.creator_id
+ user_dict = user_api.get_user(user_id, session=session)
+ user_email = user_dict['email']
+ else:
+ user_email = user.email
+ from compass.tasks import client as celery_client
+ celery_client.celery.send_task(
+ 'compass.tasks.os_installed',
+ (
+ host.id, clusterhosts_ready,
+ clusters_os_ready
+ ),
+ queue=user_email,
+ exchange=user_email,
+ routing_key=user_email
+ )
+ status = '%s: clusterhosts ready %s clusters os ready %s' % (
+ host.name, clusterhosts_ready, clusters_os_ready
+ )
+ logging.info('action status: %s', status)
+ return {
+ 'status': status,
+ 'host': host.state
+ }
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_LOG_FIELDS)
+def get_host_log_histories(host_id, user=None, session=None, **kwargs):
+ """Get host log history."""
+ host = _get_host(host_id, session=session)
+ return utils.list_db_objects(
+ session, models.HostLogHistory, id=host.id, **kwargs
+ )
+
+
+def _get_host_log_history(host_id, filename, session=None, **kwargs):
+ host = _get_host(host_id, session=session)
+ return utils.get_db_object(
+ session, models.HostLogHistory, id=host.id,
+ filename=filename, **kwargs
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_LOG_FIELDS)
+def get_host_log_history(host_id, filename, user=None, session=None, **kwargs):
+ """Get host log history."""
+ return _get_host_log_history(
+ host_id, filename, session=session
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_LOG_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_LOG_FIELDS)
+def update_host_log_history(
+ host_id, filename, user=None,
+ session=None, **kwargs
+):
+ """Update a host log history."""
+ host_log_history = _get_host_log_history(
+ host_id, filename, session=session
+ )
+ return utils.update_db_object(session, host_log_history, **kwargs)
+
+
+@utils.supported_filters(
+ ADDED_LOG_FIELDS,
+ optional_support_keys=UPDATED_LOG_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_LOG_FIELDS)
+def add_host_log_history(
+ host_id, exception_when_existing=False,
+ filename=None, user=None, session=None, **kwargs
+):
+ """add a host log history."""
+ host = _get_host(host_id, session=session)
+ return utils.add_db_object(
+ session, models.HostLogHistory, exception_when_existing,
+ host.id, filename, **kwargs
+ )
+
+
+@utils.supported_filters(optional_support_keys=['poweron'])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEPLOY_HOST
+)
+@utils.wrap_to_dict(
+ RESP_DEPLOY_FIELDS,
+ host=RESP_CONFIG_FIELDS
+)
+def poweron_host(
+ host_id, poweron={}, user=None, session=None, **kwargs
+):
+ """power on host."""
+ from compass.tasks import client as celery_client
+ host = _get_host(host_id, session=session)
+ check_host_validated(host)
+ if not user:
+ user_id = host.creator_id
+ user_dict = user_api.get_user(user_id, session=session)
+ user_email = user_dict['email']
+ else:
+ user_email = user.email
+ celery_client.celery.send_task(
+ 'compass.tasks.poweron_host',
+ (host.id,),
+ queue=user_email,
+ exchange=user_email,
+ routing_key=user_email
+ )
+ return {
+ 'status': 'poweron %s action sent' % host.name,
+ 'host': host
+ }
+
+
+@utils.supported_filters(optional_support_keys=['poweroff'])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEPLOY_HOST
+)
+@utils.wrap_to_dict(
+ RESP_DEPLOY_FIELDS,
+ host=RESP_CONFIG_FIELDS
+)
+def poweroff_host(
+ host_id, poweroff={}, user=None, session=None, **kwargs
+):
+ """power off host."""
+ from compass.tasks import client as celery_client
+ host = _get_host(host_id, session=session)
+ check_host_validated(host)
+ if not user:
+ user_id = host.creator_id
+ user_dict = user_api.get_user(user_id, session=session)
+ user_email = user_dict['email']
+ else:
+ user_email = user.email
+ celery_client.celery.send_task(
+ 'compass.tasks.poweroff_host',
+ (host.id,),
+ queue=user_email,
+ exchange=user_email,
+ routing_key=user_email
+ )
+ return {
+ 'status': 'poweroff %s action sent' % host.name,
+ 'host': host
+ }
+
+
+@utils.supported_filters(optional_support_keys=['reset'])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEPLOY_HOST
+)
+@utils.wrap_to_dict(
+ RESP_DEPLOY_FIELDS,
+ host=RESP_CONFIG_FIELDS
+)
+def reset_host(
+ host_id, reset={}, user=None, session=None, **kwargs
+):
+ """reset host."""
+ from compass.tasks import client as celery_client
+ host = _get_host(host_id, session=session)
+ check_host_validated(host)
+ if not user:
+ user_id = host.creator_id
+ user_dict = user_api.get_user(user_id, session=session)
+ user_email = user_dict['email']
+ else:
+ user_email = user.email
+ celery_client.celery.send_task(
+ 'compass.tasks.reset_host',
+ (host.id,),
+ queue=user_email,
+ exchange=user_email,
+ routing_key=user_email
+ )
+ return {
+ 'status': 'reset %s action sent' % host.name,
+ 'host': host
+ }
diff --git a/compass-tasks/db/api/machine.py b/compass-tasks/db/api/machine.py
new file mode 100644
index 0000000..b7b16b2
--- /dev/null
+++ b/compass-tasks/db/api/machine.py
@@ -0,0 +1,317 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Switch database operations."""
+import logging
+import re
+
+from compass.db.api import database
+from compass.db.api import permission
+from compass.db.api import user as user_api
+from compass.db.api import utils
+from compass.db import exception
+from compass.db import models
+
+from compass.utils import setting_wrapper as setting
+from compass.utils import util
+
+
+MACHINE_PRIMARY_FILEDS = ['mac', 'owner_id']
+SUPPORTED_FIELDS = [
+ 'mac', 'tag', 'location',
+ 'machine_attributes', 'owner_id']
+IGNORE_FIELDS = ['id', 'created_at', 'updated_at']
+UPDATED_FIELDS = [
+ 'ipmi_credentials', 'machine_attributes',
+ 'tag', 'location']
+PATCHED_FIELDS = [
+ 'patched_ipmi_credentials', 'patched_tag',
+ 'patched_location'
+]
+RESP_FIELDS = [
+ 'id', 'mac', 'ipmi_credentials', 'switches', 'switch_ip',
+ 'port', 'vlans', 'machine_attributes', 'owner_id',
+ 'tag', 'location', 'created_at', 'updated_at'
+]
+RESP_DEPLOY_FIELDS = [
+ 'status', 'machine'
+]
+
+
+def _get_machine(machine_id, session=None, **kwargs):
+ """Get machine by id."""
+ if isinstance(machine_id, (int, long)):
+ return utils.get_db_object(
+ session, models.Machine,
+ id=machine_id, **kwargs
+ )
+ raise exception.InvalidParameter(
+ 'machine id %s type is not int compatible' % machine_id
+ )
+
+
+@utils.supported_filters(
+ MACHINE_PRIMARY_FILEDS,
+ optional_support_keys=SUPPORTED_FIELDS
+)
+@utils.input_validates(mac=utils.check_mac)
+def _add_machine(mac, owner_id=None, session=None, **kwargs):
+ """Add a machine."""
+ if isinstance(owner_id, (int, long)):
+ return utils.add_db_object(
+ session, models.Machine,
+ True,
+ mac,
+ owner_id=owner_id,
+ **kwargs
+ )
+ raise exception.InvalidParameter(
+ 'owner id %s type is not int compatible' % owner_id
+ )
+
+
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_FIELDS)
+def add_machine(
+ mac, owner_id=None, session=None, user=None, **kwargs
+):
+ """Add a machine."""
+ return _add_machine(
+ mac,
+ owner_id=owner_id,
+ session=session, **kwargs
+ )
+
+
+def get_machine_internal(machine_id, session=None, **kwargs):
+ """Helper function to other files under db/api."""
+ return _get_machine(machine_id, session=session, **kwargs)
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_MACHINES
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def get_machine(
+ machine_id, exception_when_missing=True,
+ user=None, session=None, **kwargs
+):
+ """get a machine."""
+ return _get_machine(
+ machine_id, session=session,
+ exception_when_missing=exception_when_missing
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=SUPPORTED_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_MACHINES
+)
+@utils.output_filters(
+ tag=utils.general_filter_callback,
+ location=utils.general_filter_callback
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def list_machines(user=None, session=None, **filters):
+ """List machines."""
+ machines = utils.list_db_objects(
+ session, models.Machine, **filters
+ )
+ if not user.is_admin and len(machines):
+ machines = [m for m in machines if m.owner_id == user.id]
+ return machines
+
+
+@utils.wrap_to_dict(RESP_FIELDS)
+def _update_machine(machine_id, session=None, **kwargs):
+ """Update a machine."""
+ machine = _get_machine(machine_id, session=session)
+ return utils.update_db_object(session, machine, **kwargs)
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(ipmi_credentials=utils.check_ipmi_credentials)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_MACHINE
+)
+def update_machine(machine_id, user=None, session=None, **kwargs):
+ """Update a machine."""
+ return _update_machine(
+ machine_id, session=session, **kwargs
+ )
+
+
+# replace [ipmi_credentials, tag, location] to
+# [patched_ipmi_credentials, patched_tag, patched_location]
+# in kwargs. It tells db these fields will be patched.
+@utils.replace_filters(
+ ipmi_credentials='patched_ipmi_credentials',
+ tag='patched_tag',
+ location='patched_location'
+)
+@utils.supported_filters(
+ optional_support_keys=PATCHED_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@utils.output_validates(ipmi_credentials=utils.check_ipmi_credentials)
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_MACHINE
+)
+def patch_machine(machine_id, user=None, session=None, **kwargs):
+ """Patch a machine."""
+ return _update_machine(
+ machine_id, session=session, **kwargs
+ )
+
+
+def _check_machine_deletable(machine):
+ """Check a machine deletable."""
+ if machine.host:
+ host = machine.host
+ raise exception.NotAcceptable(
+ 'machine %s has host %s on it' % (
+ machine.mac, host.name
+ )
+ )
+
+
+@utils.supported_filters()
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEL_MACHINE
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def del_machine(machine_id, user=None, session=None, **kwargs):
+ """Delete a machine."""
+ machine = _get_machine(machine_id, session=session)
+ _check_machine_deletable(machine)
+ return utils.del_db_object(session, machine)
+
+
+@utils.supported_filters(optional_support_keys=['poweron'])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEPLOY_HOST
+)
+@utils.wrap_to_dict(
+ RESP_DEPLOY_FIELDS,
+ machine=RESP_FIELDS
+)
+def poweron_machine(
+ machine_id, poweron={}, user=None, session=None, **kwargs
+):
+ """power on machine."""
+ from compass.tasks import client as celery_client
+ machine = _get_machine(
+ machine_id, session=session
+ )
+ if not user:
+ user_id = machine.owner_id
+ user_dict = user_api.get_user(user_id, session=session)
+ user_email = user_dict['email']
+ else:
+ user_email = user.email
+ celery_client.celery.send_task(
+ 'compass.tasks.poweron_machine',
+ (machine_id,),
+ queue=user_email,
+ exchange=user_email,
+ routing_key=user_email
+ )
+ return {
+ 'status': 'poweron %s action sent' % machine.mac,
+ 'machine': machine
+ }
+
+
+@utils.supported_filters(optional_support_keys=['poweroff'])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEPLOY_HOST
+)
+@utils.wrap_to_dict(
+ RESP_DEPLOY_FIELDS,
+ machine=RESP_FIELDS
+)
+def poweroff_machine(
+ machine_id, poweroff={}, user=None, session=None, **kwargs
+):
+ """power off machine."""
+ from compass.tasks import client as celery_client
+ machine = _get_machine(
+ machine_id, session=session
+ )
+ if not user:
+ user_id = machine.owner_id
+ user_dict = user_api.get_user(user_id, session=session)
+ user_email = user_dict['email']
+ else:
+ user_email = user.email
+ celery_client.celery.send_task(
+ 'compass.tasks.poweroff_machine',
+ (machine_id,),
+ queue=user_email,
+ exchange=user_email,
+ routing_key=user_email
+ )
+ return {
+ 'status': 'poweroff %s action sent' % machine.mac,
+ 'machine': machine
+ }
+
+
+@utils.supported_filters(optional_support_keys=['reset'])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEPLOY_HOST
+)
+@utils.wrap_to_dict(
+ RESP_DEPLOY_FIELDS,
+ machine=RESP_FIELDS
+)
+def reset_machine(
+ machine_id, reset={}, user=None, session=None, **kwargs
+):
+ """reset machine."""
+ from compass.tasks import client as celery_client
+ machine = _get_machine(
+ machine_id, session=session
+ )
+ if not user:
+ user_id = machine.owner_id
+ user_dict = user_api.get_user(user_id, session=session)
+ user_email = user_dict['email']
+ else:
+ user_email = user.email
+ celery_client.celery.send_task(
+ 'compass.tasks.reset_machine',
+ (machine_id,),
+ queue=user_email,
+ exchange=user_email,
+ routing_key=user_email
+ )
+ return {
+ 'status': 'reset %s action sent' % machine.mac,
+ 'machine': machine
+ }
diff --git a/compass-tasks/db/api/metadata.py b/compass-tasks/db/api/metadata.py
new file mode 100644
index 0000000..16310c8
--- /dev/null
+++ b/compass-tasks/db/api/metadata.py
@@ -0,0 +1,517 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Metadata related database operations."""
+import copy
+import logging
+import string
+
+from compass.db.api import adapter as adapter_api
+from compass.db.api import database
+from compass.db.api import utils
+from compass.db import callback as metadata_callback
+from compass.db import exception
+from compass.db import models
+from compass.db import validator as metadata_validator
+
+
+from compass.utils import setting_wrapper as setting
+from compass.utils import util
+
+
+OS_FIELDS = None
+PACKAGE_FIELDS = None
+FLAVOR_FIELDS = None
+OSES_METADATA = None
+PACKAGES_METADATA = None
+FLAVORS_METADATA = None
+OSES_METADATA_UI_CONVERTERS = None
+FLAVORS_METADATA_UI_CONVERTERS = None
+
+
+def _get_field_from_configuration(configs):
+ """Get fields from configurations."""
+ fields = {}
+ for config in configs:
+ if not isinstance(config, dict):
+ raise exception.InvalidParameter(
+ 'config %s is not dict' % config
+ )
+ field_name = config['NAME']
+ fields[field_name] = {
+ 'name': field_name,
+ 'id': field_name,
+ 'field_type': config.get('FIELD_TYPE', basestring),
+ 'display_type': config.get('DISPLAY_TYPE', 'text'),
+ 'validator': config.get('VALIDATOR', None),
+ 'js_validator': config.get('JS_VALIDATOR', None),
+ 'description': config.get('DESCRIPTION', field_name)
+ }
+ return fields
+
+
+def _get_os_fields_from_configuration():
+ """Get os fields from os field config dir."""
+ env_locals = {}
+ env_locals.update(metadata_validator.VALIDATOR_LOCALS)
+ env_locals.update(metadata_callback.CALLBACK_LOCALS)
+ configs = util.load_configs(
+ setting.OS_FIELD_DIR,
+ env_locals=env_locals
+ )
+ return _get_field_from_configuration(
+ configs
+ )
+
+
+def _get_package_fields_from_configuration():
+ """Get package fields from package field config dir."""
+ env_locals = {}
+ env_locals.update(metadata_validator.VALIDATOR_LOCALS)
+ env_locals.update(metadata_callback.CALLBACK_LOCALS)
+ configs = util.load_configs(
+ setting.PACKAGE_FIELD_DIR,
+ env_locals=env_locals
+ )
+ return _get_field_from_configuration(
+ configs
+ )
+
+
+def _get_flavor_fields_from_configuration():
+ """Get flavor fields from flavor field config dir."""
+ env_locals = {}
+ env_locals.update(metadata_validator.VALIDATOR_LOCALS)
+ env_locals.update(metadata_callback.CALLBACK_LOCALS)
+ configs = util.load_configs(
+ setting.FLAVOR_FIELD_DIR,
+ env_locals=env_locals
+ )
+ return _get_field_from_configuration(
+ configs
+ )
+
+
+def _get_metadata_from_configuration(
+ path, name, config,
+ fields, **kwargs
+):
+ """Recursively get metadata from configuration.
+
+ Args:
+ path: used to indicate the path to the root element.
+ mainly for trouble shooting.
+ name: the key of the metadata section.
+ config: the value of the metadata section.
+ fields: all fields defined in os fields or package fields dir.
+ """
+ if not isinstance(config, dict):
+ raise exception.InvalidParameter(
+ '%s config %s is not dict' % (path, config)
+ )
+ metadata_self = config.get('_self', {})
+ if 'field' in metadata_self:
+ field_name = metadata_self['field']
+ field = fields[field_name]
+ else:
+ field = {}
+ # mapping to may contain $ like $partition. Here we replace the
+ # $partition to the key of the correspendent config. The backend then
+ # can use this kind of feature to support multi partitions when we
+ # only declare the partition metadata in one place.
+ mapping_to_template = metadata_self.get('mapping_to', None)
+ if mapping_to_template:
+ mapping_to = string.Template(
+ mapping_to_template
+ ).safe_substitute(
+ **kwargs
+ )
+ else:
+ mapping_to = None
+ self_metadata = {
+ 'name': name,
+ 'display_name': metadata_self.get('display_name', name),
+ 'field_type': field.get('field_type', dict),
+ 'display_type': field.get('display_type', None),
+ 'description': metadata_self.get(
+ 'description', field.get('description', None)
+ ),
+ 'is_required': metadata_self.get('is_required', False),
+ 'required_in_whole_config': metadata_self.get(
+ 'required_in_whole_config', False),
+ 'mapping_to': mapping_to,
+ 'validator': metadata_self.get(
+ 'validator', field.get('validator', None)
+ ),
+ 'js_validator': metadata_self.get(
+ 'js_validator', field.get('js_validator', None)
+ ),
+ 'default_value': metadata_self.get('default_value', None),
+ 'default_callback': metadata_self.get('default_callback', None),
+ 'default_callback_params': metadata_self.get(
+ 'default_callback_params', {}),
+ 'options': metadata_self.get('options', None),
+ 'options_callback': metadata_self.get('options_callback', None),
+ 'options_callback_params': metadata_self.get(
+ 'options_callback_params', {}),
+ 'autofill_callback': metadata_self.get(
+ 'autofill_callback', None),
+ 'autofill_callback_params': metadata_self.get(
+ 'autofill_callback_params', {}),
+ 'required_in_options': metadata_self.get(
+ 'required_in_options', False)
+ }
+ self_metadata.update(kwargs)
+ metadata = {'_self': self_metadata}
+ # Key extension used to do two things:
+ # one is to return the extended metadata that $<something>
+ # will be replace to possible extensions.
+ # The other is to record the $<something> to extended value
+ # and used in future mapping_to subsititution.
+ # TODO(grace): select proper name instead of key_extensions if
+ # you think it is better.
+ # Suppose key_extension is {'$partition': ['/var', '/']} for $partition
+ # the metadata for $partition will be mapped to {
+ # '/var': ..., '/': ...} and kwargs={'partition': '/var'} and
+ # kwargs={'partition': '/'} will be parsed to recursive metadata parsing
+ # for sub metadata under '/var' and '/'. Then in the metadata parsing
+ # for the sub metadata, this kwargs will be used to substitute mapping_to.
+ key_extensions = metadata_self.get('key_extensions', {})
+ general_keys = []
+ for key, value in config.items():
+ if key.startswith('_'):
+ continue
+ if key in key_extensions:
+ if not key.startswith('$'):
+ raise exception.InvalidParameter(
+ '%s subkey %s should start with $' % (
+ path, key
+ )
+ )
+ extended_keys = key_extensions[key]
+ for extended_key in extended_keys:
+ if extended_key.startswith('$'):
+ raise exception.InvalidParameter(
+ '%s extended key %s should not start with $' % (
+ path, extended_key
+ )
+ )
+ sub_kwargs = dict(kwargs)
+ sub_kwargs[key[1:]] = extended_key
+ metadata[extended_key] = _get_metadata_from_configuration(
+ '%s/%s' % (path, extended_key), extended_key, value,
+ fields, **sub_kwargs
+ )
+ else:
+ if key.startswith('$'):
+ general_keys.append(key)
+ metadata[key] = _get_metadata_from_configuration(
+ '%s/%s' % (path, key), key, value,
+ fields, **kwargs
+ )
+ if len(general_keys) > 1:
+ raise exception.InvalidParameter(
+ 'foud multi general keys in %s: %s' % (
+ path, general_keys
+ )
+ )
+ return metadata
+
+
+def _get_oses_metadata_from_configuration():
+ """Get os metadata from os metadata config dir."""
+ oses_metadata = {}
+ env_locals = {}
+ env_locals.update(metadata_validator.VALIDATOR_LOCALS)
+ env_locals.update(metadata_callback.CALLBACK_LOCALS)
+ configs = util.load_configs(
+ setting.OS_METADATA_DIR,
+ env_locals=env_locals
+ )
+ for config in configs:
+ os_name = config['OS']
+ os_metadata = oses_metadata.setdefault(os_name, {})
+ for key, value in config['METADATA'].items():
+ os_metadata[key] = _get_metadata_from_configuration(
+ key, key, value, OS_FIELDS
+ )
+
+ oses = adapter_api.OSES
+ parents = {}
+ for os_name, os in oses.items():
+ parent = os.get('parent', None)
+ parents[os_name] = parent
+ for os_name, os in oses.items():
+ oses_metadata[os_name] = util.recursive_merge_dict(
+ os_name, oses_metadata, parents
+ )
+ return oses_metadata
+
+
+def _get_packages_metadata_from_configuration():
+ """Get package metadata from package metadata config dir."""
+ packages_metadata = {}
+ env_locals = {}
+ env_locals.update(metadata_validator.VALIDATOR_LOCALS)
+ env_locals.update(metadata_callback.CALLBACK_LOCALS)
+ configs = util.load_configs(
+ setting.PACKAGE_METADATA_DIR,
+ env_locals=env_locals
+ )
+ for config in configs:
+ adapter_name = config['ADAPTER']
+ package_metadata = packages_metadata.setdefault(adapter_name, {})
+ for key, value in config['METADATA'].items():
+ package_metadata[key] = _get_metadata_from_configuration(
+ key, key, value, PACKAGE_FIELDS
+ )
+ adapters = adapter_api.ADAPTERS
+ parents = {}
+ for adapter_name, adapter in adapters.items():
+ parent = adapter.get('parent', None)
+ parents[adapter_name] = parent
+ for adapter_name, adapter in adapters.items():
+ packages_metadata[adapter_name] = util.recursive_merge_dict(
+ adapter_name, packages_metadata, parents
+ )
+ return packages_metadata
+
+
+def _get_flavors_metadata_from_configuration():
+ """Get flavor metadata from flavor metadata config dir."""
+ flavors_metadata = {}
+ env_locals = {}
+ env_locals.update(metadata_validator.VALIDATOR_LOCALS)
+ env_locals.update(metadata_callback.CALLBACK_LOCALS)
+ configs = util.load_configs(
+ setting.FLAVOR_METADATA_DIR,
+ env_locals=env_locals
+ )
+ for config in configs:
+ adapter_name = config['ADAPTER']
+ flavor_name = config['FLAVOR']
+ flavor_metadata = flavors_metadata.setdefault(
+ adapter_name, {}
+ ).setdefault(flavor_name, {})
+ for key, value in config['METADATA'].items():
+ flavor_metadata[key] = _get_metadata_from_configuration(
+ key, key, value, FLAVOR_FIELDS
+ )
+
+ packages_metadata = PACKAGES_METADATA
+ adapters_flavors = adapter_api.ADAPTERS_FLAVORS
+ for adapter_name, adapter_flavors in adapters_flavors.items():
+ package_metadata = packages_metadata.get(adapter_name, {})
+ for flavor_name, flavor in adapter_flavors.items():
+ flavor_metadata = flavors_metadata.setdefault(
+ adapter_name, {}
+ ).setdefault(flavor_name, {})
+ util.merge_dict(flavor_metadata, package_metadata, override=False)
+ return flavors_metadata
+
+
+def _filter_metadata(metadata, **kwargs):
+ if not isinstance(metadata, dict):
+ return metadata
+ filtered_metadata = {}
+ for key, value in metadata.items():
+ if key == '_self':
+ default_value = value.get('default_value', None)
+ if default_value is None:
+ default_callback_params = value.get(
+ 'default_callback_params', {}
+ )
+ callback_params = dict(kwargs)
+ if default_callback_params:
+ callback_params.update(default_callback_params)
+ default_callback = value.get('default_callback', None)
+ if default_callback:
+ default_value = default_callback(key, **callback_params)
+ options = value.get('options', None)
+ if options is None:
+ options_callback_params = value.get(
+ 'options_callback_params', {}
+ )
+ callback_params = dict(kwargs)
+ if options_callback_params:
+ callback_params.update(options_callback_params)
+
+ options_callback = value.get('options_callback', None)
+ if options_callback:
+ options = options_callback(key, **callback_params)
+ filtered_metadata[key] = value
+ if default_value is not None:
+ filtered_metadata[key]['default_value'] = default_value
+ if options is not None:
+ filtered_metadata[key]['options'] = options
+ else:
+ filtered_metadata[key] = _filter_metadata(value, **kwargs)
+ return filtered_metadata
+
+
+def _load_metadata(force_reload=False):
+ """Load metadata information into memory.
+
+ If force_reload, the metadata information will be reloaded
+ even if the metadata is already loaded.
+ """
+ adapter_api.load_adapters_internal(force_reload=force_reload)
+ global OS_FIELDS
+ if force_reload or OS_FIELDS is None:
+ OS_FIELDS = _get_os_fields_from_configuration()
+ global PACKAGE_FIELDS
+ if force_reload or PACKAGE_FIELDS is None:
+ PACKAGE_FIELDS = _get_package_fields_from_configuration()
+ global FLAVOR_FIELDS
+ if force_reload or FLAVOR_FIELDS is None:
+ FLAVOR_FIELDS = _get_flavor_fields_from_configuration()
+ global OSES_METADATA
+ if force_reload or OSES_METADATA is None:
+ OSES_METADATA = _get_oses_metadata_from_configuration()
+ global PACKAGES_METADATA
+ if force_reload or PACKAGES_METADATA is None:
+ PACKAGES_METADATA = _get_packages_metadata_from_configuration()
+ global FLAVORS_METADATA
+ if force_reload or FLAVORS_METADATA is None:
+ FLAVORS_METADATA = _get_flavors_metadata_from_configuration()
+ global OSES_METADATA_UI_CONVERTERS
+ if force_reload or OSES_METADATA_UI_CONVERTERS is None:
+ OSES_METADATA_UI_CONVERTERS = (
+ _get_oses_metadata_ui_converters_from_configuration()
+ )
+ global FLAVORS_METADATA_UI_CONVERTERS
+ if force_reload or FLAVORS_METADATA_UI_CONVERTERS is None:
+ FLAVORS_METADATA_UI_CONVERTERS = (
+ _get_flavors_metadata_ui_converters_from_configuration()
+ )
+
+
+def _get_oses_metadata_ui_converters_from_configuration():
+ """Get os metadata ui converters from os metadata mapping config dir.
+
+ os metadata ui converter is used to convert os metadata to
+ the format UI can understand and show.
+ """
+ oses_metadata_ui_converters = {}
+ configs = util.load_configs(setting.OS_MAPPING_DIR)
+ for config in configs:
+ os_name = config['OS']
+ oses_metadata_ui_converters[os_name] = config.get('CONFIG_MAPPING', {})
+
+ oses = adapter_api.OSES
+ parents = {}
+ for os_name, os in oses.items():
+ parent = os.get('parent', None)
+ parents[os_name] = parent
+ for os_name, os in oses.items():
+ oses_metadata_ui_converters[os_name] = util.recursive_merge_dict(
+ os_name, oses_metadata_ui_converters, parents
+ )
+ return oses_metadata_ui_converters
+
+
+def _get_flavors_metadata_ui_converters_from_configuration():
+ """Get flavor metadata ui converters from flavor mapping config dir."""
+ flavors_metadata_ui_converters = {}
+ configs = util.load_configs(setting.FLAVOR_MAPPING_DIR)
+ for config in configs:
+ adapter_name = config['ADAPTER']
+ flavor_name = config['FLAVOR']
+ flavors_metadata_ui_converters.setdefault(
+ adapter_name, {}
+ )[flavor_name] = config.get('CONFIG_MAPPING', {})
+ adapters = adapter_api.ADAPTERS
+ parents = {}
+ for adapter_name, adapter in adapters.items():
+ parent = adapter.get('parent', None)
+ parents[adapter_name] = parent
+ for adapter_name, adapter in adapters.items():
+ flavors_metadata_ui_converters[adapter_name] = (
+ util.recursive_merge_dict(
+ adapter_name, flavors_metadata_ui_converters, parents
+ )
+ )
+ return flavors_metadata_ui_converters
+
+
+def get_packages_metadata_internal(force_reload=False):
+ """Get deployable package metadata."""
+ _load_metadata(force_reload=force_reload)
+ metadata_mapping = {}
+ adapters = adapter_api.ADAPTERS
+ for adapter_name, adapter in adapters.items():
+ if adapter.get('deployable'):
+ metadata_mapping[adapter_name] = _filter_metadata(
+ PACKAGES_METADATA.get(adapter_name, {})
+ )
+ else:
+ logging.info(
+ 'ignore metadata since its adapter %s is not deployable',
+ adapter_name
+ )
+ return metadata_mapping
+
+
+def get_flavors_metadata_internal(force_reload=False):
+ """Get deployable flavor metadata."""
+ _load_metadata(force_reload=force_reload)
+ metadata_mapping = {}
+ adapters_flavors = adapter_api.ADAPTERS_FLAVORS
+ for adapter_name, adapter_flavors in adapters_flavors.items():
+ adapter = adapter_api.ADAPTERS[adapter_name]
+ if not adapter.get('deployable'):
+ logging.info(
+ 'ignore metadata since its adapter %s is not deployable',
+ adapter_name
+ )
+ continue
+ for flavor_name, flavor in adapter_flavors.items():
+ flavor_metadata = FLAVORS_METADATA.get(
+ adapter_name, {}
+ ).get(flavor_name, {})
+ metadata = _filter_metadata(flavor_metadata)
+ metadata_mapping.setdefault(
+ adapter_name, {}
+ )[flavor_name] = metadata
+ return metadata_mapping
+
+
+def get_flavors_metadata_ui_converters_internal(force_reload=False):
+ """Get usable flavor metadata ui converters."""
+ _load_metadata(force_reload=force_reload)
+ return FLAVORS_METADATA_UI_CONVERTERS
+
+
+def get_oses_metadata_internal(force_reload=False):
+ """Get deployable os metadata."""
+ _load_metadata(force_reload=force_reload)
+ metadata_mapping = {}
+ oses = adapter_api.OSES
+ for os_name, os in oses.items():
+ if os.get('deployable'):
+ metadata_mapping[os_name] = _filter_metadata(
+ OSES_METADATA.get(os_name, {})
+ )
+ else:
+ logging.info(
+ 'ignore metadata since its os %s is not deployable',
+ os_name
+ )
+ return metadata_mapping
+
+
+def get_oses_metadata_ui_converters_internal(force_reload=False):
+ """Get usable os metadata ui converters."""
+ _load_metadata(force_reload=force_reload)
+ return OSES_METADATA_UI_CONVERTERS
diff --git a/compass-tasks/db/api/metadata_holder.py b/compass-tasks/db/api/metadata_holder.py
new file mode 100644
index 0000000..24afc67
--- /dev/null
+++ b/compass-tasks/db/api/metadata_holder.py
@@ -0,0 +1,731 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Metadata related object holder."""
+import logging
+
+from compass.db.api import adapter as adapter_api
+from compass.db.api import adapter_holder as adapter_holder_api
+from compass.db.api import database
+from compass.db.api import metadata as metadata_api
+from compass.db.api import permission
+from compass.db.api import user as user_api
+from compass.db.api import utils
+from compass.db import exception
+from compass.db import models
+from compass.utils import setting_wrapper as setting
+from compass.utils import util
+
+
+RESP_METADATA_FIELDS = [
+ 'os_config', 'package_config'
+]
+RESP_UI_METADATA_FIELDS = [
+ 'os_global_config', 'flavor_config'
+]
+
+
+def load_metadatas(force_reload=False):
+ """Load metadatas."""
+ # TODO(xicheng): today we load metadata in memory as it original
+ # format in files in metadata.py. We get these inmemory metadata
+ # and do some translation, store the translated metadata into memory
+ # too in metadata_holder.py. api can only access the global inmemory
+ # data in metadata_holder.py.
+ _load_os_metadatas(force_reload=force_reload)
+ _load_package_metadatas(force_reload=force_reload)
+ _load_flavor_metadatas(force_reload=force_reload)
+ _load_os_metadata_ui_converters(force_reload=force_reload)
+ _load_flavor_metadata_ui_converters(force_reload=force_reload)
+
+
+def _load_os_metadata_ui_converters(force_reload=False):
+ global OS_METADATA_UI_CONVERTERS
+ if force_reload or OS_METADATA_UI_CONVERTERS is None:
+ logging.info('load os metadatas ui converters into memory')
+ OS_METADATA_UI_CONVERTERS = (
+ metadata_api.get_oses_metadata_ui_converters_internal(
+ force_reload=force_reload
+ )
+ )
+
+
+def _load_os_metadatas(force_reload=False):
+ """Load os metadata from inmemory db and map it by os_id."""
+ global OS_METADATA_MAPPING
+ if force_reload or OS_METADATA_MAPPING is None:
+ logging.info('load os metadatas into memory')
+ OS_METADATA_MAPPING = metadata_api.get_oses_metadata_internal(
+ force_reload=force_reload
+ )
+
+
+def _load_flavor_metadata_ui_converters(force_reload=False):
+ """Load flavor metadata ui converters from inmemory db.
+
+ The loaded metadata is mapped by flavor id.
+ """
+ global FLAVOR_METADATA_UI_CONVERTERS
+ if force_reload or FLAVOR_METADATA_UI_CONVERTERS is None:
+ logging.info('load flavor metadata ui converters into memory')
+ FLAVOR_METADATA_UI_CONVERTERS = {}
+ adapters_flavors_metadata_ui_converters = (
+ metadata_api.get_flavors_metadata_ui_converters_internal(
+ force_reload=force_reload
+ )
+ )
+ for adapter_name, adapter_flavors_metadata_ui_converters in (
+ adapters_flavors_metadata_ui_converters.items()
+ ):
+ for flavor_name, flavor_metadata_ui_converter in (
+ adapter_flavors_metadata_ui_converters.items()
+ ):
+ FLAVOR_METADATA_UI_CONVERTERS[
+ '%s:%s' % (adapter_name, flavor_name)
+ ] = flavor_metadata_ui_converter
+
+
+@util.deprecated
+def _load_package_metadatas(force_reload=False):
+ """Load deployable package metadata from inmemory db."""
+ global PACKAGE_METADATA_MAPPING
+ if force_reload or PACKAGE_METADATA_MAPPING is None:
+ logging.info('load package metadatas into memory')
+ PACKAGE_METADATA_MAPPING = (
+ metadata_api.get_packages_metadata_internal(
+ force_reload=force_reload
+ )
+ )
+
+
+def _load_flavor_metadatas(force_reload=False):
+ """Load flavor metadata from inmemory db.
+
+ The loaded metadata are mapped by flavor id.
+ """
+ global FLAVOR_METADATA_MAPPING
+ if force_reload or FLAVOR_METADATA_MAPPING is None:
+ logging.info('load flavor metadatas into memory')
+ FLAVOR_METADATA_MAPPING = {}
+ adapters_flavors_metadata = (
+ metadata_api.get_flavors_metadata_internal(
+ force_reload=force_reload
+ )
+ )
+ for adapter_name, adapter_flavors_metadata in (
+ adapters_flavors_metadata.items()
+ ):
+ for flavor_name, flavor_metadata in (
+ adapter_flavors_metadata.items()
+ ):
+ FLAVOR_METADATA_MAPPING[
+ '%s:%s' % (adapter_name, flavor_name)
+ ] = flavor_metadata
+
+
+OS_METADATA_MAPPING = None
+PACKAGE_METADATA_MAPPING = None
+FLAVOR_METADATA_MAPPING = None
+OS_METADATA_UI_CONVERTERS = None
+FLAVOR_METADATA_UI_CONVERTERS = None
+
+
+def validate_os_config(
+ config, os_id, whole_check=False, **kwargs
+):
+ """Validate os config."""
+ load_metadatas()
+ if os_id not in OS_METADATA_MAPPING:
+ raise exception.InvalidParameter(
+ 'os %s is not found in os metadata mapping' % os_id
+ )
+ _validate_config(
+ '', config, OS_METADATA_MAPPING[os_id],
+ whole_check, **kwargs
+ )
+
+
+@util.deprecated
+def validate_package_config(
+ config, adapter_id, whole_check=False, **kwargs
+):
+ """Validate package config."""
+ load_metadatas()
+ if adapter_id not in PACKAGE_METADATA_MAPPING:
+ raise exception.InvalidParameter(
+ 'adapter %s is not found in package metedata mapping' % adapter_id
+ )
+ _validate_config(
+ '', config, PACKAGE_METADATA_MAPPING[adapter_id],
+ whole_check, **kwargs
+ )
+
+
+def validate_flavor_config(
+ config, flavor_id, whole_check=False, **kwargs
+):
+ """Validate flavor config."""
+ load_metadatas()
+ if not flavor_id:
+ logging.info('There is no flavor, skipping flavor validation...')
+ elif flavor_id not in FLAVOR_METADATA_MAPPING:
+ raise exception.InvalidParameter(
+ 'flavor %s is not found in flavor metedata mapping' % flavor_id
+ )
+ else:
+ _validate_config(
+ '', config, FLAVOR_METADATA_MAPPING[flavor_id],
+ whole_check, **kwargs
+ )
+
+
+def _filter_metadata(metadata, **kwargs):
+ """Filter metadata before return it to api.
+
+ Some metadata fields are not json compatible or
+ only used in db/api internally.
+ We should strip these fields out before return to api.
+ """
+ if not isinstance(metadata, dict):
+ return metadata
+ filtered_metadata = {}
+ for key, value in metadata.items():
+ if key == '_self':
+ filtered_metadata[key] = {
+ 'name': value['name'],
+ 'description': value.get('description', None),
+ 'default_value': value.get('default_value', None),
+ 'is_required': value.get('is_required', False),
+ 'required_in_whole_config': value.get(
+ 'required_in_whole_config', False),
+ 'js_validator': value.get('js_validator', None),
+ 'options': value.get('options', None),
+ 'required_in_options': value.get(
+ 'required_in_options', False),
+ 'field_type': value.get(
+ 'field_type_data', 'str'),
+ 'display_type': value.get('display_type', None),
+ 'mapping_to': value.get('mapping_to', None)
+ }
+ else:
+ filtered_metadata[key] = _filter_metadata(value, **kwargs)
+ return filtered_metadata
+
+
+@util.deprecated
+def _get_package_metadata(adapter_id):
+ """get package metadata."""
+ load_metadatas()
+ if adapter_id not in PACKAGE_METADATA_MAPPING:
+ raise exception.RecordNotExists(
+ 'adpater %s does not exist' % adapter_id
+ )
+ return _filter_metadata(
+ PACKAGE_METADATA_MAPPING[adapter_id]
+ )
+
+
+@util.deprecated
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_METADATA_FIELDS)
+def get_package_metadata(adapter_id, user=None, session=None, **kwargs):
+ """Get package metadata from adapter."""
+ return {
+ 'package_config': _get_package_metadata(adapter_id)
+ }
+
+
+def _get_flavor_metadata(flavor_id):
+ """get flavor metadata."""
+ load_metadatas()
+ if not flavor_id:
+ logging.info('There is no flavor id, skipping...')
+ elif flavor_id not in FLAVOR_METADATA_MAPPING:
+ raise exception.RecordNotExists(
+ 'flavor %s does not exist' % flavor_id
+ )
+ else:
+ return _filter_metadata(FLAVOR_METADATA_MAPPING[flavor_id])
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_METADATA_FIELDS)
+def get_flavor_metadata(flavor_id, user=None, session=None, **kwargs):
+ """Get flavor metadata by flavor."""
+ return {
+ 'package_config': _get_flavor_metadata(flavor_id)
+ }
+
+
+def _get_os_metadata(os_id):
+ """get os metadata."""
+ load_metadatas()
+ if os_id not in OS_METADATA_MAPPING:
+ raise exception.RecordNotExists(
+ 'os %s does not exist' % os_id
+ )
+ return _filter_metadata(OS_METADATA_MAPPING[os_id])
+
+
+def _get_os_metadata_ui_converter(os_id):
+ """get os metadata ui converter."""
+ load_metadatas()
+ if os_id not in OS_METADATA_UI_CONVERTERS:
+ raise exception.RecordNotExists(
+ 'os %s does not exist' % os_id
+ )
+ return OS_METADATA_UI_CONVERTERS[os_id]
+
+
+def _get_flavor_metadata_ui_converter(flavor_id):
+ """get flavor metadata ui converter."""
+ load_metadatas()
+ if flavor_id not in FLAVOR_METADATA_UI_CONVERTERS:
+ raise exception.RecordNotExists(
+ 'flavor %s does not exist' % flavor_id
+ )
+ return FLAVOR_METADATA_UI_CONVERTERS[flavor_id]
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_METADATA_FIELDS)
+def get_os_metadata(os_id, user=None, session=None, **kwargs):
+ """get os metadatas."""
+ return {'os_config': _get_os_metadata(os_id)}
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_UI_METADATA_FIELDS)
+def get_os_ui_metadata(os_id, user=None, session=None, **kwargs):
+ """Get os metadata ui converter by os."""
+ metadata = _get_os_metadata(os_id)
+ metadata_ui_converter = _get_os_metadata_ui_converter(os_id)
+ return _get_ui_metadata(metadata, metadata_ui_converter)
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_UI_METADATA_FIELDS)
+def get_flavor_ui_metadata(flavor_id, user=None, session=None, **kwargs):
+ """Get flavor ui metadata by flavor."""
+ metadata = _get_flavor_metadata(flavor_id)
+ metadata_ui_converter = _get_flavor_metadata_ui_converter(flavor_id)
+ return _get_ui_metadata(metadata, metadata_ui_converter)
+
+
+def _get_ui_metadata(metadata, metadata_ui_converter):
+ """convert metadata to ui metadata.
+
+ Args:
+ metadata: metadata we defined in metadata files.
+ metadata_ui_converter: metadata ui converter defined in metadata
+ mapping files. Used to convert orignal
+ metadata to ui understandable metadata.
+
+ Returns:
+ ui understandable metadata.
+ """
+ ui_metadata = {}
+ ui_metadata[metadata_ui_converter['mapped_name']] = []
+ for mapped_child in metadata_ui_converter['mapped_children']:
+ data_dict = {}
+ for ui_key, ui_value in mapped_child.items():
+ for key, value in ui_value.items():
+ if 'data' == key:
+ result_data = []
+ _get_ui_metadata_data(
+ metadata[ui_key], value, result_data
+ )
+ data_dict['data'] = result_data
+ else:
+ data_dict[key] = value
+ ui_metadata[metadata_ui_converter['mapped_name']].append(data_dict)
+ return ui_metadata
+
+
+def _get_ui_metadata_data(metadata, config, result_data):
+ """Get ui metadata data and fill to result."""
+ data_dict = {}
+ for key, config_value in config.items():
+ if isinstance(config_value, dict) and key != 'content_data':
+ if key in metadata.keys():
+ _get_ui_metadata_data(metadata[key], config_value, result_data)
+ else:
+ _get_ui_metadata_data(metadata, config_value, result_data)
+ elif isinstance(config_value, list):
+ option_list = []
+ for item in config_value:
+ if isinstance(item, dict):
+ option_list.append(item)
+ data_dict[key] = option_list
+ else:
+ if isinstance(metadata['_self'][item], bool):
+ data_dict[item] = str(metadata['_self'][item]).lower()
+ else:
+ data_dict[item] = metadata['_self'][item]
+ else:
+ data_dict[key] = config_value
+ if data_dict:
+ result_data.append(data_dict)
+ return result_data
+
+
+@util.deprecated
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_METADATA_FIELDS)
+def get_package_os_metadata(
+ adapter_id, os_id,
+ user=None, session=None, **kwargs
+):
+ """Get metadata by adapter and os."""
+ adapter = adapter_holder_api.get_adapter(
+ adapter_id, user=user, session=session
+ )
+ os_ids = [os['id'] for os in adapter['supported_oses']]
+ if os_id not in os_ids:
+ raise exception.InvalidParameter(
+ 'os %s is not in the supported os list of adapter %s' % (
+ os_id, adapter_id
+ )
+ )
+ metadatas = {}
+ metadatas['os_config'] = _get_os_metadata(
+ os_id
+ )
+ metadatas['package_config'] = _get_package_metadata(
+ adapter_id
+ )
+ return metadatas
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_METADATA_FIELDS)
+def get_flavor_os_metadata(
+ flavor_id, os_id,
+ user=None, session=None, **kwargs
+):
+ """Get metadata by flavor and os."""
+ flavor = adapter_holder_api.get_flavor(
+ flavor_id, user=user, session=session
+ )
+ adapter_id = flavor['adapter_id']
+ adapter = adapter_holder_api.get_adapter(
+ adapter_id, user=user, session=session
+ )
+ os_ids = [os['id'] for os in adapter['supported_oses']]
+ if os_id not in os_ids:
+ raise exception.InvalidParameter(
+ 'os %s is not in the supported os list of adapter %s' % (
+ os_id, adapter_id
+ )
+ )
+ metadatas = {}
+ metadatas['os_config'] = _get_os_metadata(
+ session, os_id
+ )
+ metadatas['package_config'] = _get_flavor_metadata(
+ session, flavor_id
+ )
+ return metadatas
+
+
+def _validate_self(
+ config_path, config_key, config,
+ metadata, whole_check,
+ **kwargs
+):
+ """validate config by metadata self section."""
+ logging.debug('validate config self %s', config_path)
+ if '_self' not in metadata:
+ if isinstance(config, dict):
+ _validate_config(
+ config_path, config, metadata, whole_check, **kwargs
+ )
+ return
+ field_type = metadata['_self'].get('field_type', basestring)
+ if not isinstance(config, field_type):
+ raise exception.InvalidParameter(
+ '%s config type is not %s: %s' % (config_path, field_type, config)
+ )
+ is_required = metadata['_self'].get(
+ 'is_required', False
+ )
+ required_in_whole_config = metadata['_self'].get(
+ 'required_in_whole_config', False
+ )
+ if isinstance(config, basestring):
+ if config == '' and not is_required and not required_in_whole_config:
+ # ignore empty config when it is optional
+ return
+ required_in_options = metadata['_self'].get(
+ 'required_in_options', False
+ )
+ options = metadata['_self'].get('options', None)
+ if required_in_options:
+ if field_type in [int, basestring, float, bool]:
+ if options and config not in options:
+ raise exception.InvalidParameter(
+ '%s config is not in %s: %s' % (
+ config_path, options, config
+ )
+ )
+ elif field_type in [list, tuple]:
+ if options and not set(config).issubset(set(options)):
+ raise exception.InvalidParameter(
+ '%s config is not in %s: %s' % (
+ config_path, options, config
+ )
+ )
+ elif field_type == dict:
+ if options and not set(config.keys()).issubset(set(options)):
+ raise exception.InvalidParameter(
+ '%s config is not in %s: %s' % (
+ config_path, options, config
+ )
+ )
+ validator = metadata['_self'].get('validator', None)
+ logging.debug('validate by validator %s', validator)
+ if validator:
+ if not validator(config_key, config, **kwargs):
+ raise exception.InvalidParameter(
+ '%s config is invalid' % config_path
+ )
+ if isinstance(config, dict):
+ _validate_config(
+ config_path, config, metadata, whole_check, **kwargs
+ )
+
+
+def _validate_config(
+ config_path, config, metadata, whole_check,
+ **kwargs
+):
+ """validate config by metadata."""
+ logging.debug('validate config %s', config_path)
+ generals = {}
+ specified = {}
+ for key, value in metadata.items():
+ if key.startswith('$'):
+ generals[key] = value
+ elif key.startswith('_'):
+ pass
+ else:
+ specified[key] = value
+ config_keys = set(config.keys())
+ specified_keys = set(specified.keys())
+ intersect_keys = config_keys & specified_keys
+ not_found_keys = config_keys - specified_keys
+ redundant_keys = specified_keys - config_keys
+ for key in redundant_keys:
+ if '_self' not in specified[key]:
+ continue
+ if specified[key]['_self'].get('is_required', False):
+ raise exception.InvalidParameter(
+ '%s/%s does not find but it is required' % (
+ config_path, key
+ )
+ )
+ if (
+ whole_check and
+ specified[key]['_self'].get(
+ 'required_in_whole_config', False
+ )
+ ):
+ raise exception.InvalidParameter(
+ '%s/%s does not find but it is required in whole config' % (
+ config_path, key
+ )
+ )
+ for key in intersect_keys:
+ _validate_self(
+ '%s/%s' % (config_path, key),
+ key, config[key], specified[key], whole_check,
+ **kwargs
+ )
+ for key in not_found_keys:
+ if not generals:
+ raise exception.InvalidParameter(
+ 'key %s missing in metadata %s' % (
+ key, config_path
+ )
+ )
+ for general_key, general_value in generals.items():
+ _validate_self(
+ '%s/%s' % (config_path, key),
+ key, config[key], general_value, whole_check,
+ **kwargs
+ )
+
+
+def _autofill_self_config(
+ config_path, config_key, config,
+ metadata,
+ **kwargs
+):
+ """Autofill config by metadata self section."""
+ if '_self' not in metadata:
+ if isinstance(config, dict):
+ _autofill_config(
+ config_path, config, metadata, **kwargs
+ )
+ return config
+ logging.debug(
+ 'autofill %s by metadata %s', config_path, metadata['_self']
+ )
+ autofill_callback = metadata['_self'].get(
+ 'autofill_callback', None
+ )
+ autofill_callback_params = metadata['_self'].get(
+ 'autofill_callback_params', {}
+ )
+ callback_params = dict(kwargs)
+ if autofill_callback_params:
+ callback_params.update(autofill_callback_params)
+ default_value = metadata['_self'].get(
+ 'default_value', None
+ )
+ if default_value is not None:
+ callback_params['default_value'] = default_value
+ options = metadata['_self'].get(
+ 'options', None
+ )
+ if options is not None:
+ callback_params['options'] = options
+ if autofill_callback:
+ config = autofill_callback(
+ config_key, config, **callback_params
+ )
+ if config is None:
+ new_config = {}
+ else:
+ new_config = config
+ if isinstance(new_config, dict):
+ _autofill_config(
+ config_path, new_config, metadata, **kwargs
+ )
+ if new_config:
+ config = new_config
+ return config
+
+
+def _autofill_config(
+ config_path, config, metadata, **kwargs
+):
+ """autofill config by metadata."""
+ generals = {}
+ specified = {}
+ for key, value in metadata.items():
+ if key.startswith('$'):
+ generals[key] = value
+ elif key.startswith('_'):
+ pass
+ else:
+ specified[key] = value
+ config_keys = set(config.keys())
+ specified_keys = set(specified.keys())
+ intersect_keys = config_keys & specified_keys
+ not_found_keys = config_keys - specified_keys
+ redundant_keys = specified_keys - config_keys
+ for key in redundant_keys:
+ self_config = _autofill_self_config(
+ '%s/%s' % (config_path, key),
+ key, None, specified[key], **kwargs
+ )
+ if self_config is not None:
+ config[key] = self_config
+ for key in intersect_keys:
+ config[key] = _autofill_self_config(
+ '%s/%s' % (config_path, key),
+ key, config[key], specified[key],
+ **kwargs
+ )
+ for key in not_found_keys:
+ for general_key, general_value in generals.items():
+ config[key] = _autofill_self_config(
+ '%s/%s' % (config_path, key),
+ key, config[key], general_value,
+ **kwargs
+ )
+ return config
+
+
+def autofill_os_config(
+ config, os_id, **kwargs
+):
+ load_metadatas()
+ if os_id not in OS_METADATA_MAPPING:
+ raise exception.InvalidParameter(
+ 'os %s is not found in os metadata mapping' % os_id
+ )
+
+ return _autofill_config(
+ '', config, OS_METADATA_MAPPING[os_id], **kwargs
+ )
+
+
+def autofill_package_config(
+ config, adapter_id, **kwargs
+):
+ load_metadatas()
+ if adapter_id not in PACKAGE_METADATA_MAPPING:
+ raise exception.InvalidParameter(
+ 'adapter %s is not found in package metadata mapping' % adapter_id
+ )
+
+ return _autofill_config(
+ '', config, PACKAGE_METADATA_MAPPING[adapter_id], **kwargs
+ )
+
+
+def autofill_flavor_config(
+ config, flavor_id, **kwargs
+):
+ load_metadatas()
+ if not flavor_id:
+ logging.info('There is no flavor, skipping...')
+ elif flavor_id not in FLAVOR_METADATA_MAPPING:
+ raise exception.InvalidParameter(
+ 'flavor %s is not found in flavor metadata mapping' % flavor_id
+ )
+ else:
+ return _autofill_config(
+ '', config, FLAVOR_METADATA_MAPPING[flavor_id], **kwargs
+ )
diff --git a/compass-tasks/db/api/network.py b/compass-tasks/db/api/network.py
new file mode 100644
index 0000000..e2bf7d3
--- /dev/null
+++ b/compass-tasks/db/api/network.py
@@ -0,0 +1,160 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Network related database operations."""
+import logging
+import netaddr
+import re
+
+from compass.db.api import database
+from compass.db.api import permission
+from compass.db.api import user as user_api
+from compass.db.api import utils
+from compass.db import exception
+from compass.db import models
+
+
+SUPPORTED_FIELDS = ['subnet', 'name']
+RESP_FIELDS = [
+ 'id', 'name', 'subnet', 'created_at', 'updated_at'
+]
+ADDED_FIELDS = ['subnet']
+OPTIONAL_ADDED_FIELDS = ['name']
+IGNORE_FIELDS = [
+ 'id', 'created_at', 'updated_at'
+]
+UPDATED_FIELDS = ['subnet', 'name']
+
+
+def _check_subnet(subnet):
+ """Check subnet format is correct."""
+ try:
+ netaddr.IPNetwork(subnet)
+ except Exception as error:
+ logging.exception(error)
+ raise exception.InvalidParameter(
+ 'subnet %s format unrecognized' % subnet)
+
+
+@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_SUBNETS
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def list_subnets(user=None, session=None, **filters):
+ """List subnets."""
+ return utils.list_db_objects(
+ session, models.Subnet, **filters
+ )
+
+
+def _get_subnet(subnet_id, session=None, **kwargs):
+ """Get subnet by subnet id."""
+ if isinstance(subnet_id, (int, long)):
+ return utils.get_db_object(
+ session, models.Subnet,
+ id=subnet_id, **kwargs
+ )
+ raise exception.InvalidParameter(
+ 'subnet id %s type is not int compatible' % subnet_id
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_SUBNETS
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def get_subnet(
+ subnet_id, exception_when_missing=True,
+ user=None, session=None, **kwargs
+):
+ """Get subnet info."""
+ return _get_subnet(
+ subnet_id, session=session,
+ exception_when_missing=exception_when_missing
+ )
+
+
+@utils.supported_filters(
+ ADDED_FIELDS, optional_support_keys=OPTIONAL_ADDED_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(subnet=_check_subnet)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_SUBNET
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def add_subnet(
+ exception_when_existing=True, subnet=None,
+ user=None, session=None, **kwargs
+):
+ """Create a subnet."""
+ return utils.add_db_object(
+ session, models.Subnet,
+ exception_when_existing, subnet, **kwargs
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(subnet=_check_subnet)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_SUBNET
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def update_subnet(subnet_id, user=None, session=None, **kwargs):
+ """Update a subnet."""
+ subnet = _get_subnet(
+ subnet_id, session=session
+ )
+ return utils.update_db_object(session, subnet, **kwargs)
+
+
+def _check_subnet_deletable(subnet):
+ """Check a subnet deletable."""
+ if subnet.host_networks:
+ host_networks = [
+ '%s:%s=%s' % (
+ host_network.host.name, host_network.interface,
+ host_network.ip
+ )
+ for host_network in subnet.host_networks
+ ]
+ raise exception.NotAcceptable(
+ 'subnet %s contains host networks %s' % (
+ subnet.subnet, host_networks
+ )
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEL_SUBNET
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def del_subnet(subnet_id, user=None, session=None, **kwargs):
+ """Delete a subnet."""
+ subnet = _get_subnet(
+ subnet_id, session=session
+ )
+ _check_subnet_deletable(subnet)
+ return utils.del_db_object(session, subnet)
diff --git a/compass-tasks/db/api/permission.py b/compass-tasks/db/api/permission.py
new file mode 100644
index 0000000..f4d777a
--- /dev/null
+++ b/compass-tasks/db/api/permission.py
@@ -0,0 +1,357 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Permission database operations."""
+import re
+
+from compass.db.api import database
+from compass.db.api import user as user_api
+from compass.db.api import utils
+from compass.db import exception
+from compass.db import models
+from compass.utils import util
+
+
+SUPPORTED_FIELDS = ['id', 'name', 'alias', 'description']
+RESP_FIELDS = ['id', 'name', 'alias', 'description']
+
+
+class PermissionWrapper(object):
+ def __init__(self, name, alias, description):
+ self.name = name
+ self.alias = alias
+ self.description = description
+
+ def to_dict(self):
+ return {
+ 'name': self.name,
+ 'alias': self.alias,
+ 'description': self.description
+ }
+
+
+PERMISSION_LIST_PERMISSIONS = PermissionWrapper(
+ 'list_permissions', 'list permissions', 'list all permissions'
+)
+PERMISSION_LIST_SWITCHES = PermissionWrapper(
+ 'list_switches', 'list switches', 'list all switches'
+)
+PERMISSION_LIST_SWITCH_FILTERS = PermissionWrapper(
+ 'list_switch_filters',
+ 'list switch filters',
+ 'list switch filters'
+)
+PERMISSION_ADD_SWITCH = PermissionWrapper(
+ 'add_switch', 'add switch', 'add switch'
+)
+PERMISSION_UPDATE_SWITCH_FILTERS = PermissionWrapper(
+ 'update_switch_filters',
+ 'update switch filters',
+ 'update switch filters'
+)
+PERMISSION_DEL_SWITCH = PermissionWrapper(
+ 'delete_switch', 'delete switch', 'delete switch'
+)
+PERMISSION_LIST_SWITCH_MACHINES = PermissionWrapper(
+ 'list_switch_machines', 'list switch machines', 'list switch machines'
+)
+PERMISSION_ADD_SWITCH_MACHINE = PermissionWrapper(
+ 'add_switch_machine', 'add switch machine', 'add switch machine'
+)
+PERMISSION_DEL_SWITCH_MACHINE = PermissionWrapper(
+ 'del_switch_machine', 'delete switch machine', 'del switch machine'
+)
+PERMISSION_UPDATE_SWITCH_MACHINES = PermissionWrapper(
+ 'update_switch_machines',
+ 'update switch machines',
+ 'update switch machines'
+)
+PERMISSION_LIST_MACHINES = PermissionWrapper(
+ 'list_machines', 'list machines', 'list machines'
+)
+PERMISSION_ADD_MACHINE = PermissionWrapper(
+ 'add_machine', 'add machine', 'add machine'
+)
+PERMISSION_DEL_MACHINE = PermissionWrapper(
+ 'delete_machine', 'delete machine', 'delete machine'
+)
+PERMISSION_LIST_ADAPTERS = PermissionWrapper(
+ 'list_adapters', 'list adapters', 'list adapters'
+)
+PERMISSION_LIST_METADATAS = PermissionWrapper(
+ 'list_metadatas', 'list metadatas', 'list metadatas'
+)
+PERMISSION_LIST_SUBNETS = PermissionWrapper(
+ 'list_subnets', 'list subnets', 'list subnets'
+)
+PERMISSION_ADD_SUBNET = PermissionWrapper(
+ 'add_subnet', 'add subnet', 'add subnet'
+)
+PERMISSION_DEL_SUBNET = PermissionWrapper(
+ 'del_subnet', 'del subnet', 'del subnet'
+)
+PERMISSION_LIST_CLUSTERS = PermissionWrapper(
+ 'list_clusters', 'list clusters', 'list clusters'
+)
+PERMISSION_ADD_CLUSTER = PermissionWrapper(
+ 'add_cluster', 'add cluster', 'add cluster'
+)
+PERMISSION_DEL_CLUSTER = PermissionWrapper(
+ 'del_cluster', 'del cluster', 'del cluster'
+)
+PERMISSION_LIST_CLUSTER_CONFIG = PermissionWrapper(
+ 'list_cluster_config', 'list cluster config', 'list cluster config'
+)
+PERMISSION_ADD_CLUSTER_CONFIG = PermissionWrapper(
+ 'add_cluster_config', 'add cluster config', 'add cluster config'
+)
+PERMISSION_DEL_CLUSTER_CONFIG = PermissionWrapper(
+ 'del_cluster_config', 'del cluster config', 'del cluster config'
+)
+PERMISSION_UPDATE_CLUSTER_HOSTS = PermissionWrapper(
+ 'update_cluster_hosts',
+ 'update cluster hosts',
+ 'update cluster hosts'
+)
+PERMISSION_DEL_CLUSTER_HOST = PermissionWrapper(
+ 'del_clusterhost', 'delete clusterhost', 'delete clusterhost'
+)
+PERMISSION_REVIEW_CLUSTER = PermissionWrapper(
+ 'review_cluster', 'review cluster', 'review cluster'
+)
+PERMISSION_DEPLOY_CLUSTER = PermissionWrapper(
+ 'deploy_cluster', 'deploy cluster', 'deploy cluster'
+)
+PERMISSION_DEPLOY_HOST = PermissionWrapper(
+ 'deploy_host', 'deploy host', 'deploy host'
+)
+PERMISSION_GET_CLUSTER_STATE = PermissionWrapper(
+ 'get_cluster_state', 'get cluster state', 'get cluster state'
+)
+PERMISSION_UPDATE_CLUSTER_STATE = PermissionWrapper(
+ 'update_cluster_state', 'update cluster state',
+ 'update cluster state'
+)
+PERMISSION_LIST_HOSTS = PermissionWrapper(
+ 'list_hosts', 'list hosts', 'list hosts'
+)
+PERMISSION_LIST_HOST_CLUSTERS = PermissionWrapper(
+ 'list_host_clusters',
+ 'list host clusters',
+ 'list host clusters'
+)
+PERMISSION_UPDATE_HOST = PermissionWrapper(
+ 'update_host', 'update host', 'update host'
+)
+PERMISSION_DEL_HOST = PermissionWrapper(
+ 'del_host', 'del host', 'del host'
+)
+PERMISSION_LIST_HOST_CONFIG = PermissionWrapper(
+ 'list_host_config', 'list host config', 'list host config'
+)
+PERMISSION_ADD_HOST_CONFIG = PermissionWrapper(
+ 'add_host_config', 'add host config', 'add host config'
+)
+PERMISSION_DEL_HOST_CONFIG = PermissionWrapper(
+ 'del_host_config', 'del host config', 'del host config'
+)
+PERMISSION_LIST_HOST_NETWORKS = PermissionWrapper(
+ 'list_host_networks',
+ 'list host networks',
+ 'list host networks'
+)
+PERMISSION_ADD_HOST_NETWORK = PermissionWrapper(
+ 'add_host_network', 'add host network', 'add host network'
+)
+PERMISSION_DEL_HOST_NETWORK = PermissionWrapper(
+ 'del_host_network', 'del host network', 'del host network'
+)
+PERMISSION_GET_HOST_STATE = PermissionWrapper(
+ 'get_host_state', 'get host state', 'get host state'
+)
+PERMISSION_UPDATE_HOST_STATE = PermissionWrapper(
+ 'update_host_state', 'update host sate', 'update host state'
+)
+PERMISSION_LIST_CLUSTERHOSTS = PermissionWrapper(
+ 'list_clusterhosts', 'list cluster hosts', 'list cluster hosts'
+)
+PERMISSION_LIST_CLUSTERHOST_CONFIG = PermissionWrapper(
+ 'list_clusterhost_config',
+ 'list clusterhost config',
+ 'list clusterhost config'
+)
+PERMISSION_ADD_CLUSTERHOST_CONFIG = PermissionWrapper(
+ 'add_clusterhost_config',
+ 'add clusterhost config',
+ 'add clusterhost config'
+)
+PERMISSION_DEL_CLUSTERHOST_CONFIG = PermissionWrapper(
+ 'del_clusterhost_config',
+ 'del clusterhost config',
+ 'del clusterhost config'
+)
+PERMISSION_GET_CLUSTERHOST_STATE = PermissionWrapper(
+ 'get_clusterhost_state',
+ 'get clusterhost state',
+ 'get clusterhost state'
+)
+PERMISSION_UPDATE_CLUSTERHOST_STATE = PermissionWrapper(
+ 'update_clusterhost_state',
+ 'update clusterhost state',
+ 'update clusterhost state'
+)
+PERMISSION_LIST_HEALTH_REPORT = PermissionWrapper(
+ 'list_health_reports',
+ 'list health check report',
+ 'list health check report'
+)
+PERMISSION_GET_HEALTH_REPORT = PermissionWrapper(
+ 'get_health_report',
+ 'get health report',
+ 'get health report'
+)
+PERMISSION_CHECK_CLUSTER_HEALTH = PermissionWrapper(
+ 'start_check_cluster_health',
+ 'start check cluster health',
+ 'start check cluster health'
+)
+PERMISSION_SET_HEALTH_CHECK_ERROR = PermissionWrapper(
+ 'set_error_state',
+ 'set health check into error state',
+ 'set health check into error state'
+)
+PERMISSION_DELETE_REPORT = PermissionWrapper(
+ 'delete_reports',
+ 'delete health reports',
+ 'delete health reports'
+)
+PERMISSIONS = [
+ PERMISSION_LIST_PERMISSIONS,
+ PERMISSION_LIST_SWITCHES,
+ PERMISSION_ADD_SWITCH,
+ PERMISSION_DEL_SWITCH,
+ PERMISSION_LIST_SWITCH_FILTERS,
+ PERMISSION_UPDATE_SWITCH_FILTERS,
+ PERMISSION_LIST_SWITCH_MACHINES,
+ PERMISSION_ADD_SWITCH_MACHINE,
+ PERMISSION_DEL_SWITCH_MACHINE,
+ PERMISSION_UPDATE_SWITCH_MACHINES,
+ PERMISSION_LIST_MACHINES,
+ PERMISSION_ADD_MACHINE,
+ PERMISSION_DEL_MACHINE,
+ PERMISSION_LIST_ADAPTERS,
+ PERMISSION_LIST_METADATAS,
+ PERMISSION_LIST_SUBNETS,
+ PERMISSION_ADD_SUBNET,
+ PERMISSION_DEL_SUBNET,
+ PERMISSION_LIST_CLUSTERS,
+ PERMISSION_ADD_CLUSTER,
+ PERMISSION_DEL_CLUSTER,
+ PERMISSION_LIST_CLUSTER_CONFIG,
+ PERMISSION_ADD_CLUSTER_CONFIG,
+ PERMISSION_DEL_CLUSTER_CONFIG,
+ PERMISSION_UPDATE_CLUSTER_HOSTS,
+ PERMISSION_DEL_CLUSTER_HOST,
+ PERMISSION_REVIEW_CLUSTER,
+ PERMISSION_DEPLOY_CLUSTER,
+ PERMISSION_GET_CLUSTER_STATE,
+ PERMISSION_UPDATE_CLUSTER_STATE,
+ PERMISSION_LIST_HOSTS,
+ PERMISSION_LIST_HOST_CLUSTERS,
+ PERMISSION_UPDATE_HOST,
+ PERMISSION_DEL_HOST,
+ PERMISSION_LIST_HOST_CONFIG,
+ PERMISSION_ADD_HOST_CONFIG,
+ PERMISSION_DEL_HOST_CONFIG,
+ PERMISSION_LIST_HOST_NETWORKS,
+ PERMISSION_ADD_HOST_NETWORK,
+ PERMISSION_DEL_HOST_NETWORK,
+ PERMISSION_GET_HOST_STATE,
+ PERMISSION_UPDATE_HOST_STATE,
+ PERMISSION_DEPLOY_HOST,
+ PERMISSION_LIST_CLUSTERHOSTS,
+ PERMISSION_LIST_CLUSTERHOST_CONFIG,
+ PERMISSION_ADD_CLUSTERHOST_CONFIG,
+ PERMISSION_DEL_CLUSTERHOST_CONFIG,
+ PERMISSION_GET_CLUSTERHOST_STATE,
+ PERMISSION_UPDATE_CLUSTERHOST_STATE,
+ PERMISSION_LIST_HEALTH_REPORT,
+ PERMISSION_GET_HEALTH_REPORT,
+ PERMISSION_CHECK_CLUSTER_HEALTH,
+ PERMISSION_SET_HEALTH_CHECK_ERROR,
+ PERMISSION_DELETE_REPORT
+]
+
+
+@util.deprecated
+def list_permissions_internal(session, **filters):
+ """internal functions used only by other db.api modules."""
+ return utils.list_db_objects(session, models.Permission, **filters)
+
+
+@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS)
+@database.run_in_session()
+@user_api.check_user_permission(PERMISSION_LIST_PERMISSIONS)
+@utils.wrap_to_dict(RESP_FIELDS)
+def list_permissions(user=None, session=None, **filters):
+ """list permissions."""
+ return utils.list_db_objects(
+ session, models.Permission, **filters
+ )
+
+
+def _get_permission(permission_id, session=None, **kwargs):
+ """Get permission object by the unique key of Permission table."""
+ if isinstance(permission_id, (int, long)):
+ return utils.get_db_object(
+ session, models.Permission, id=permission_id, **kwargs)
+ raise exception.InvalidParameter(
+ 'permission id %s type is not int compatible' % permission_id
+ )
+
+
+def get_permission_internal(permission_id, session=None, **kwargs):
+ return _get_permission(permission_id, session=session, **kwargs)
+
+
+@utils.supported_filters()
+@database.run_in_session()
+@user_api.check_user_permission(PERMISSION_LIST_PERMISSIONS)
+@utils.wrap_to_dict(RESP_FIELDS)
+def get_permission(
+ permission_id, exception_when_missing=True,
+ user=None, session=None, **kwargs
+):
+ """get permissions."""
+ return _get_permission(
+ permission_id, session=session,
+ exception_when_missing=exception_when_missing
+ )
+
+
+def add_permissions_internal(session=None):
+ """internal functions used by other db.api modules only."""
+ permissions = []
+ for permission in PERMISSIONS:
+ permissions.append(
+ utils.add_db_object(
+ session, models.Permission,
+ True,
+ permission.name,
+ alias=permission.alias,
+ description=permission.description
+ )
+ )
+
+ return permissions
diff --git a/compass-tasks/db/api/switch.py b/compass-tasks/db/api/switch.py
new file mode 100644
index 0000000..647eec0
--- /dev/null
+++ b/compass-tasks/db/api/switch.py
@@ -0,0 +1,1213 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Switch database operations."""
+import logging
+import netaddr
+import re
+
+from compass.db.api import database
+from compass.db.api import permission
+from compass.db.api import user as user_api
+from compass.db.api import utils
+from compass.db import exception
+from compass.db import models
+from compass.utils import setting_wrapper as setting
+from compass.utils import util
+
+
+SUPPORTED_FIELDS = ['ip_int', 'vendor', 'state']
+SUPPORTED_FILTER_FIELDS = ['ip_int', 'vendor', 'state']
+SUPPORTED_SWITCH_MACHINES_FIELDS = [
+ 'switch_ip_int', 'port', 'vlans', 'mac', 'tag', 'location',
+ 'owner_id'
+]
+SUPPORTED_MACHINES_FIELDS = [
+ 'port', 'vlans', 'mac', 'tag', 'location', 'owner_id'
+]
+SUPPORTED_SWITCH_MACHINES_HOSTS_FIELDS = [
+ 'switch_ip_int', 'port', 'vlans', 'mac',
+ 'tag', 'location', 'os_name'
+]
+SUPPORTED_MACHINES_HOSTS_FIELDS = [
+ 'port', 'vlans', 'mac', 'tag', 'location',
+ 'os_name'
+]
+IGNORE_FIELDS = ['id', 'created_at', 'updated_at']
+ADDED_FIELDS = ['ip']
+OPTIONAL_ADDED_FIELDS = [
+ 'credentials', 'vendor', 'state', 'err_msg', 'machine_filters'
+]
+UPDATED_FIELDS = [
+ 'ip', 'credentials', 'vendor', 'state',
+ 'err_msg', 'put_machine_filters'
+]
+PATCHED_FIELDS = ['patched_credentials', 'patched_machine_filters']
+UPDATED_FILTERS_FIELDS = ['put_machine_filters']
+PATCHED_FILTERS_FIELDS = ['patched_machine_filters']
+ADDED_MACHINES_FIELDS = ['mac']
+OPTIONAL_ADDED_MACHINES_FIELDS = [
+ 'ipmi_credentials', 'tag', 'location', 'owner_id'
+]
+ADDED_SWITCH_MACHINES_FIELDS = ['port']
+OPTIONAL_ADDED_SWITCH_MACHINES_FIELDS = ['vlans']
+UPDATED_MACHINES_FIELDS = [
+ 'ipmi_credentials',
+ 'tag', 'location'
+]
+UPDATED_SWITCH_MACHINES_FIELDS = ['port', 'vlans', 'owner_id']
+PATCHED_MACHINES_FIELDS = [
+ 'patched_ipmi_credentials',
+ 'patched_tag', 'patched_location'
+]
+PATCHED_SWITCH_MACHINES_FIELDS = ['patched_vlans']
+RESP_FIELDS = [
+ 'id', 'ip', 'credentials', 'vendor', 'state', 'err_msg',
+ 'filters', 'created_at', 'updated_at'
+]
+RESP_FILTERS_FIELDS = [
+ 'id', 'ip', 'filters', 'created_at', 'updated_at'
+]
+RESP_ACTION_FIELDS = [
+ 'status', 'details'
+]
+RESP_MACHINES_FIELDS = [
+ 'id', 'switch_id', 'switch_ip', 'machine_id', 'switch_machine_id',
+ 'port', 'vlans', 'mac', 'owner_id',
+ 'ipmi_credentials', 'tag', 'location',
+ 'created_at', 'updated_at'
+]
+RESP_MACHINES_HOSTS_FIELDS = [
+ 'id', 'switch_id', 'switch_ip', 'machine_id', 'switch_machine_id',
+ 'port', 'vlans', 'mac',
+ 'ipmi_credentials', 'tag', 'location', 'ip',
+ 'name', 'hostname', 'os_name', 'owner',
+ 'os_installer', 'reinstall_os', 'os_installed',
+ 'clusters', 'created_at', 'updated_at'
+]
+RESP_CLUSTER_FIELDS = [
+ 'name', 'id'
+]
+
+
+def _check_machine_filters(machine_filters):
+ """Check if machine filters format is acceptable."""
+ logging.debug('check machine filters: %s', machine_filters)
+ models.Switch.parse_filters(machine_filters)
+
+
+def _check_vlans(vlans):
+ """Check vlans format is acceptable."""
+ for vlan in vlans:
+ if not isinstance(vlan, int):
+ raise exception.InvalidParameter(
+ 'vlan %s is not int' % vlan
+ )
+
+
+@utils.supported_filters(
+ ADDED_FIELDS,
+ optional_support_keys=OPTIONAL_ADDED_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(
+ ip=utils.check_ip,
+ credentials=utils.check_switch_credentials,
+ machine_filters=_check_machine_filters
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def _add_switch(
+ ip, exception_when_existing=True,
+ machine_filters=setting.SWITCHES_DEFAULT_FILTERS,
+ session=None, **kwargs
+):
+ """Add switch by switch ip."""
+ ip_int = long(netaddr.IPAddress(ip))
+ return utils.add_db_object(
+ session, models.Switch, exception_when_existing, ip_int,
+ machine_filters=machine_filters, **kwargs
+ )
+
+
+def get_switch_internal(
+ switch_id, session=None, **kwargs
+):
+ """Get switch by switch id.
+
+ Should only be used by other files under db/api
+ """
+ return _get_switch(switch_id, session=session, **kwargs)
+
+
+def _get_switch(switch_id, session=None, **kwargs):
+ """Get Switch object switch id."""
+ if isinstance(switch_id, (int, long)):
+ return utils.get_db_object(
+ session, models.Switch,
+ id=switch_id, **kwargs
+ )
+ raise exception.InvalidParameter(
+ 'switch id %s type is not int compatible' % switch_id)
+
+
+def _get_switch_by_ip(switch_ip, session=None, **kwargs):
+ """Get switch by switch ip."""
+ switch_ip_int = long(netaddr.IPAddress(switch_ip))
+ return utils.get_db_object(
+ session, models.Switch,
+ ip_int=switch_ip_int, **kwargs
+ )
+
+
+def _get_switch_machine(switch_id, machine_id, session=None, **kwargs):
+ """Get switch machine by switch id and machine id."""
+ switch = _get_switch(switch_id, session=session)
+ from compass.db.api import machine as machine_api
+ machine = machine_api.get_machine_internal(machine_id, session=session)
+ return utils.get_db_object(
+ session, models.SwitchMachine,
+ switch_id=switch.id, machine_id=machine.id, **kwargs
+ )
+
+
+def _get_switchmachine(switch_machine_id, session=None, **kwargs):
+ """Get switch machine by switch_machine_id."""
+ if not isinstance(switch_machine_id, (int, long)):
+ raise exception.InvalidParameter(
+ 'switch machine id %s type is not int compatible' % (
+ switch_machine_id
+ )
+ )
+ return utils.get_db_object(
+ session, models.SwitchMachine,
+ switch_machine_id=switch_machine_id, **kwargs
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_SWITCHES
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def get_switch(
+ switch_id, exception_when_missing=True,
+ user=None, session=None, **kwargs
+):
+ """get a switch by switch id."""
+ return _get_switch(
+ switch_id, session=session,
+ exception_when_missing=exception_when_missing
+ )
+
+
+@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_SWITCHES
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def list_switches(user=None, session=None, **filters):
+ """List switches."""
+ # TODO(xicheng): should discuss with weidong.
+ # If we can deprecate the use of DEFAULT_SWITCH_IP,
+ # The code will be simpler.
+ # The UI should use /machines-hosts instead of
+ # /switches-machines-hosts and can show multi switch ip/port
+ # under one row of machine info.
+ switches = utils.list_db_objects(
+ session, models.Switch, **filters
+ )
+ if 'ip_int' in filters:
+ return switches
+ else:
+ return [
+ switch for switch in switches
+ if switch.ip != setting.DEFAULT_SWITCH_IP
+ ]
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEL_SWITCH
+)
+@utils.wrap_to_dict(RESP_FIELDS)
+def del_switch(switch_id, user=None, session=None, **kwargs):
+ """Delete a switch.
+
+ If switch is not the default switch, and the machine under this switch
+ is only connected to this switch, the machine will be moved to connect
+ to default switch. Otherwise we can only simply delete the switch
+ machine. The purpose here to make sure there is no machine not
+ connecting to any switch.
+ """
+ # TODO(xicheng): Simplify the logic if the default switch feature
+ # can be deprecated.
+ switch = _get_switch(switch_id, session=session)
+ default_switch = _get_switch_by_ip(
+ setting.DEFAULT_SWITCH_IP, session=session
+ )
+ if switch.id != default_switch.id:
+ for switch_machine in switch.switch_machines:
+ machine = switch_machine.machine
+ if len(machine.switch_machines) <= 1:
+ utils.add_db_object(
+ session, models.SwitchMachine,
+ False,
+ default_switch.id, machine.id,
+ port=switch_machine.port
+ )
+ return utils.del_db_object(session, switch)
+
+
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_SWITCH
+)
+def add_switch(
+ exception_when_existing=True, ip=None,
+ user=None, session=None, **kwargs
+):
+ """Create a switch."""
+ return _add_switch(
+ ip,
+ exception_when_existing=exception_when_existing,
+ session=session, **kwargs
+ )
+
+
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_SWITCH
+)
+def add_switches(
+ exception_when_existing=False,
+ data=[], user=None, session=None
+):
+ """Create switches."""
+ # TODO(xicheng): simplify the batch api.
+ switches = []
+ fail_switches = []
+ for switch_data in data:
+ switch_object = _get_switch_by_ip(
+ switch_data['ip'], session=session,
+ exception_when_missing=False
+ )
+ if switch_object:
+ logging.error('ip %s exists in switch %s' % (
+ switch_data['ip'], switch_object.id
+ ))
+ fail_switches.append(switch_data)
+ else:
+ switches.append(
+ _add_switch(
+ exception_when_existing=exception_when_existing,
+ session=session,
+ **switch_data
+ )
+ )
+ return {
+ 'switches': switches,
+ 'fail_switches': fail_switches
+ }
+
+
+@utils.wrap_to_dict(RESP_FIELDS)
+def _update_switch(switch_id, session=None, **kwargs):
+ """Update a switch."""
+ switch = _get_switch(switch_id, session=session)
+ return utils.update_db_object(session, switch, **kwargs)
+
+
+# replace machine_filters in kwargs to put_machine_filters,
+# which is used to tell db this is a put action for the field.
+@utils.replace_filters(
+ machine_filters='put_machine_filters'
+)
+@utils.supported_filters(
+ optional_support_keys=UPDATED_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(
+ credentials=utils.check_switch_credentials,
+ put_machine_filters=_check_machine_filters
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_SWITCH
+)
+def update_switch(switch_id, user=None, session=None, **kwargs):
+ """Update fields of a switch."""
+ return _update_switch(switch_id, session=session, **kwargs)
+
+
+# replace credentials to patched_credentials,
+# machine_filters to patched_machine_filters in kwargs.
+# This is to tell db they are patch action to the above fields.
+@utils.replace_filters(
+ credentials='patched_credentials',
+ machine_filters='patched_machine_filters'
+)
+@utils.supported_filters(
+ optional_support_keys=PATCHED_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(
+ patched_machine_filters=_check_machine_filters
+)
+@database.run_in_session()
+@utils.output_validates(
+ credentials=utils.check_switch_credentials
+)
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_SWITCH
+)
+def patch_switch(switch_id, user=None, session=None, **kwargs):
+ """Patch fields of a switch."""
+ return _update_switch(switch_id, session=session, **kwargs)
+
+
+@util.deprecated
+@utils.supported_filters(optional_support_keys=SUPPORTED_FILTER_FIELDS)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_SWITCH_FILTERS
+)
+@utils.wrap_to_dict(RESP_FILTERS_FIELDS)
+def list_switch_filters(user=None, session=None, **filters):
+ """List all switches' filters."""
+ return utils.list_db_objects(
+ session, models.Switch, **filters
+ )
+
+
+@util.deprecated
+@utils.supported_filters()
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_SWITCH_FILTERS
+)
+@utils.wrap_to_dict(RESP_FILTERS_FIELDS)
+def get_switch_filters(
+ switch_id, exception_when_missing=True,
+ user=None, session=None, **kwargs
+):
+ """get filters of a switch."""
+ return _get_switch(
+ switch_id, session=session,
+ exception_when_missing=exception_when_missing
+ )
+
+
+@util.deprecated
+@utils.replace_filters(
+ machine_filters='put_machine_filters'
+)
+@utils.supported_filters(
+ optional_support_keys=UPDATED_FILTERS_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(put_machine_filters=_check_machine_filters)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_SWITCH_FILTERS
+)
+@utils.wrap_to_dict(RESP_FILTERS_FIELDS)
+def update_switch_filters(switch_id, user=None, session=None, **kwargs):
+ """Update filters of a switch."""
+ switch = _get_switch(switch_id, session=session)
+ return utils.update_db_object(session, switch, **kwargs)
+
+
+@util.deprecated
+@utils.replace_filters(
+ machine_filters='patched_machine_filters'
+)
+@utils.supported_filters(
+ optional_support_keys=PATCHED_FILTERS_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(patched_machine_filters=_check_machine_filters)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_SWITCH_FILTERS
+)
+@utils.wrap_to_dict(RESP_FILTERS_FIELDS)
+def patch_switch_filter(switch_id, user=None, session=None, **kwargs):
+ """Patch filters to a switch."""
+ switch = _get_switch(switch_id, session=session)
+ return utils.update_db_object(session, switch, **kwargs)
+
+
+@util.deprecated
+def get_switch_machines_internal(session, **filters):
+ return utils.list_db_objects(
+ session, models.SwitchMachine, **filters
+ )
+
+
+def _filter_port(port_filter, obj):
+ """filter switch machines by port.
+
+ supported port_filter keys: [
+ 'startswith', 'endswith', 'resp_lt',
+ 'resp_le', 'resp_gt', 'resp_ge', 'resp_range'
+ ]
+
+ port_filter examples:
+ {
+ 'startswitch': 'ae', 'endswith': '',
+ 'resp_ge': 20, 'resp_le': 30,
+ }
+ """
+ port_prefix = port_filter.get('startswith', '')
+ port_suffix = port_filter.get('endswith', '')
+ pattern = re.compile(r'%s(\d+)%s' % (port_prefix, port_suffix))
+ match = pattern.match(obj)
+ if not match:
+ return False
+ port_number = int(match.group(1))
+ if (
+ 'resp_lt' in port_filter and
+ port_number >= port_filter['resp_lt']
+ ):
+ return False
+ if (
+ 'resp_le' in port_filter and
+ port_number > port_filter['resp_le']
+ ):
+ return False
+ if (
+ 'resp_gt' in port_filter and
+ port_number <= port_filter['resp_gt']
+ ):
+ return False
+ if (
+ 'resp_ge' in port_filter and
+ port_number < port_filter['resp_ge']
+ ):
+ return False
+ if 'resp_range' in port_filter:
+ resp_range = port_filter['resp_range']
+ if not isinstance(resp_range, list):
+ resp_range = [resp_range]
+ in_range = False
+ for port_start, port_end in resp_range:
+ if port_start <= port_number <= port_end:
+ in_range = True
+ break
+ if not in_range:
+ return False
+ return True
+
+
+def _filter_vlans(vlan_filter, obj):
+ """Filter switch machines by vlan.
+
+ supported keys in vlan_filter:
+ ['resp_in']
+ """
+ vlans = set(obj)
+ if 'resp_in' in vlan_filter:
+ resp_vlans = set(vlan_filter['resp_in'])
+ if not (vlans & resp_vlans):
+ return False
+ return True
+
+
+@utils.output_filters(
+ port=_filter_port, vlans=_filter_vlans,
+ tag=utils.general_filter_callback,
+ location=utils.general_filter_callback
+)
+@utils.wrap_to_dict(RESP_MACHINES_FIELDS)
+def _filter_switch_machines(switch_machines):
+ """Get filtered switch machines.
+
+ The filters are defined in each switch.
+ """
+ return [
+ switch_machine for switch_machine in switch_machines
+ if not switch_machine.filtered
+ ]
+
+
+@utils.output_filters(
+ missing_ok=True,
+ port=_filter_port, vlans=_filter_vlans,
+ tag=utils.general_filter_callback,
+ location=utils.general_filter_callback,
+ os_name=utils.general_filter_callback,
+)
+@utils.wrap_to_dict(
+ RESP_MACHINES_HOSTS_FIELDS,
+ clusters=RESP_CLUSTER_FIELDS
+)
+def _filter_switch_machines_hosts(switch_machines):
+ """Similar as _filter_switch_machines, but also return host info."""
+ filtered_switch_machines = [
+ switch_machine for switch_machine in switch_machines
+ if not switch_machine.filtered
+ ]
+ switch_machines_hosts = []
+ for switch_machine in filtered_switch_machines:
+ machine = switch_machine.machine
+ host = machine.host
+ if host:
+ switch_machine_host_dict = host.to_dict()
+ else:
+ switch_machine_host_dict = machine.to_dict()
+ switch_machine_host_dict.update(
+ switch_machine.to_dict()
+ )
+ switch_machines_hosts.append(switch_machine_host_dict)
+ return switch_machines_hosts
+
+
+@utils.supported_filters(
+ optional_support_keys=SUPPORTED_MACHINES_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_SWITCH_MACHINES
+)
+def list_switch_machines(
+ switch_id, user=None, session=None, **filters
+):
+ """Get switch machines of a switch."""
+ switch = _get_switch(switch_id, session=session)
+ switch_machines = utils.list_db_objects(
+ session, models.SwitchMachine, switch_id=switch.id, **filters
+ )
+ if not user.is_admin and len(switch_machines):
+ switch_machines = [m for m in switch_machines if m.machine.owner_id == user.id]
+ return _filter_switch_machines(switch_machines)
+
+
+# replace ip_int to switch_ip_int in kwargs
+@utils.replace_filters(
+ ip_int='switch_ip_int'
+)
+@utils.supported_filters(
+ optional_support_keys=SUPPORTED_SWITCH_MACHINES_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_SWITCH_MACHINES
+)
+def list_switchmachines(user=None, session=None, **filters):
+ """List switch machines."""
+ switch_machines = utils.list_db_objects(
+ session, models.SwitchMachine, **filters
+ )
+ return _filter_switch_machines(
+ switch_machines
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=SUPPORTED_MACHINES_HOSTS_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_SWITCH_MACHINES
+)
+def list_switch_machines_hosts(
+ switch_id, user=None, session=None, **filters
+):
+ """Get switch machines and possible hosts of a switch."""
+ switch = _get_switch(switch_id, session=session)
+ switch_machines = utils.list_db_objects(
+ session, models.SwitchMachine, switch_id=switch.id, **filters
+ )
+ return _filter_switch_machines_hosts(
+ switch_machines
+ )
+
+
+# replace ip_int to switch_ip_int in kwargs
+@utils.replace_filters(
+ ip_int='switch_ip_int'
+)
+@utils.supported_filters(
+ optional_support_keys=SUPPORTED_SWITCH_MACHINES_HOSTS_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_SWITCH_MACHINES
+)
+def list_switchmachines_hosts(user=None, session=None, **filters):
+ """List switch machines hnd possible hosts."""
+ switch_machines = utils.list_db_objects(
+ session, models.SwitchMachine, **filters
+ )
+ if not user.is_admin and len(switch_machines):
+ switch_machines = [m for m in switch_machines if m.machine.owner_id == user.id]
+ return _filter_switch_machines_hosts(
+ switch_machines
+ )
+
+
+@utils.supported_filters(
+ ADDED_MACHINES_FIELDS,
+ optional_support_keys=OPTIONAL_ADDED_MACHINES_FIELDS,
+ ignore_support_keys=OPTIONAL_ADDED_SWITCH_MACHINES_FIELDS
+)
+@utils.input_validates(mac=utils.check_mac)
+def _add_machine_if_not_exist(mac=None, session=None, **kwargs):
+ """Add machine if the mac does not exist in any machine."""
+ return utils.add_db_object(
+ session, models.Machine, False,
+ mac, **kwargs)
+
+
+@utils.supported_filters(
+ ADDED_SWITCH_MACHINES_FIELDS,
+ optional_support_keys=OPTIONAL_ADDED_SWITCH_MACHINES_FIELDS,
+ ignore_support_keys=OPTIONAL_ADDED_MACHINES_FIELDS
+)
+@utils.input_validates(vlans=_check_vlans)
+def _add_switch_machine_only(
+ switch, machine, exception_when_existing=True,
+ session=None, owner_id=None, port=None, **kwargs
+):
+ """add a switch machine."""
+ return utils.add_db_object(
+ session, models.SwitchMachine,
+ exception_when_existing,
+ switch.id, machine.id, port=port,
+ owner_id=owner_id,
+ **kwargs
+ )
+
+
+@utils.supported_filters(
+ ADDED_MACHINES_FIELDS + ADDED_SWITCH_MACHINES_FIELDS,
+ optional_support_keys=(
+ OPTIONAL_ADDED_MACHINES_FIELDS +
+ OPTIONAL_ADDED_SWITCH_MACHINES_FIELDS
+ ),
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.wrap_to_dict(RESP_MACHINES_FIELDS)
+def _add_switch_machine(
+ switch_id, exception_when_existing=True,
+ mac=None, port=None, session=None, owner_id=None, **kwargs
+):
+ """Add switch machine.
+
+ If underlying machine does not exist, also create the underlying
+ machine.
+ """
+ switch = _get_switch(switch_id, session=session)
+ machine = _add_machine_if_not_exist(
+ mac=mac, session=session, owner_id=owner_id, **kwargs
+ )
+ return _add_switch_machine_only(
+ switch, machine,
+ exception_when_existing,
+ port=port, session=session, **kwargs
+ )
+
+
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_SWITCH_MACHINE
+)
+def add_switch_machine(
+ switch_id, exception_when_existing=True,
+ mac=None, user=None, session=None,
+ owner_id=None, **kwargs
+):
+ """Add switch machine to a switch."""
+ return _add_switch_machine(
+ switch_id,
+ exception_when_existing=exception_when_existing,
+ mac=mac, session=session, owner_id=owner_id, **kwargs
+ )
+
+
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_SWITCH_MACHINE
+)
+@utils.wrap_to_dict(
+ [
+ 'switches_machines',
+ 'duplicate_switches_machines',
+ 'fail_switches_machines'
+ ],
+ switches_machines=RESP_MACHINES_FIELDS,
+ duplicate_switches_machines=RESP_MACHINES_FIELDS
+)
+def add_switch_machines(
+ exception_when_existing=False,
+ data=[], user=None, session=None, owner_id=None
+):
+ """Add switch machines."""
+ switch_machines = []
+ duplicate_switch_machines = []
+ failed_switch_machines = []
+ switches_mapping = {}
+ switch_machines_mapping = {}
+ switch_ips = []
+ for item_data in data:
+ switch_ip = item_data['switch_ip']
+ if switch_ip not in switches_mapping:
+ switch_object = _get_switch_by_ip(
+ switch_ip, session=session,
+ exception_when_missing=False
+ )
+ if switch_object:
+ switch_ips.append(switch_ip)
+ switches_mapping[switch_ip] = switch_object
+ else:
+ logging.error(
+ 'switch %s does not exist' % switch_ip
+ )
+ item_data.pop('switch_ip')
+ failed_switch_machines.append(item_data)
+ else:
+ switch_object = switches_mapping[switch_ip]
+ if switch_object:
+ item_data.pop('switch_ip')
+ switch_machines_mapping.setdefault(
+ switch_object.id, []
+ ).append(item_data)
+
+ for switch_ip in switch_ips:
+ switch_object = switches_mapping[switch_ip]
+ switch_id = switch_object.id
+ machines = switch_machines_mapping[switch_id]
+ for machine in machines:
+ mac = machine['mac']
+ machine_object = _add_machine_if_not_exist(
+ mac=mac, session=session
+ )
+ switch_machine_object = _get_switch_machine(
+ switch_id, machine_object.id, session=session,
+ exception_when_missing=False
+ )
+ if switch_machine_object:
+ port = machine['port']
+ switch_machine_id = switch_machine_object.switch_machine_id
+ exist_port = switch_machine_object.port
+ if exist_port != port:
+ logging.error(
+ 'switch machine %s exist port %s is '
+ 'different from added port %s' % (
+ switch_machine_id,
+ exist_port, port
+ )
+ )
+ failed_switch_machines.append(machine)
+ else:
+ logging.error(
+ 'iswitch machine %s is dulicate, '
+ 'will not be override' % switch_machine_id
+ )
+ duplicate_switch_machines.append(machine)
+ else:
+ del machine['mac']
+ switch_machines.append(_add_switch_machine_only(
+ switch_object, machine_object,
+ exception_when_existing,
+ session=session, owner_id=owner_id, **machine
+ ))
+ return {
+ 'switches_machines': switch_machines,
+ 'duplicate_switches_machines': duplicate_switch_machines,
+ 'fail_switches_machines': failed_switch_machines
+ }
+
+
+@utils.supported_filters(optional_support_keys=['find_machines'])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_SWITCH_MACHINES
+)
+@utils.wrap_to_dict(RESP_ACTION_FIELDS)
+def poll_switch(switch_id, user=None, session=None, **kwargs):
+ """poll switch to get machines."""
+ from compass.tasks import client as celery_client
+ switch = _get_switch(switch_id, session=session)
+ celery_client.celery.send_task(
+ 'compass.tasks.pollswitch',
+ (user.email, switch.ip, switch.credentials),
+ queue=user.email,
+ exchange=user.email,
+ routing_key=user.email
+ )
+ return {
+ 'status': 'action %s sent' % kwargs,
+ 'details': {
+ }
+ }
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_SWITCH_MACHINES
+)
+@utils.wrap_to_dict(RESP_MACHINES_FIELDS)
+def get_switch_machine(
+ switch_id, machine_id, exception_when_missing=True,
+ user=None, session=None, **kwargs
+):
+ """get a switch machine by switch id and machine id."""
+ return _get_switch_machine(
+ switch_id, machine_id, session=session,
+ exception_when_missing=exception_when_missing
+ )
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_SWITCH_MACHINES
+)
+@utils.wrap_to_dict(RESP_MACHINES_FIELDS)
+def get_switchmachine(
+ switch_machine_id, exception_when_missing=True,
+ user=None, session=None, **kwargs
+):
+ """get a switch machine by switch_machine_id."""
+ return _get_switchmachine(
+ switch_machine_id, session=session,
+ exception_when_missing=exception_when_missing
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=(
+ UPDATED_MACHINES_FIELDS + PATCHED_MACHINES_FIELDS
+ ),
+ ignore_support_keys=(
+ UPDATED_SWITCH_MACHINES_FIELDS + PATCHED_SWITCH_MACHINES_FIELDS
+ )
+)
+def _update_machine_if_necessary(
+ machine, session=None, **kwargs
+):
+ """Update machine is there is something to update."""
+ utils.update_db_object(
+ session, machine, **kwargs
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=(
+ UPDATED_SWITCH_MACHINES_FIELDS + PATCHED_SWITCH_MACHINES_FIELDS
+ ),
+ ignore_support_keys=(
+ UPDATED_MACHINES_FIELDS + PATCHED_MACHINES_FIELDS
+ )
+)
+def _update_switch_machine_only(switch_machine, session=None, **kwargs):
+ """Update switch machine."""
+ return utils.update_db_object(
+ session, switch_machine, **kwargs
+ )
+
+
+def _update_switch_machine(
+ switch_machine, session=None, **kwargs
+):
+ """Update switch machine.
+
+ If there are some attributes of underlying machine need to update,
+ also update them in underlying machine.
+ """
+ _update_machine_if_necessary(
+ switch_machine.machine, session=session, **kwargs
+ )
+ return _update_switch_machine_only(
+ switch_machine, session=session, **kwargs
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=(
+ UPDATED_MACHINES_FIELDS + UPDATED_SWITCH_MACHINES_FIELDS
+ ),
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(vlans=_check_vlans)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_SWITCH_MACHINE
+)
+@utils.wrap_to_dict(RESP_MACHINES_FIELDS)
+def update_switch_machine(
+ switch_id, machine_id, user=None,
+ session=None, **kwargs
+):
+ """Update switch machine by switch id and machine id."""
+ switch_machine = _get_switch_machine(
+ switch_id, machine_id, session=session
+ )
+ return _update_switch_machine(
+ switch_machine,
+ session=session, **kwargs
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=(
+ UPDATED_MACHINES_FIELDS + UPDATED_SWITCH_MACHINES_FIELDS
+ ),
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(vlans=_check_vlans)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_SWITCH_MACHINE
+)
+@utils.wrap_to_dict(RESP_MACHINES_FIELDS)
+def update_switchmachine(switch_machine_id, user=None, session=None, **kwargs):
+ """Update switch machine by switch_machine_id."""
+ switch_machine = _get_switchmachine(
+ switch_machine_id, session=session
+ )
+ return _update_switch_machine(
+ switch_machine,
+ session=session, **kwargs
+ )
+
+
+# replace [vlans, ipmi_credentials, tag, location] to
+# [patched_vlans, patched_ipmi_credentials, patched_tag,
+# patched_location] in kwargs. It tells db these fields will
+# be patched.
+@utils.replace_filters(
+ vlans='patched_vlans',
+ ipmi_credentials='patched_ipmi_credentials',
+ tag='patched_tag',
+ location='patched_location'
+)
+@utils.supported_filters(
+ optional_support_keys=(
+ PATCHED_MACHINES_FIELDS + PATCHED_SWITCH_MACHINES_FIELDS
+ ),
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(patched_vlans=_check_vlans)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_SWITCH_MACHINE
+)
+@utils.wrap_to_dict(RESP_MACHINES_FIELDS)
+def patch_switch_machine(
+ switch_id, machine_id, user=None,
+ session=None, **kwargs
+):
+ """Patch switch machine by switch_id and machine_id."""
+ switch_machine = _get_switch_machine(
+ switch_id, machine_id, session=session
+ )
+ return _update_switch_machine(
+ switch_machine,
+ session=session, **kwargs
+ )
+
+
+# replace [vlans, ipmi_credentials, tag, location] to
+# [patched_vlans, patched_ipmi_credentials, patched_tag,
+# patched_location] in kwargs. It tells db these fields will
+# be patched.
+@utils.replace_filters(
+ vlans='patched_vlans',
+ ipmi_credentials='patched_ipmi_credentials',
+ tag='patched_tag',
+ location='patched_location'
+)
+@utils.supported_filters(
+ optional_support_keys=(
+ PATCHED_MACHINES_FIELDS + PATCHED_SWITCH_MACHINES_FIELDS
+ ),
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(patched_vlans=_check_vlans)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_ADD_SWITCH_MACHINE
+)
+@utils.wrap_to_dict(RESP_MACHINES_FIELDS)
+def patch_switchmachine(switch_machine_id, user=None, session=None, **kwargs):
+ """Patch switch machine by switch_machine_id."""
+ switch_machine = _get_switchmachine(
+ switch_machine_id, session=session
+ )
+ return _update_switch_machine(
+ switch_machine,
+ session=session, **kwargs
+ )
+
+
+def _del_switch_machine(
+ switch_machine, session=None
+):
+ """Delete switch machine.
+
+ If this is the last switch machine associated to underlying machine,
+ add a switch machine record to default switch to make the machine
+ searchable.
+ """
+ default_switch = _get_switch_by_ip(
+ setting.DEFAULT_SWITCH_IP, session=session
+ )
+ machine = switch_machine.machine
+ if len(machine.switch_machines) <= 1:
+ utils.add_db_object(
+ session, models.SwitchMachine,
+ False,
+ default_switch.id, machine.id,
+ port=switch_machine.port
+ )
+ return utils.del_db_object(session, switch_machine)
+
+
+@utils.supported_filters()
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEL_SWITCH_MACHINE
+)
+@utils.wrap_to_dict(RESP_MACHINES_FIELDS)
+def del_switch_machine(
+ switch_id, machine_id, user=None,
+ session=None, **kwargs
+):
+ """Delete switch machine by switch id and machine id."""
+ switch_machine = _get_switch_machine(
+ switch_id, machine_id, session=session
+ )
+ return _del_switch_machine(switch_machine, session=session)
+
+
+@utils.supported_filters()
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_DEL_SWITCH_MACHINE
+)
+@utils.wrap_to_dict(RESP_MACHINES_FIELDS)
+def del_switchmachine(switch_machine_id, user=None, session=None, **kwargs):
+ """Delete switch machine by switch_machine_id."""
+ switch_machine = _get_switchmachine(
+ switch_machine_id, session=session
+ )
+ return _del_switch_machine(switch_machine, session=session)
+
+
+@utils.supported_filters(
+ ['machine_id'],
+ optional_support_keys=UPDATED_SWITCH_MACHINES_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+def _add_machine_to_switch(
+ switch_id, machine_id, session=None, **kwargs
+):
+ """Add machine to switch."""
+ switch = _get_switch(switch_id, session=session)
+ from compass.db.api import machine as machine_api
+ machine = machine_api.get_machine_internal(
+ machine_id, session=session
+ )
+ _add_switch_machine_only(
+ switch, machine, False,
+ owner_id=machine.owner_id, **kwargs
+ )
+
+
+def _add_machines(switch, machines, session=None):
+ """Add machines to switch.
+
+ Args:
+ machines: list of dict which contains attributes to
+ add machine to switch.
+
+ machines example:
+ {{'machine_id': 1, 'port': 'ae20'}]
+ """
+ for machine in machines:
+ _add_machine_to_switch(
+ switch.id, session=session, **machine
+ )
+
+
+def _remove_machines(switch, machines, session=None):
+ """Remove machines from switch.
+
+ Args:
+ machines: list of machine id.
+
+ machines example:
+ [1,2]
+ """
+ utils.del_db_objects(
+ session, models.SwitchMachine,
+ switch_id=switch.id, machine_id=machines
+ )
+
+
+def _set_machines(switch, machines, session=None):
+ """Reset machines to a switch.
+
+ Args:
+ machines: list of dict which contains attributes to
+ add machine to switch.
+
+ machines example:
+ {{'machine_id': 1, 'port': 'ae20'}]
+ """
+ utils.del_db_objects(
+ session, models.SwitchMachine,
+ switch_id=switch.id
+ )
+ for switch_machine in machines:
+ _add_machine_to_switch(
+ switch.id, session=session, **switch_machine
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=[
+ 'add_machines', 'remove_machines', 'set_machines'
+ ]
+)
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_UPDATE_SWITCH_MACHINES
+)
+@utils.wrap_to_dict(RESP_MACHINES_FIELDS)
+def update_switch_machines(
+ switch_id, add_machines=[], remove_machines=[],
+ set_machines=None, user=None, session=None, **kwargs
+):
+ """update switch's machines"""
+ switch = _get_switch(switch_id, session=session)
+ if remove_machines:
+ _remove_machines(
+ switch, remove_machines, session=session
+ )
+ if add_machines:
+ _add_machines(
+ switch, add_machines, session=session
+ )
+ if set_machines is not None:
+ _set_machines(
+ switch, set_machines, session=session
+ )
+ return switch.switch_machines
diff --git a/compass-tasks/db/api/user.py b/compass-tasks/db/api/user.py
new file mode 100644
index 0000000..db039eb
--- /dev/null
+++ b/compass-tasks/db/api/user.py
@@ -0,0 +1,553 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""User database operations."""
+import datetime
+import functools
+import logging
+import re
+
+from flask.ext.login import UserMixin
+
+from compass.db.api import database
+from compass.db.api import utils
+from compass.db import exception
+from compass.db import models
+
+from compass.utils import setting_wrapper as setting
+from compass.utils import util
+
+
+SUPPORTED_FIELDS = ['email', 'is_admin', 'active']
+PERMISSION_SUPPORTED_FIELDS = ['name']
+SELF_UPDATED_FIELDS = ['email', 'firstname', 'lastname', 'password']
+ADMIN_UPDATED_FIELDS = ['is_admin', 'active']
+IGNORE_FIELDS = ['id', 'created_at', 'updated_at']
+UPDATED_FIELDS = [
+ 'email', 'firstname', 'lastname', 'password', 'is_admin', 'active'
+]
+ADDED_FIELDS = ['email', 'password']
+OPTIONAL_ADDED_FIELDS = ['is_admin', 'active']
+PERMISSION_ADDED_FIELDS = ['permission_id']
+RESP_FIELDS = [
+ 'id', 'email', 'is_admin', 'active', 'firstname',
+ 'lastname', 'created_at', 'updated_at'
+]
+RESP_TOKEN_FIELDS = [
+ 'id', 'user_id', 'token', 'expire_timestamp'
+]
+PERMISSION_RESP_FIELDS = [
+ 'id', 'user_id', 'permission_id', 'name', 'alias', 'description',
+ 'created_at', 'updated_at'
+]
+
+
+def _check_email(email):
+ """Check email is email format."""
+ if '@' not in email:
+ raise exception.InvalidParameter(
+ 'there is no @ in email address %s.' % email
+ )
+
+
+def _check_user_permission(user, permission, session=None):
+ """Check user has permission."""
+ if not user:
+ logging.info('empty user means the call is from internal')
+ return
+ if user.is_admin:
+ return
+
+ user_permission = utils.get_db_object(
+ session, models.UserPermission,
+ False, user_id=user.id, name=permission.name
+ )
+ if not user_permission:
+ raise exception.Forbidden(
+ 'user %s does not have permission %s' % (
+ user.email, permission.name
+ )
+ )
+
+
+def check_user_permission(permission):
+ """Decorator to check user having permission."""
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ user = kwargs.get('user')
+ if user is not None:
+ session = kwargs.get('session')
+ if session is None:
+ raise exception.DatabaseException(
+ 'wrapper check_user_permission does not run in session'
+ )
+ _check_user_permission(user, permission, session=session)
+ return func(*args, **kwargs)
+ else:
+ return func(*args, **kwargs)
+ return wrapper
+ return decorator
+
+
+def check_user_admin():
+ """Decorator to check user is admin."""
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ user = kwargs.get('user')
+ if user is not None:
+ if not user.is_admin:
+ raise exception.Forbidden(
+ 'User %s is not admin.' % (
+ user.email
+ )
+ )
+ return func(*args, **kwargs)
+ else:
+ return func(*args, **kwargs)
+ return wrapper
+ return decorator
+
+
+def check_user_admin_or_owner():
+ """Decorator to check user is admin or the owner of the resource."""
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(user_id, *args, **kwargs):
+ user = kwargs.get('user')
+ if user is not None:
+ session = kwargs.get('session')
+ if session is None:
+ raise exception.DatabaseException(
+ 'wrapper check_user_admin_or_owner is '
+ 'not called in session'
+ )
+ check_user = _get_user(user_id, session=session)
+ if not user.is_admin and user.id != check_user.id:
+ raise exception.Forbidden(
+ 'User %s is not admin or the owner of user %s.' % (
+ user.email, check_user.email
+ )
+ )
+
+ return func(
+ user_id, *args, **kwargs
+ )
+ else:
+ return func(
+ user_id, *args, **kwargs
+ )
+ return wrapper
+ return decorator
+
+
+def _add_user_permissions(user, session=None, **permission_filters):
+ """add permissions to a user."""
+ from compass.db.api import permission as permission_api
+ for api_permission in permission_api.list_permissions(
+ session=session, **permission_filters
+ ):
+ utils.add_db_object(
+ session, models.UserPermission, False,
+ user.id, api_permission['id']
+ )
+
+
+def _remove_user_permissions(user, session=None, **permission_filters):
+ """remove permissions from a user."""
+ from compass.db.api import permission as permission_api
+ permission_ids = [
+ api_permission['id']
+ for api_permission in permission_api.list_permissions(
+ session=session, **permission_filters
+ )
+ ]
+ utils.del_db_objects(
+ session, models.UserPermission,
+ user_id=user.id, permission_id=permission_ids
+ )
+
+
+def _set_user_permissions(user, session=None, **permission_filters):
+ """set permissions to a user."""
+ utils.del_db_objects(
+ session, models.UserPermission,
+ user_id=user.id
+ )
+ _add_user_permissions(session, user, **permission_filters)
+
+
+class UserWrapper(UserMixin):
+ """Wrapper class provided to flask."""
+
+ def __init__(
+ self, id, email, crypted_password,
+ active=True, is_admin=False,
+ expire_timestamp=None, token='', **kwargs
+ ):
+ self.id = id
+ self.email = email
+ self.password = crypted_password
+ self.active = active
+ self.is_admin = is_admin
+ self.expire_timestamp = expire_timestamp
+ if not token:
+ self.token = self.get_auth_token()
+ else:
+ self.token = token
+ super(UserWrapper, self).__init__()
+
+ def authenticate(self, password):
+ if not util.encrypt(password, self.password) == self.password:
+ raise exception.Unauthorized('%s password mismatch' % self.email)
+
+ def get_auth_token(self):
+ return util.encrypt(self.email)
+
+ def is_active(self):
+ return self.active
+
+ def get_id(self):
+ return self.token
+
+ def is_authenticated(self):
+ current_time = datetime.datetime.now()
+ return (
+ not self.expire_timestamp or
+ current_time < self.expire_timestamp
+ )
+
+ def __str__(self):
+ return '%s[email:%s,password:%s]' % (
+ self.__class__.__name__, self.email, self.password)
+
+
+@database.run_in_session()
+def get_user_object(email, session=None, **kwargs):
+ """get user and convert to UserWrapper object."""
+ user = utils.get_db_object(
+ session, models.User, False, email=email
+ )
+ if not user:
+ raise exception.Unauthorized(
+ '%s unauthorized' % email
+ )
+ user_dict = user.to_dict()
+ user_dict.update(kwargs)
+ return UserWrapper(**user_dict)
+
+
+@database.run_in_session(exception_when_in_session=False)
+def get_user_object_from_token(token, session=None):
+ """Get user from token and convert to UserWrapper object.
+
+ ::note:
+ get_user_object_from_token may be called in session.
+ """
+ expire_timestamp = {
+ 'ge': datetime.datetime.now()
+ }
+ user_token = utils.get_db_object(
+ session, models.UserToken, False,
+ token=token, expire_timestamp=expire_timestamp
+ )
+ if not user_token:
+ raise exception.Unauthorized(
+ 'invalid user token: %s' % token
+ )
+ user_dict = _get_user(
+ user_token.user_id, session=session
+ ).to_dict()
+ user_dict['token'] = token
+ expire_timestamp = user_token.expire_timestamp
+ user_dict['expire_timestamp'] = expire_timestamp
+ return UserWrapper(**user_dict)
+
+
+@utils.supported_filters()
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_TOKEN_FIELDS)
+def record_user_token(
+ token, expire_timestamp, user=None, session=None
+):
+ """record user token in database."""
+ user_token = utils.get_db_object(
+ session, models.UserToken, False,
+ user_id=user.id, token=token
+ )
+ if not user_token:
+ return utils.add_db_object(
+ session, models.UserToken, True,
+ token, user_id=user.id,
+ expire_timestamp=expire_timestamp
+ )
+ elif expire_timestamp > user_token.expire_timestamp:
+ return utils.update_db_object(
+ session, user_token, expire_timestamp=expire_timestamp
+ )
+ return user_token
+
+
+@utils.supported_filters()
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_TOKEN_FIELDS)
+def clean_user_token(token, user=None, session=None):
+ """clean user token in database."""
+ return utils.del_db_objects(
+ session, models.UserToken,
+ token=token, user_id=user.id
+ )
+
+
+def _get_user(user_id, session=None, **kwargs):
+ """Get user object by user id."""
+ if isinstance(user_id, (int, long)):
+ return utils.get_db_object(
+ session, models.User, id=user_id, **kwargs
+ )
+ raise exception.InvalidParameter(
+ 'user id %s type is not int compatible' % user_id
+ )
+
+
+@utils.supported_filters()
+@database.run_in_session()
+@check_user_admin_or_owner()
+@utils.wrap_to_dict(RESP_FIELDS)
+def get_user(
+ user_id, exception_when_missing=True,
+ user=None, session=None, **kwargs
+):
+ """get a user."""
+ return _get_user(
+ user_id, session=session,
+ exception_when_missing=exception_when_missing
+ )
+
+
+@utils.supported_filters()
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_FIELDS)
+def get_current_user(
+ exception_when_missing=True, user=None,
+ session=None, **kwargs
+):
+ """get current user."""
+ return _get_user(
+ user.id, session=session,
+ exception_when_missing=exception_when_missing
+ )
+
+
+@utils.supported_filters(
+ optional_support_keys=SUPPORTED_FIELDS
+)
+@database.run_in_session()
+@check_user_admin()
+@utils.wrap_to_dict(RESP_FIELDS)
+def list_users(user=None, session=None, **filters):
+ """List all users."""
+ return utils.list_db_objects(
+ session, models.User, **filters
+ )
+
+
+@utils.input_validates(email=_check_email)
+@utils.supported_filters(
+ ADDED_FIELDS,
+ optional_support_keys=OPTIONAL_ADDED_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@check_user_admin()
+@utils.wrap_to_dict(RESP_FIELDS)
+def add_user(
+ exception_when_existing=True, user=None,
+ session=None, email=None, **kwargs
+):
+ """Create a user and return created user object."""
+ add_user = utils.add_db_object(
+ session, models.User,
+ exception_when_existing, email,
+ **kwargs)
+ _add_user_permissions(
+ add_user,
+ session=session,
+ name=setting.COMPASS_DEFAULT_PERMISSIONS
+ )
+ return add_user
+
+
+@utils.supported_filters()
+@database.run_in_session()
+@check_user_admin()
+@utils.wrap_to_dict(RESP_FIELDS)
+def del_user(user_id, user=None, session=None, **kwargs):
+ """delete a user and return the deleted user object."""
+ del_user = _get_user(user_id, session=session)
+ return utils.del_db_object(session, del_user)
+
+
+@utils.supported_filters(
+ optional_support_keys=UPDATED_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@utils.input_validates(email=_check_email)
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_FIELDS)
+def update_user(user_id, user=None, session=None, **kwargs):
+ """Update a user and return the updated user object."""
+ update_user = _get_user(
+ user_id, session=session,
+ )
+ allowed_fields = set()
+ if user.is_admin:
+ allowed_fields |= set(ADMIN_UPDATED_FIELDS)
+ if user.id == update_user.id:
+ allowed_fields |= set(SELF_UPDATED_FIELDS)
+ unsupported_fields = set(kwargs) - allowed_fields
+ if unsupported_fields:
+ # The user is not allowed to update a user.
+ raise exception.Forbidden(
+ 'User %s has no permission to update user %s fields %s.' % (
+ user.email, user.email, unsupported_fields
+ )
+ )
+ return utils.update_db_object(session, update_user, **kwargs)
+
+
+@utils.supported_filters(optional_support_keys=PERMISSION_SUPPORTED_FIELDS)
+@database.run_in_session()
+@check_user_admin_or_owner()
+@utils.wrap_to_dict(PERMISSION_RESP_FIELDS)
+def get_permissions(
+ user_id, user=None, exception_when_missing=True,
+ session=None, **kwargs
+):
+ """List permissions of a user."""
+ get_user = _get_user(
+ user_id, session=session,
+ exception_when_missing=exception_when_missing
+ )
+ return utils.list_db_objects(
+ session, models.UserPermission, user_id=get_user.id, **kwargs
+ )
+
+
+def _get_permission(user_id, permission_id, session=None, **kwargs):
+ """Get user permission by user id and permission id."""
+ user = _get_user(user_id, session=session)
+ from compass.db.api import permission as permission_api
+ permission = permission_api.get_permission_internal(
+ permission_id, session=session
+ )
+ return utils.get_db_object(
+ session, models.UserPermission,
+ user_id=user.id, permission_id=permission.id,
+ **kwargs
+ )
+
+
+@utils.supported_filters()
+@database.run_in_session()
+@check_user_admin_or_owner()
+@utils.wrap_to_dict(PERMISSION_RESP_FIELDS)
+def get_permission(
+ user_id, permission_id, exception_when_missing=True,
+ user=None, session=None, **kwargs
+):
+ """Get a permission of a user."""
+ return _get_permission(
+ user_id, permission_id,
+ exception_when_missing=exception_when_missing,
+ session=session,
+ **kwargs
+ )
+
+
+@utils.supported_filters()
+@database.run_in_session()
+@check_user_admin_or_owner()
+@utils.wrap_to_dict(PERMISSION_RESP_FIELDS)
+def del_permission(user_id, permission_id, user=None, session=None, **kwargs):
+ """Delete a permission from a user."""
+ user_permission = _get_permission(
+ user_id, permission_id,
+ session=session, **kwargs
+ )
+ return utils.del_db_object(session, user_permission)
+
+
+@utils.supported_filters(
+ PERMISSION_ADDED_FIELDS,
+ ignore_support_keys=IGNORE_FIELDS
+)
+@database.run_in_session()
+@check_user_admin()
+@utils.wrap_to_dict(PERMISSION_RESP_FIELDS)
+def add_permission(
+ user_id, permission_id=None, exception_when_existing=True,
+ user=None, session=None
+):
+ """Add a permission to a user."""
+ get_user = _get_user(user_id, session=session)
+ from compass.db.api import permission as permission_api
+ get_permission = permission_api.get_permission_internal(
+ permission_id, session=session
+ )
+ return utils.add_db_object(
+ session, models.UserPermission, exception_when_existing,
+ get_user.id, get_permission.id
+ )
+
+
+def _get_permission_filters(permission_ids):
+ """Helper function to filter permissions."""
+ if permission_ids == 'all':
+ return {}
+ else:
+ return {'id': permission_ids}
+
+
+@utils.supported_filters(
+ optional_support_keys=[
+ 'add_permissions', 'remove_permissions', 'set_permissions'
+ ]
+)
+@database.run_in_session()
+@check_user_admin()
+@utils.wrap_to_dict(PERMISSION_RESP_FIELDS)
+def update_permissions(
+ user_id, add_permissions=[], remove_permissions=[],
+ set_permissions=None, user=None, session=None, **kwargs
+):
+ """update user permissions."""
+ update_user = _get_user(user_id, session=session)
+ if remove_permissions:
+ _remove_user_permissions(
+ update_user, session=session,
+ **_get_permission_filters(remove_permissions)
+ )
+ if add_permissions:
+ _add_user_permissions(
+ update_user, session=session,
+ **_get_permission_filters(add_permissions)
+ )
+ if set_permissions is not None:
+ _set_user_permissions(
+ update_user, session=session,
+ **_get_permission_filters(set_permissions)
+ )
+ return update_user.user_permissions
diff --git a/compass-tasks/db/api/user_log.py b/compass-tasks/db/api/user_log.py
new file mode 100644
index 0000000..70de9db
--- /dev/null
+++ b/compass-tasks/db/api/user_log.py
@@ -0,0 +1,82 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""UserLog database operations."""
+import logging
+
+from compass.db.api import database
+from compass.db.api import user as user_api
+from compass.db.api import utils
+from compass.db import exception
+from compass.db import models
+
+
+SUPPORTED_FIELDS = ['user_email', 'timestamp']
+USER_SUPPORTED_FIELDS = ['timestamp']
+RESP_FIELDS = ['user_id', 'action', 'timestamp']
+
+
+@database.run_in_session()
+def log_user_action(user_id, action, session=None):
+ """Log user action."""
+ utils.add_db_object(
+ session, models.UserLog, True, user_id=user_id, action=action
+ )
+
+
+@utils.supported_filters(optional_support_keys=USER_SUPPORTED_FIELDS)
+@database.run_in_session()
+@user_api.check_user_admin_or_owner()
+@utils.wrap_to_dict(RESP_FIELDS)
+def list_user_actions(user_id, user=None, session=None, **filters):
+ """list user actions of a user."""
+ list_user = user_api.get_user(user_id, user=user, session=session)
+ return utils.list_db_objects(
+ session, models.UserLog, order_by=['timestamp'],
+ user_id=list_user['id'], **filters
+ )
+
+
+@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS)
+@user_api.check_user_admin()
+@database.run_in_session()
+@utils.wrap_to_dict(RESP_FIELDS)
+def list_actions(user=None, session=None, **filters):
+ """list actions of all users."""
+ return utils.list_db_objects(
+ session, models.UserLog, order_by=['timestamp'], **filters
+ )
+
+
+@utils.supported_filters()
+@database.run_in_session()
+@user_api.check_user_admin_or_owner()
+@utils.wrap_to_dict(RESP_FIELDS)
+def del_user_actions(user_id, user=None, session=None, **filters):
+ """delete actions of a user."""
+ del_user = user_api.get_user(user_id, user=user, session=session)
+ return utils.del_db_objects(
+ session, models.UserLog, user_id=del_user['id'], **filters
+ )
+
+
+@utils.supported_filters()
+@database.run_in_session()
+@user_api.check_user_admin()
+@utils.wrap_to_dict(RESP_FIELDS)
+def del_actions(user=None, session=None, **filters):
+ """delete actions of all users."""
+ return utils.del_db_objects(
+ session, models.UserLog, **filters
+ )
diff --git a/compass-tasks/db/api/utils.py b/compass-tasks/db/api/utils.py
new file mode 100644
index 0000000..a44f26e
--- /dev/null
+++ b/compass-tasks/db/api/utils.py
@@ -0,0 +1,1286 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utils for database usage."""
+
+import functools
+import inspect
+import logging
+import netaddr
+import re
+
+from inspect import isfunction
+from sqlalchemy import and_
+from sqlalchemy import or_
+
+from compass.db import exception
+from compass.db import models
+from compass.utils import util
+
+
+def model_query(session, model):
+ """model query.
+
+ Return sqlalchemy query object.
+ """
+ if not issubclass(model, models.BASE):
+ raise exception.DatabaseException("model should be sublass of BASE!")
+
+ return session.query(model)
+
+
+def _default_list_condition_func(col_attr, value, condition_func):
+ """The default condition func for a list of data.
+
+ Given the condition func for single item of data, this function
+ wrap the condition_func and return another condition func using
+ or_ to merge the conditions of each single item to deal with a
+ list of data item.
+
+ Args:
+ col_attr: the colomn name
+ value: the column value need to be compared.
+ condition_func: the sqlalchemy condition object like ==
+
+ Examples:
+ col_attr is name, value is ['a', 'b', 'c'] and
+ condition_func is ==, the returned condition is
+ name == 'a' or name == 'b' or name == 'c'
+ """
+ conditions = []
+ for sub_value in value:
+ condition = condition_func(col_attr, sub_value)
+ if condition is not None:
+ conditions.append(condition)
+ if conditions:
+ return or_(*conditions)
+ else:
+ return None
+
+
+def _one_item_list_condition_func(col_attr, value, condition_func):
+ """The wrapper condition func to deal with one item data list.
+
+ For simplification, it is used to reduce generating too complex
+ sql conditions.
+ """
+ if value:
+ return condition_func(col_attr, value[0])
+ else:
+ return None
+
+
+def _model_condition_func(
+ col_attr, value,
+ item_condition_func,
+ list_condition_func=_default_list_condition_func
+):
+ """Return sql condition based on value type."""
+ if isinstance(value, list):
+ if not value:
+ return None
+ if len(value) == 1:
+ return item_condition_func(col_attr, value)
+ return list_condition_func(
+ col_attr, value, item_condition_func
+ )
+ else:
+ return item_condition_func(col_attr, value)
+
+
+def _between_condition(col_attr, value):
+ """Return sql range condition."""
+ if value[0] is not None and value[1] is not None:
+ return col_attr.between(value[0], value[1])
+ if value[0] is not None:
+ return col_attr >= value[0]
+ if value[1] is not None:
+ return col_attr <= value[1]
+ return None
+
+
+def model_order_by(query, model, order_by):
+ """append order by into sql query model."""
+ if not order_by:
+ return query
+ order_by_cols = []
+ for key in order_by:
+ if isinstance(key, tuple):
+ key, is_desc = key
+ else:
+ is_desc = False
+ if isinstance(key, basestring):
+ if hasattr(model, key):
+ col_attr = getattr(model, key)
+ else:
+ continue
+ else:
+ col_attr = key
+ if is_desc:
+ order_by_cols.append(col_attr.desc())
+ else:
+ order_by_cols.append(col_attr)
+ return query.order_by(*order_by_cols)
+
+
+def _model_condition(col_attr, value):
+ """Generate condition for one column.
+
+ Example for col_attr is name:
+ value is 'a': name == 'a'
+ value is ['a']: name == 'a'
+ value is ['a', 'b']: name == 'a' or name == 'b'
+ value is {'eq': 'a'}: name == 'a'
+ value is {'lt': 'a'}: name < 'a'
+ value is {'le': 'a'}: name <= 'a'
+ value is {'gt': 'a'}: name > 'a'
+ value is {'ge': 'a'}: name >= 'a'
+ value is {'ne': 'a'}: name != 'a'
+ value is {'in': ['a', 'b']}: name in ['a', 'b']
+ value is {'notin': ['a', 'b']}: name not in ['a', 'b']
+ value is {'startswith': 'abc'}: name like 'abc%'
+ value is {'endswith': 'abc'}: name like '%abc'
+ value is {'like': 'abc'}: name like '%abc%'
+ value is {'between': ('a', 'c')}: name >= 'a' and name <= 'c'
+ value is [{'lt': 'a'}]: name < 'a'
+ value is [{'lt': 'a'}, {'gt': c'}]: name < 'a' or name > 'c'
+ value is {'lt': 'c', 'gt': 'a'}: name > 'a' and name < 'c'
+
+ If value is a list, the condition is the or relationship among
+ conditions of each item.
+ If value is dict and there are multi keys in the dict, the relationship
+ is and conditions of each key.
+ Otherwise the condition is to compare the column with the value.
+ """
+ if isinstance(value, list):
+ basetype_values = []
+ composite_values = []
+ for item in value:
+ if isinstance(item, (list, dict)):
+ composite_values.append(item)
+ else:
+ basetype_values.append(item)
+ conditions = []
+ if basetype_values:
+ if len(basetype_values) == 1:
+ condition = (col_attr == basetype_values[0])
+ else:
+ condition = col_attr.in_(basetype_values)
+ conditions.append(condition)
+ for composite_value in composite_values:
+ condition = _model_condition(col_attr, composite_value)
+ if condition is not None:
+ conditions.append(condition)
+ if not conditions:
+ return None
+ if len(conditions) == 1:
+ return conditions[0]
+ return or_(*conditions)
+ elif isinstance(value, dict):
+ conditions = []
+ if 'eq' in value:
+ conditions.append(_model_condition_func(
+ col_attr, value['eq'],
+ lambda attr, data: attr == data,
+ lambda attr, data, item_condition_func: attr.in_(data)
+ ))
+ if 'lt' in value:
+ conditions.append(_model_condition_func(
+ col_attr, value['lt'],
+ lambda attr, data: attr < data,
+ _one_item_list_condition_func
+ ))
+ if 'gt' in value:
+ conditions.append(_model_condition_func(
+ col_attr, value['gt'],
+ lambda attr, data: attr > data,
+ _one_item_list_condition_func
+ ))
+ if 'le' in value:
+ conditions.append(_model_condition_func(
+ col_attr, value['le'],
+ lambda attr, data: attr <= data,
+ _one_item_list_condition_func
+ ))
+ if 'ge' in value:
+ conditions.append(_model_condition_func(
+ col_attr, value['ge'],
+ lambda attr, data: attr >= data,
+ _one_item_list_condition_func
+ ))
+ if 'ne' in value:
+ conditions.append(_model_condition_func(
+ col_attr, value['ne'],
+ lambda attr, data: attr != data,
+ lambda attr, data, item_condition_func: attr.notin_(data)
+ ))
+ if 'in' in value:
+ conditions.append(col_attr.in_(value['in']))
+ if 'notin' in value:
+ conditions.append(col_attr.notin_(value['notin']))
+ if 'startswith' in value:
+ conditions.append(_model_condition_func(
+ col_attr, value['startswith'],
+ lambda attr, data: attr.like('%s%%' % data)
+ ))
+ if 'endswith' in value:
+ conditions.append(_model_condition_func(
+ col_attr, value['endswith'],
+ lambda attr, data: attr.like('%%%s' % data)
+ ))
+ if 'like' in value:
+ conditions.append(_model_condition_func(
+ col_attr, value['like'],
+ lambda attr, data: attr.like('%%%s%%' % data)
+ ))
+ if 'between' in value:
+ conditions.append(_model_condition_func(
+ col_attr, value['between'],
+ _between_condition
+ ))
+ conditions = [
+ condition
+ for condition in conditions
+ if condition is not None
+ ]
+ if not conditions:
+ return None
+ if len(conditions) == 1:
+ return conditions[0]
+ return and_(conditions)
+ else:
+ condition = (col_attr == value)
+ return condition
+
+
+def model_filter(query, model, **filters):
+ """Append conditons to query for each possible column."""
+ for key, value in filters.items():
+ if isinstance(key, basestring):
+ if hasattr(model, key):
+ col_attr = getattr(model, key)
+ else:
+ continue
+ else:
+ col_attr = key
+ condition = _model_condition(col_attr, value)
+ if condition is not None:
+ query = query.filter(condition)
+ return query
+
+
+def replace_output(**output_mapping):
+ """Decorator to recursively relace output by output mapping.
+
+ The replacement detail is described in _replace_output.
+ """
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ return _replace_output(
+ func(*args, **kwargs), **output_mapping
+ )
+ return wrapper
+ return decorator
+
+
+def _replace_output(data, **output_mapping):
+ """Helper to replace output data.
+
+ Example:
+ data = {'a': 'hello'}
+ output_mapping = {'a': 'b'}
+ returns: {'b': 'hello'}
+
+ data = {'a': {'b': 'hello'}}
+ output_mapping = {'a': 'b'}
+ returns: {'b': {'b': 'hello'}}
+
+ data = {'a': {'b': 'hello'}}
+ output_mapping = {'a': {'b': 'c'}}
+ returns: {'a': {'c': 'hello'}}
+
+ data = [{'a': 'hello'}, {'a': 'hi'}]
+ output_mapping = {'a': 'b'}
+ returns: [{'b': 'hello'}, {'b': 'hi'}]
+ """
+ if isinstance(data, list):
+ return [
+ _replace_output(item, **output_mapping)
+ for item in data
+ ]
+ if not isinstance(data, dict):
+ raise exception.InvalidResponse(
+ '%s type is not dict' % data
+ )
+ info = {}
+ for key, value in data.items():
+ if key in output_mapping:
+ output_key = output_mapping[key]
+ if isinstance(output_key, basestring):
+ info[output_key] = value
+ else:
+ info[key] = (
+ _replace_output(value, **output_key)
+ )
+ else:
+ info[key] = value
+ return info
+
+
+def get_wrapped_func(func):
+ """Get wrapped function instance.
+
+ Example:
+ @dec1
+ @dec2
+ myfunc(*args, **kwargs)
+
+ get_wrapped_func(myfunc) returns function object with
+ following attributes:
+ __name__: 'myfunc'
+ args: args
+ kwargs: kwargs
+ otherwise myfunc is function object with following attributes:
+ __name__: partial object ...
+ args: ...
+ kwargs: ...
+ """
+ if func.func_closure:
+ for closure in func.func_closure:
+ if isfunction(closure.cell_contents):
+ return get_wrapped_func(closure.cell_contents)
+ return func
+ else:
+ return func
+
+
+def wrap_to_dict(support_keys=[], **filters):
+ """Decrator to convert returned object to dict.
+
+ The details is decribed in _wrapper_dict.
+ """
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ return _wrapper_dict(
+ func(*args, **kwargs), support_keys, **filters
+ )
+ return wrapper
+ return decorator
+
+
+def _wrapper_dict(data, support_keys, **filters):
+ """Helper for warpping db object into dictionary.
+
+ If data is list, convert it to a list of dict
+ If data is Base model, convert it to dict
+ for the data as a dict, filter it with the supported keys.
+ For each filter_key, filter_value in filters, also filter
+ data[filter_key] by filter_value recursively if it exists.
+
+ Example:
+ data is models.Switch, it will be converted to
+ {
+ 'id': 1, 'ip': '10.0.0.1', 'ip_int': 123456,
+ 'credentials': {'version': 2, 'password': 'abc'}
+ }
+ Then if support_keys are ['id', 'ip', 'credentials'],
+ it will be filtered to {
+ 'id': 1, 'ip': '10.0.0.1',
+ 'credentials': {'version': 2, 'password': 'abc'}
+ }
+ Then if filters is {'credentials': ['version']},
+ it will be filtered to {
+ 'id': 1, 'ip': '10.0.0.1',
+ 'credentials': {'version': 2}
+ }
+ """
+ logging.debug(
+ 'wrap dict %s by support_keys=%s filters=%s',
+ data, support_keys, filters
+ )
+ if isinstance(data, list):
+ return [
+ _wrapper_dict(item, support_keys, **filters)
+ for item in data
+ ]
+ if isinstance(data, models.HelperMixin):
+ data = data.to_dict()
+ if not isinstance(data, dict):
+ raise exception.InvalidResponse(
+ 'response %s type is not dict' % data
+ )
+ info = {}
+ try:
+ for key in support_keys:
+ if key in data and data[key] is not None:
+ if key in filters:
+ filter_keys = filters[key]
+ if isinstance(filter_keys, dict):
+ info[key] = _wrapper_dict(
+ data[key], filter_keys.keys(),
+ **filter_keys
+ )
+ else:
+ info[key] = _wrapper_dict(
+ data[key], filter_keys
+ )
+ else:
+ info[key] = data[key]
+ return info
+ except Exception as error:
+ logging.exception(error)
+ raise error
+
+
+def replace_filters(**kwarg_mapping):
+ """Decorator to replace kwargs.
+
+ Examples:
+ kwargs: {'a': 'b'}, kwarg_mapping: {'a': 'c'}
+ replaced kwargs to decorated func:
+ {'c': 'b'}
+
+ replace_filters is used to replace caller's input
+ to make it understandable by models.py.
+ """
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ replaced_kwargs = {}
+ for key, value in kwargs.items():
+ if key in kwarg_mapping:
+ replaced_kwargs[kwarg_mapping[key]] = value
+ else:
+ replaced_kwargs[key] = value
+ return func(*args, **replaced_kwargs)
+ return wrapper
+ return decorator
+
+
+def supported_filters(
+ support_keys=[],
+ optional_support_keys=[],
+ ignore_support_keys=[],
+):
+ """Decorator to check kwargs keys.
+
+ keys in kwargs and in ignore_support_keys will be removed.
+ If any unsupported keys found, a InvalidParameter
+ exception raises.
+
+ Args:
+ support_keys: keys that must exist.
+ optional_support_keys: keys that may exist.
+ ignore_support_keys: keys should be ignored.
+
+ Assumption: args without default value is supposed to exist.
+ You can add them in support_keys or not but we will make sure
+ it appears when we call the decorated function.
+ We do best match on both args and kwargs to make sure if the
+ key appears or not.
+
+ Examples:
+ decorated func: func(a, b, c=3, d=4, **kwargs)
+
+ support_keys=['e'] and call func(e=5):
+ raises: InvalidParameter: missing declared arg
+ support_keys=['e'] and call func(1,2,3,4,5,e=6):
+ raises: InvalidParameter: caller sending more args
+ support_keys=['e'] and call func(1,2):
+ raises: InvalidParameter: supported keys ['e'] missing
+ support_keys=['d', 'e'] and call func(1,2,e=3):
+ raises: InvalidParameter: supported keys ['d'] missing
+ support_keys=['d', 'e'] and call func(1,2,d=4, e=3):
+ passed
+ support_keys=['d'], optional_support_keys=['e']
+ and call func(1,2, d=3):
+ passed
+ support_keys=['d'], optional_support_keys=['e']
+ and call func(1,2, d=3, e=4, f=5):
+ raises: InvalidParameter: unsupported keys ['f']
+ support_keys=['d'], optional_support_keys=['e'],
+ ignore_support_keys=['f']
+ and call func(1,2, d=3, e=4, f=5):
+ passed to decorated keys: func(1,2, d=3, e=4)
+ """
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(*args, **filters):
+ wrapped_func = get_wrapped_func(func)
+ argspec = inspect.getargspec(wrapped_func)
+ wrapped_args = argspec.args
+ args_defaults = argspec.defaults
+ # wrapped_must_args are positional args caller must pass in.
+ if args_defaults:
+ wrapped_must_args = wrapped_args[:-len(args_defaults)]
+ else:
+ wrapped_must_args = wrapped_args[:]
+ # make sure any positional args without default value in
+ # decorated function should appear in args or filters.
+ if len(args) < len(wrapped_must_args):
+ remain_args = wrapped_must_args[len(args):]
+ for remain_arg in remain_args:
+ if remain_arg not in filters:
+ raise exception.InvalidParameter(
+ 'function missing declared arg %s '
+ 'while caller sends args %s' % (
+ remain_arg, args
+ )
+ )
+ # make sure args should be no more than positional args
+ # declared in decorated function.
+ if len(args) > len(wrapped_args):
+ raise exception.InvalidParameter(
+ 'function definition args %s while the caller '
+ 'sends args %s' % (
+ wrapped_args, args
+ )
+ )
+ # exist_args are positional args caller has given.
+ exist_args = dict(zip(wrapped_args, args)).keys()
+ must_support_keys = set(support_keys)
+ all_support_keys = must_support_keys | set(optional_support_keys)
+ wrapped_supported_keys = set(filters) | set(exist_args)
+ unsupported_keys = (
+ set(filters) - set(wrapped_args) -
+ all_support_keys - set(ignore_support_keys)
+ )
+ # unsupported_keys are the keys that are not in support_keys,
+ # optional_support_keys, ignore_support_keys and are not passed in
+ # by positional args. It means the decorated function may
+ # not understand these parameters.
+ if unsupported_keys:
+ raise exception.InvalidParameter(
+ 'filter keys %s are not supported for %s' % (
+ list(unsupported_keys), wrapped_func
+ )
+ )
+ # missing_keys are the keys that must exist but missing in
+ # both positional args or kwargs.
+ missing_keys = must_support_keys - wrapped_supported_keys
+ if missing_keys:
+ raise exception.InvalidParameter(
+ 'filter keys %s not found for %s' % (
+ list(missing_keys), wrapped_func
+ )
+ )
+ # We filter kwargs to eliminate ignore_support_keys in kwargs
+ # passed to decorated function.
+ filtered_filters = dict([
+ (key, value)
+ for key, value in filters.items()
+ if key not in ignore_support_keys
+ ])
+ return func(*args, **filtered_filters)
+ return wrapper
+ return decorator
+
+
+def input_filters(
+ **filters
+):
+ """Decorator to filter kwargs.
+
+ For key in kwargs, if the key exists and filters
+ and the return of call filters[key] is False, the key
+ will be removed from kwargs.
+
+ The function definition of filters[key] is
+ func(value, *args, **kwargs) compared with decorated
+ function func(*args, **kwargs)
+
+ The function is used to filter kwargs in case some
+ kwargs should be removed conditionally depends on the
+ related filters.
+
+ Examples:
+ filters={'a': func(value, *args, **kwargs)}
+ @input_filters(**filters)
+ decorated_func(*args, **kwargs)
+ func returns False.
+ Then when call decorated_func(a=1, b=2)
+ it will be actually called the decorated func with
+ b=2. a=1 will be removed since it does not pass filtering.
+ """
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ filtered_kwargs = {}
+ for key, value in kwargs.items():
+ if key in filters:
+ if filters[key](value, *args, **kwargs):
+ filtered_kwargs[key] = value
+ else:
+ logging.debug(
+ 'ignore filtered input key %s' % key
+ )
+ else:
+ filtered_kwargs[key] = value
+ return func(*args, **filtered_kwargs)
+ return wrapper
+ return decorator
+
+
+def _obj_equal_or_subset(check, obj):
+ """Used by output filter to check if obj is in check."""
+ if check == obj:
+ return True
+ if not issubclass(obj.__class__, check.__class__):
+ return False
+ if isinstance(obj, dict):
+ return _dict_equal_or_subset(check, obj)
+ elif isinstance(obj, list):
+ return _list_equal_or_subset(check, obj)
+ else:
+ return False
+
+
+def _list_equal_or_subset(check_list, obj_list):
+ """Used by output filter to check if obj_list is in check_list"""
+ if not isinstance(check_list, list):
+ return False
+ return set(check_list).issubset(set(obj_list))
+
+
+def _dict_equal_or_subset(check_dict, obj_dict):
+ """Used by output filter to check if obj_dict in check_dict."""
+ if not isinstance(check_dict, dict):
+ return False
+ for key, value in check_dict.items():
+ if (
+ key not in obj_dict or
+ not _obj_equal_or_subset(check_dict[key], obj_dict[key])
+ ):
+ return False
+ return True
+
+
+def general_filter_callback(general_filter, obj):
+ """General filter function to filter output.
+
+ Since some fields stored in database is json encoded and
+ we want to do the deep match for the json encoded field to
+ do the filtering in some cases, we introduces the output_filters
+ and general_filter_callback to deal with this kind of cases.
+
+ We do special treatment for key 'resp_eq' to check if
+ obj is the recursively subset of general_filter['resp_eq']
+
+
+ Example:
+ obj: 'b'
+ general_filter: {}
+ returns: True
+
+ obj: 'b'
+ general_filter: {'resp_in': ['a', 'b']}
+ returns: True
+
+ obj: 'b'
+ general_filter: {'resp_in': ['a']}
+ returns: False
+
+ obj: 'b'
+ general_filter: {'resp_eq': 'b'}
+ returns: True
+
+ obj: 'b'
+ general_filter: {'resp_eq': 'a'}
+ returns: False
+
+ obj: 'b'
+ general_filter: {'resp_range': ('a', 'c')}
+ returns: True
+
+ obj: 'd'
+ general_filter: {'resp_range': ('a', 'c')}
+ returns: False
+
+ If there are multi keys in dict, the output is filtered
+ by and relationship.
+
+ If the general_filter is a list, the output is filtered
+ by or relationship.
+
+ Supported general filters: [
+ 'resp_eq', 'resp_in', 'resp_lt',
+ 'resp_le', 'resp_gt', 'resp_ge',
+ 'resp_match', 'resp_range'
+ ]
+ """
+ if isinstance(general_filter, list):
+ if not general_filter:
+ return True
+ return any([
+ general_filter_callback(item, obj)
+ for item in general_filter
+ ])
+ elif isinstance(general_filter, dict):
+ if 'resp_eq' in general_filter:
+ if not _obj_equal_or_subset(
+ general_filter['resp_eq'], obj
+ ):
+ return False
+ if 'resp_in' in general_filter:
+ in_filters = general_filter['resp_in']
+ if not any([
+ _obj_equal_or_subset(in_filer, obj)
+ for in_filer in in_filters
+ ]):
+ return False
+ if 'resp_lt' in general_filter:
+ if obj >= general_filter['resp_lt']:
+ return False
+ if 'resp_le' in general_filter:
+ if obj > general_filter['resp_le']:
+ return False
+ if 'resp_gt' in general_filter:
+ if obj <= general_filter['resp_gt']:
+ return False
+ if 'resp_ge' in general_filter:
+ if obj < general_filter['resp_gt']:
+ return False
+ if 'resp_match' in general_filter:
+ if not re.match(general_filter['resp_match'], obj):
+ return False
+ if 'resp_range' in general_filter:
+ resp_range = general_filter['resp_range']
+ if not isinstance(resp_range, list):
+ resp_range = [resp_range]
+ in_range = False
+ for range_start, range_end in resp_range:
+ if range_start <= obj <= range_end:
+ in_range = True
+ if not in_range:
+ return False
+ return True
+ else:
+ return True
+
+
+def filter_output(filter_callbacks, kwargs, obj, missing_ok=False):
+ """Filter ouput.
+
+ For each key in filter_callbacks, if it exists in kwargs,
+ kwargs[key] tells what we need to filter. If the call of
+ filter_callbacks[key] returns False, it tells the obj should be
+ filtered out of output.
+ """
+ for callback_key, callback_value in filter_callbacks.items():
+ if callback_key not in kwargs:
+ continue
+ if callback_key not in obj:
+ if missing_ok:
+ continue
+ else:
+ raise exception.InvalidResponse(
+ '%s is not in %s' % (callback_key, obj)
+ )
+ if not callback_value(
+ kwargs[callback_key], obj[callback_key]
+ ):
+ return False
+ return True
+
+
+def output_filters(missing_ok=False, **filter_callbacks):
+ """Decorator to filter output list.
+
+ Each filter_callback should have the definition like:
+ func({'resp_eq': 'a'}, 'a')
+ """
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ filtered_obj_list = []
+ obj_list = func(*args, **kwargs)
+ for obj in obj_list:
+ if filter_output(
+ filter_callbacks, kwargs, obj, missing_ok
+ ):
+ filtered_obj_list.append(obj)
+ return filtered_obj_list
+ return wrapper
+ return decorator
+
+
+def _input_validates(args_validators, kwargs_validators, *args, **kwargs):
+ """Used by input_validators to validate inputs."""
+ for i, value in enumerate(args):
+ if i < len(args_validators) and args_validators[i]:
+ args_validators[i](value)
+ for key, value in kwargs.items():
+ if kwargs_validators.get(key):
+ kwargs_validators[key](value)
+
+
+def input_validates(*args_validators, **kwargs_validators):
+ """Decorator to validate input.
+
+ Each validator should have definition like:
+ func('00:01:02:03:04:05')
+ """
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ _input_validates(
+ args_validators, kwargs_validators,
+ *args, **kwargs
+ )
+ return func(*args, **kwargs)
+ return wrapper
+ return decorator
+
+
+def _input_validates_with_args(
+ args_validators, kwargs_validators, *args, **kwargs
+):
+ """Validate input with validators.
+
+ Each validator takes the arguments of the decorated function
+ as its arguments. The function definition is like:
+ func(value, *args, **kwargs) compared with the decorated
+ function func(*args, **kwargs).
+ """
+ for i, value in enumerate(args):
+ if i < len(args_validators) and args_validators[i]:
+ args_validators[i](value, *args, **kwargs)
+ for key, value in kwargs.items():
+ if kwargs_validators.get(key):
+ kwargs_validators[key](value, *args, **kwargs)
+
+
+def input_validates_with_args(
+ *args_validators, **kwargs_validators
+):
+ """Decorator to validate input."""
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ _input_validates_with_args(
+ args_validators, kwargs_validators,
+ *args, **kwargs
+ )
+ return func(*args, **kwargs)
+ return wrapper
+ return decorator
+
+
+def _output_validates_with_args(
+ kwargs_validators, obj, *args, **kwargs
+):
+ """Validate output with validators.
+
+ Each validator takes the arguments of the decorated function
+ as its arguments. The function definition is like:
+ func(value, *args, **kwargs) compared with the decorated
+ function func(*args, **kwargs).
+ """
+ if isinstance(obj, list):
+ for item in obj:
+ _output_validates_with_args(
+ kwargs_validators, item, *args, **kwargs
+ )
+ return
+ if isinstance(obj, models.HelperMixin):
+ obj = obj.to_dict()
+ if not isinstance(obj, dict):
+ raise exception.InvalidResponse(
+ 'response %s type is not dict' % str(obj)
+ )
+ try:
+ for key, value in obj.items():
+ if key in kwargs_validators:
+ kwargs_validators[key](value, *args, **kwargs)
+ except Exception as error:
+ logging.exception(error)
+ raise error
+
+
+def output_validates_with_args(**kwargs_validators):
+ """Decorator to validate output.
+
+ The validator can take the arguments of the decorated
+ function as its arguments.
+ """
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ obj = func(*args, **kwargs)
+ if isinstance(obj, list):
+ for obj_item in obj:
+ _output_validates_with_args(
+ kwargs_validators, obj_item,
+ *args, **kwargs
+ )
+ else:
+ _output_validates_with_args(
+ kwargs_validators, obj,
+ *args, **kwargs
+ )
+ return obj
+ return wrapper
+ return decorator
+
+
+def _output_validates(kwargs_validators, obj):
+ """Validate output.
+
+ Each validator has following signature:
+ func(value)
+ """
+ if isinstance(obj, list):
+ for item in obj:
+ _output_validates(kwargs_validators, item)
+ return
+ if isinstance(obj, models.HelperMixin):
+ obj = obj.to_dict()
+ if not isinstance(obj, dict):
+ raise exception.InvalidResponse(
+ 'response %s type is not dict' % str(obj)
+ )
+ try:
+ for key, value in obj.items():
+ if key in kwargs_validators:
+ kwargs_validators[key](value)
+ except Exception as error:
+ logging.exception(error)
+ raise error
+
+
+def output_validates(**kwargs_validators):
+ """Decorator to validate output."""
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ obj = func(*args, **kwargs)
+ if isinstance(obj, list):
+ for obj_item in obj:
+ _output_validates(kwargs_validators, obj_item)
+ else:
+ _output_validates(kwargs_validators, obj)
+ return obj
+ return wrapper
+ return decorator
+
+
+def get_db_object(session, table, exception_when_missing=True, **kwargs):
+ """Get db object.
+
+ If not exception_when_missing and the db object can not be found,
+ return None instead of raising exception.
+ """
+ if not session:
+ raise exception.DatabaseException('session param is None')
+ with session.begin(subtransactions=True):
+ logging.debug(
+ 'session %s get db object %s from table %s',
+ id(session), kwargs, table.__name__)
+ db_object = model_filter(
+ model_query(session, table), table, **kwargs
+ ).first()
+ logging.debug(
+ 'session %s got db object %s', id(session), db_object
+ )
+ if db_object:
+ return db_object
+
+ if not exception_when_missing:
+ return None
+
+ raise exception.RecordNotExists(
+ 'Cannot find the record in table %s: %s' % (
+ table.__name__, kwargs
+ )
+ )
+
+
+def add_db_object(session, table, exception_when_existing=True,
+ *args, **kwargs):
+ """Create db object.
+
+ If not exception_when_existing and the db object exists,
+ Instead of raising exception, updating the existing db object.
+ """
+ if not session:
+ raise exception.DatabaseException('session param is None')
+ with session.begin(subtransactions=True):
+ logging.debug(
+ 'session %s add object %s atributes %s to table %s',
+ id(session), args, kwargs, table.__name__)
+ argspec = inspect.getargspec(table.__init__)
+ arg_names = argspec.args[1:]
+ arg_defaults = argspec.defaults
+ if not arg_defaults:
+ arg_defaults = []
+ if not (
+ len(arg_names) - len(arg_defaults) <= len(args) <= len(arg_names)
+ ):
+ raise exception.InvalidParameter(
+ 'arg names %s does not match arg values %s' % (
+ arg_names, args)
+ )
+ db_keys = dict(zip(arg_names, args))
+ if db_keys:
+ db_object = session.query(table).filter_by(**db_keys).first()
+ else:
+ db_object = None
+
+ new_object = False
+ if db_object:
+ logging.debug(
+ 'got db object %s: %s', db_keys, db_object
+ )
+ if exception_when_existing:
+ raise exception.DuplicatedRecord(
+ '%s exists in table %s' % (db_keys, table.__name__)
+ )
+ else:
+ db_object = table(**db_keys)
+ new_object = True
+
+ for key, value in kwargs.items():
+ setattr(db_object, key, value)
+
+ if new_object:
+ session.add(db_object)
+ session.flush()
+ db_object.initialize()
+ db_object.validate()
+ logging.debug(
+ 'session %s db object %s added', id(session), db_object
+ )
+ return db_object
+
+
+def list_db_objects(session, table, order_by=[], **filters):
+ """List db objects.
+
+ If order by given, the db objects should be sorted by the ordered keys.
+ """
+ if not session:
+ raise exception.DatabaseException('session param is None')
+ with session.begin(subtransactions=True):
+ logging.debug(
+ 'session %s list db objects by filters %s in table %s',
+ id(session), filters, table.__name__
+ )
+ db_objects = model_order_by(
+ model_filter(
+ model_query(session, table),
+ table,
+ **filters
+ ),
+ table,
+ order_by
+ ).all()
+ logging.debug(
+ 'session %s got listed db objects: %s',
+ id(session), db_objects
+ )
+ return db_objects
+
+
+def del_db_objects(session, table, **filters):
+ """delete db objects."""
+ if not session:
+ raise exception.DatabaseException('session param is None')
+ with session.begin(subtransactions=True):
+ logging.debug(
+ 'session %s delete db objects by filters %s in table %s',
+ id(session), filters, table.__name__
+ )
+ query = model_filter(
+ model_query(session, table), table, **filters
+ )
+ db_objects = query.all()
+ query.delete(synchronize_session=False)
+ logging.debug(
+ 'session %s db objects %s deleted', id(session), db_objects
+ )
+ return db_objects
+
+
+def update_db_objects(session, table, updates={}, **filters):
+ """Update db objects."""
+ if not session:
+ raise exception.DatabaseException('session param is None')
+ with session.begin(subtransactions=True):
+ logging.debug(
+ 'session %s update db objects by filters %s in table %s',
+ id(session), filters, table.__name__)
+ db_objects = model_filter(
+ model_query(session, table), table, **filters
+ ).all()
+ for db_object in db_objects:
+ logging.debug('update db object %s: %s', db_object, updates)
+ update_db_object(session, db_object, **updates)
+ logging.debug(
+ 'session %s db objects %s updated',
+ id(session), db_objects
+ )
+ return db_objects
+
+
+def update_db_object(session, db_object, **kwargs):
+ """Update db object."""
+ if not session:
+ raise exception.DatabaseException('session param is None')
+ with session.begin(subtransactions=True):
+ logging.debug(
+ 'session %s update db object %s by value %s',
+ id(session), db_object, kwargs
+ )
+ for key, value in kwargs.items():
+ setattr(db_object, key, value)
+ session.flush()
+ db_object.update()
+ db_object.validate()
+ logging.debug(
+ 'session %s db object %s updated',
+ id(session), db_object
+ )
+ return db_object
+
+
+def del_db_object(session, db_object):
+ """Delete db object."""
+ if not session:
+ raise exception.DatabaseException('session param is None')
+ with session.begin(subtransactions=True):
+ logging.debug(
+ 'session %s delete db object %s',
+ id(session), db_object
+ )
+ session.delete(db_object)
+ logging.debug(
+ 'session %s db object %s deleted',
+ id(session), db_object
+ )
+ return db_object
+
+
+def check_ip(ip):
+ """Check ip is ip address formatted."""
+ try:
+ netaddr.IPAddress(ip)
+ except Exception as error:
+ logging.exception(error)
+ raise exception.InvalidParameter(
+ 'ip address %s format uncorrect' % ip
+ )
+
+
+def check_mac(mac):
+ """Check mac is mac address formatted."""
+ try:
+ netaddr.EUI(mac)
+ except Exception as error:
+ logging.exception(error)
+ raise exception.InvalidParameter(
+ 'invalid mac address %s' % mac
+ )
+
+
+NAME_PATTERN = re.compile(r'[a-zA-Z0-9][a-zA-Z0-9_-]*')
+
+
+def check_name(name):
+ """Check name meeting name format requirement."""
+ if not NAME_PATTERN.match(name):
+ raise exception.InvalidParameter(
+ 'name %s does not match the pattern %s' % (
+ name, NAME_PATTERN.pattern
+ )
+ )
+
+
+def _check_ipmi_credentials_ip(ip):
+ check_ip(ip)
+
+
+def check_ipmi_credentials(ipmi_credentials):
+ """Check ipmi credentials format is correct."""
+ if not ipmi_credentials:
+ return
+ if not isinstance(ipmi_credentials, dict):
+ raise exception.InvalidParameter(
+ 'invalid ipmi credentials %s' % ipmi_credentials
+
+ )
+ for key in ipmi_credentials:
+ if key not in ['ip', 'username', 'password']:
+ raise exception.InvalidParameter(
+ 'unrecognized field %s in ipmi credentials %s' % (
+ key, ipmi_credentials
+ )
+ )
+ for key in ['ip', 'username', 'password']:
+ if key not in ipmi_credentials:
+ raise exception.InvalidParameter(
+ 'no field %s in ipmi credentials %s' % (
+ key, ipmi_credentials
+ )
+ )
+ check_ipmi_credential_field = '_check_ipmi_credentials_%s' % key
+ this_module = globals()
+ if check_ipmi_credential_field in this_module:
+ this_module[check_ipmi_credential_field](
+ ipmi_credentials[key]
+ )
+ else:
+ logging.debug(
+ 'function %s is not defined', check_ipmi_credential_field
+ )
+
+
+def _check_switch_credentials_version(version):
+ if version not in ['1', '2c', '3']:
+ raise exception.InvalidParameter(
+ 'unknown snmp version %s' % version
+ )
+
+
+def check_switch_credentials(credentials):
+ """Check switch credentials format is correct."""
+ if not credentials:
+ return
+ if not isinstance(credentials, dict):
+ raise exception.InvalidParameter(
+ 'credentials %s is not dict' % credentials
+ )
+ for key in credentials:
+ if key not in ['version', 'community']:
+ raise exception.InvalidParameter(
+ 'unrecognized key %s in credentials %s' % (key, credentials)
+ )
+ for key in ['version', 'community']:
+ if key not in credentials:
+ raise exception.InvalidParameter(
+ 'there is no %s field in credentials %s' % (key, credentials)
+ )
+
+ key_check_func_name = '_check_switch_credentials_%s' % key
+ this_module = globals()
+ if key_check_func_name in this_module:
+ this_module[key_check_func_name](
+ credentials[key]
+ )
+ else:
+ logging.debug(
+ 'function %s is not defined',
+ key_check_func_name
+ )
diff --git a/compass-tasks/db/callback.py b/compass-tasks/db/callback.py
new file mode 100644
index 0000000..35798bc
--- /dev/null
+++ b/compass-tasks/db/callback.py
@@ -0,0 +1,204 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Metadata Callback methods."""
+import logging
+import netaddr
+import random
+import re
+import socket
+
+from compass.db import exception
+from compass.utils import setting_wrapper as setting
+from compass.utils import util
+
+
+CALLBACK_GLOBALS = globals()
+CALLBACK_LOCALS = locals()
+CALLBACK_CONFIGS = util.load_configs(
+ setting.CALLBACK_DIR,
+ config_name_suffix='.py',
+ env_globals=CALLBACK_GLOBALS,
+ env_locals=CALLBACK_LOCALS
+)
+for callback_config in CALLBACK_CONFIGS:
+ CALLBACK_LOCALS.update(callback_config)
+
+
+def default_proxy(name, **kwargs):
+ return setting.COMPASS_SUPPORTED_PROXY
+
+
+def proxy_options(name, **kwargs):
+ return [setting.COMPASS_SUPPORTED_PROXY]
+
+
+def default_noproxy(name, **kwargs):
+ return setting.COMPASS_SUPPORTED_DEFAULT_NOPROXY
+
+
+def noproxy_options(name, **kwargs):
+ return setting.COMPASS_SUPPORTED_DEFAULT_NOPROXY
+
+
+def default_ntp_server(name, **kwargs):
+ return setting.COMPASS_SUPPORTED_NTP_SERVER
+
+
+def ntp_server_options(name, **kwargs):
+ return setting.COMPASS_SUPPORTED_NTP_SERVER
+
+
+def default_dns_servers(name, **kwargs):
+ return setting.COMPASS_SUPPORTED_DNS_SERVERS
+
+
+def dns_servers_options(name, **kwargs):
+ return setting.COMPASS_SUPPORTED_DNS_SERVERS
+
+
+def default_domain(name, **kwargs):
+ if setting.COMPASS_SUPPORTED_DOMAINS:
+ return setting.COMPASS_SUPPORTED_DOMAINS[0]
+ else:
+ return None
+
+
+def domain_options(name, **kwargs):
+ return setting.COMPASS_SUPPORTED_DOMAINS
+
+
+def default_search_path(name, **kwargs):
+ return setting.COMPASS_SUPPORTED_DOMAINS
+
+
+def search_path_options(name, **kwargs):
+ return setting.COMPASS_SUPPORTED_DOMAINS
+
+
+def default_gateway(name, **kwargs):
+ return setting.COMPASS_SUPPORTED_DEFAULT_GATEWAY
+
+
+def default_gateway_options(name, **kwargs):
+ return [setting.COMPASS_SUPPORTED_DEFAULT_GATEWAY]
+
+
+def default_localrepo(name, **kwargs):
+ return setting.COMPASS_SUPPORTED_LOCAL_REPO
+
+
+def default_localrepo_options(name, **kwargs):
+ return [setting.COMPASS_SUPPORTED_LOCAL_REPO]
+
+
+def autofill_callback_default(name, config, **kwargs):
+ if config is None:
+ if (
+ 'autofill_types' not in kwargs or
+ not (set(kwargs['autofill_types']) & set(kwargs))
+ ):
+ return None
+ if 'default_value' not in kwargs:
+ return None
+ return kwargs['default_value']
+ return config
+
+
+def autofill_callback_random_option(name, config, **kwargs):
+ if config is None:
+ if (
+ 'autofill_types' not in kwargs or
+ not (set(kwargs['autofill_types']) & set(kwargs))
+ ):
+ return None
+ if 'options' not in kwargs or not kwargs['options']:
+ return None
+ return random.choice(kwargs['options'])
+ return config
+
+
+def autofill_no_proxy(name, config, **kwargs):
+ logging.debug(
+ 'autofill %s config %s by params %s',
+ name, config, kwargs
+ )
+ if 'cluster' in kwargs:
+ if config is None:
+ config = []
+ if 'default_value' in kwargs:
+ for default_no_proxy in kwargs['default_value']:
+ if default_no_proxy and default_no_proxy not in config:
+ config.append(default_no_proxy)
+ cluster = kwargs['cluster']
+ for clusterhost in cluster.clusterhosts:
+ host = clusterhost.host
+ hostname = host.name
+ if hostname not in config:
+ config.append(hostname)
+ for host_network in host.host_networks:
+ if host_network.is_mgmt:
+ ip = host_network.ip
+ if ip not in config:
+ config.append(ip)
+ if not config:
+ return config
+ return [no_proxy for no_proxy in config if no_proxy]
+
+
+def autofill_network_mapping(name, config, **kwargs):
+ logging.debug(
+ 'autofill %s config %s by params %s',
+ name, config, kwargs
+ )
+ if not config:
+ return config
+ if isinstance(config, basestring):
+ config = {
+ 'interface': config,
+ 'subnet': None
+ }
+ if not isinstance(config, dict):
+ return config
+ if 'interface' not in config:
+ return config
+ subnet = None
+ interface = config['interface']
+ if 'cluster' in kwargs:
+ cluster = kwargs['cluster']
+ for clusterhost in cluster.clusterhosts:
+ host = clusterhost.host
+ for host_network in host.host_networks:
+ if host_network.interface == interface:
+ subnet = host_network.subnet.subnet
+ elif 'clusterhost' in kwargs:
+ clusterhost = kwargs['clusterhost']
+ host = clusterhost.host
+ for host_network in host.host_networks:
+ if host_network.interface == interface:
+ subnet = host_network.subnet.subnet
+ if not subnet:
+ raise exception.InvalidParameter(
+ 'interface %s not found in host(s)' % interface
+ )
+ if 'subnet' not in config or not config['subnet']:
+ config['subnet'] = subnet
+ else:
+ if config['subnet'] != subnet:
+ raise exception.InvalidParameter(
+ 'subnet %s in config is not equal to subnet %s in hosts' % (
+ config['subnet'], subnet
+ )
+ )
+ return config
diff --git a/compass-tasks/db/config_validation/__init__.py b/compass-tasks/db/config_validation/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/compass-tasks/db/config_validation/__init__.py
diff --git a/compass-tasks/db/config_validation/default_validator.py b/compass-tasks/db/config_validation/default_validator.py
new file mode 100644
index 0000000..224447f
--- /dev/null
+++ b/compass-tasks/db/config_validation/default_validator.py
@@ -0,0 +1,131 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Default config validation function."""
+
+from sqlalchemy import or_
+
+from compass.db.models import OSConfigField
+from compass.db.models import OSConfigMetadata
+from compass.db import validator
+
+MAPPER = {
+ "os_id": {
+ "metaTable": OSConfigMetadata,
+ "metaFieldTable": OSConfigField
+ }
+ # "adapter_id": {
+ # "metaTable": AdapterConfigMetadata,
+ # "metaFieldTable": AdapterConfigField
+ # }
+}
+
+
+def validate_config(session, config, id_name, id_value, patch=True):
+ """Validates config.
+
+ Validates the given config value according to the config
+ metadata of the asscoiated os_id or adapter_id. Returns
+ a tuple (status, message).
+ """
+ if id_name not in MAPPER.keys():
+ return (False, "Invalid id type %s" % id_name)
+
+ meta_table = MAPPER[id_name]['metaTable']
+ metafield_table = MAPPER[id_name]['metaFieldTable']
+ with session.begin(subtransactions=True):
+ name_col = name_col = getattr(meta_table, 'name')
+ id_col = getattr(meta_table, id_name)
+
+ return _validate_config_helper(session, config,
+ name_col, id_col, id_value,
+ meta_table, metafield_table,
+ patch)
+
+
+def _validate_config_helper(session, config,
+ name_col, id_col, id_value,
+ meta_table, metafield_table, patch=True):
+
+ with session.begin(subtransactions=True):
+ for elem in config:
+
+ obj = session.query(meta_table).filter(name_col == elem)\
+ .filter(or_(id_col is None,
+ id_col == id_value)).first()
+
+ if not obj and "_type" not in config[elem]:
+ return (False, "Invalid metadata '%s'!" % elem)
+
+ if "_type" in config[elem]:
+ # Metadata is a variable
+ metadata_name = config[elem]['_type']
+ obj = session.query(meta_table).filter_by(name=metadata_name)\
+ .first()
+
+ if not obj:
+ err_msg = ("Invalid metatdata '%s' or missing '_type'"
+ "to indicate this is a variable metatdata."
+ % elem)
+ return (False, err_msg)
+
+ # TODO(Grace): validate metadata here
+ del config[elem]['_type']
+
+ fields = obj.fields
+
+ if not fields:
+ is_valid, message = _validate_config_helper(session,
+ config[elem],
+ name_col, id_col,
+ id_value,
+ meta_table,
+ metafield_table,
+ patch)
+ if not is_valid:
+ return (False, message)
+
+ else:
+ field_config = config[elem]
+ for key in field_config:
+ field = session.query(metafield_table)\
+ .filter_by(field=key).first()
+ if not field:
+ # The field is not in schema
+ return (False, "Invalid field '%s'!" % key)
+
+ value = field_config[key]
+ if field.is_required and value is None:
+ # The value of this field is required
+ # and cannot be none
+ err = "The value of field '%s' cannot be null!" % key
+ return (False, err)
+
+ if field.validator:
+ func = getattr(validator, field.validator)
+ if not func or not func(value):
+ err_msg = ("The value of the field '%s' is "
+ "invalid format or None!" % key)
+ return (False, err_msg)
+
+ # This is a PUT request. We need to check presence of all
+ # required fields.
+ if not patch:
+ for field in fields:
+ name = field.field
+ if field.is_required and name not in field_config:
+ return (False,
+ "Missing required field '%s'" % name)
+
+ return (True, None)
diff --git a/compass-tasks/db/config_validation/extension/__init__.py b/compass-tasks/db/config_validation/extension/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/compass-tasks/db/config_validation/extension/__init__.py
diff --git a/compass-tasks/db/config_validation/extension/openstack.py b/compass-tasks/db/config_validation/extension/openstack.py
new file mode 100644
index 0000000..6b3af69
--- /dev/null
+++ b/compass-tasks/db/config_validation/extension/openstack.py
@@ -0,0 +1,18 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def validate_cluster_config():
+ # TODO(xiaodong): Add openstack specific validation here.
+ pass
diff --git a/compass-tasks/db/exception.py b/compass-tasks/db/exception.py
new file mode 100644
index 0000000..44556c9
--- /dev/null
+++ b/compass-tasks/db/exception.py
@@ -0,0 +1,116 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Custom exception"""
+import traceback
+
+
+class DatabaseException(Exception):
+ """Base class for all database exceptions."""
+ def __init__(self, message):
+ super(DatabaseException, self).__init__(message)
+ self.traceback = traceback.format_exc()
+ self.status_code = 400
+
+ def to_dict(self):
+ return {'message': str(self)}
+
+
+class RecordNotExists(DatabaseException):
+ """Define the exception for referring non-existing object in DB."""
+ def __init__(self, message):
+ super(RecordNotExists, self).__init__(message)
+ self.status_code = 404
+
+
+class DuplicatedRecord(DatabaseException):
+ """Define the exception for trying to insert an existing object in DB."""
+ def __init__(self, message):
+ super(DuplicatedRecord, self).__init__(message)
+ self.status_code = 409
+
+
+class Unauthorized(DatabaseException):
+ """Define the exception for invalid user login."""
+ def __init__(self, message):
+ super(Unauthorized, self).__init__(message)
+ self.status_code = 401
+
+
+class UserDisabled(DatabaseException):
+ """Define the exception that a disabled user tries to do some operations.
+
+ """
+ def __init__(self, message):
+ super(UserDisabled, self).__init__(message)
+ self.status_code = 403
+
+
+class Forbidden(DatabaseException):
+ """Define the exception that a user is trying to make some action
+
+ without the right permission.
+
+ """
+ def __init__(self, message):
+ super(Forbidden, self).__init__(message)
+ self.status_code = 403
+
+
+class NotAcceptable(DatabaseException):
+ """The data is not acceptable."""
+ def __init__(self, message):
+ super(NotAcceptable, self).__init__(message)
+ self.status_code = 406
+
+
+class InvalidParameter(DatabaseException):
+ """Define the exception that the request has invalid or missing parameters.
+
+ """
+ def __init__(self, message):
+ super(InvalidParameter, self).__init__(message)
+ self.status_code = 400
+
+
+class InvalidResponse(DatabaseException):
+ """Define the exception that the response is invalid.
+
+ """
+ def __init__(self, message):
+ super(InvalidResponse, self).__init__(message)
+ self.status_code = 400
+
+
+class MultiDatabaseException(DatabaseException):
+ """Define the exception composites with multi exceptions."""
+ def __init__(self, exceptions):
+ super(MultiDatabaseException, self).__init__('multi exceptions')
+ self.exceptions = exceptions
+ self.status_code = 400
+
+ @property
+ def traceback(self):
+ tracebacks = []
+ for exception in self.exceptions:
+ tracebacks.append(exception.trackback)
+
+ def to_dict(self):
+ dict_info = super(MultiDatabaseException, self).to_dict()
+ dict_info.update({
+ 'exceptions': [
+ exception.to_dict() for exception in self.exceptions
+ ]
+ })
+ return dict_info
diff --git a/compass-tasks/db/models.py b/compass-tasks/db/models.py
new file mode 100644
index 0000000..d4b0324
--- /dev/null
+++ b/compass-tasks/db/models.py
@@ -0,0 +1,1924 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Database model"""
+import copy
+import datetime
+import logging
+import netaddr
+import re
+import simplejson as json
+
+from sqlalchemy import BigInteger
+from sqlalchemy import Boolean
+from sqlalchemy import Column
+from sqlalchemy import ColumnDefault
+from sqlalchemy import DateTime
+from sqlalchemy import Enum
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.ext.hybrid import hybrid_property
+from sqlalchemy import Float
+from sqlalchemy import ForeignKey
+from sqlalchemy import Integer
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy import String
+from sqlalchemy import Table
+from sqlalchemy import Text
+from sqlalchemy.types import TypeDecorator
+from sqlalchemy import UniqueConstraint
+
+from compass.db import exception
+from compass.utils import util
+
+
+BASE = declarative_base()
+
+
+class JSONEncoded(TypeDecorator):
+ """Represents an immutable structure as a json-encoded string."""
+
+ impl = Text
+
+ def process_bind_param(self, value, dialect):
+ if value is not None:
+ value = json.dumps(value)
+ return value
+
+ def process_result_value(self, value, dialect):
+ if value is not None:
+ value = json.loads(value)
+ return value
+
+
+class TimestampMixin(object):
+ """Provides table fields for each row created/updated timestamp."""
+ created_at = Column(DateTime, default=lambda: datetime.datetime.now())
+ updated_at = Column(DateTime, default=lambda: datetime.datetime.now(),
+ onupdate=lambda: datetime.datetime.now())
+
+
+class HelperMixin(object):
+ """Provides general fuctions for all compass table models."""
+
+ def initialize(self):
+ self.update()
+
+ def update(self):
+ pass
+
+ @staticmethod
+ def type_compatible(value, column_type):
+ """Check if value type is compatible with the column type."""
+ if value is None:
+ return True
+ if not hasattr(column_type, 'python_type'):
+ return True
+ column_python_type = column_type.python_type
+ if isinstance(value, column_python_type):
+ return True
+ if issubclass(column_python_type, basestring):
+ return isinstance(value, basestring)
+ if column_python_type in [int, long]:
+ return type(value) in [int, long]
+ if column_python_type in [float]:
+ return type(value) in [float]
+ if column_python_type in [bool]:
+ return type(value) in [bool]
+ return False
+
+ def validate(self):
+ """Generate validate function to make sure the record is legal."""
+ columns = self.__mapper__.columns
+ for key, column in columns.items():
+ value = getattr(self, key)
+ if not self.type_compatible(value, column.type):
+ raise exception.InvalidParameter(
+ 'column %s value %r type is unexpected: %s' % (
+ key, value, column.type
+ )
+ )
+
+ def to_dict(self):
+ """General function to convert record to dict.
+
+ Convert all columns not starting with '_' to
+ {<column_name>: <column_value>}
+ """
+ keys = self.__mapper__.columns.keys()
+ dict_info = {}
+ for key in keys:
+ if key.startswith('_'):
+ continue
+ value = getattr(self, key)
+ if value is not None:
+ if isinstance(value, datetime.datetime):
+ value = util.format_datetime(value)
+ dict_info[key] = value
+ return dict_info
+
+
+class StateMixin(TimestampMixin, HelperMixin):
+ """Provides general fields and functions for state related table."""
+
+ state = Column(
+ Enum(
+ 'UNINITIALIZED', 'INITIALIZED', 'UPDATE_PREPARING',
+ 'INSTALLING', 'SUCCESSFUL', 'ERROR'
+ ),
+ ColumnDefault('UNINITIALIZED')
+ )
+ percentage = Column(Float, default=0.0)
+ message = Column(Text, default='')
+ severity = Column(
+ Enum('INFO', 'WARNING', 'ERROR'),
+ ColumnDefault('INFO')
+ )
+ ready = Column(Boolean, default=False)
+
+ def update(self):
+ # In state table, some field information is redundant.
+ # The update function to make sure all related fields
+ # are set to correct state.
+ if self.ready:
+ self.state = 'SUCCESSFUL'
+ if self.state in ['UNINITIALIZED', 'INITIALIZED']:
+ self.percentage = 0.0
+ self.severity = 'INFO'
+ self.message = ''
+ if self.state == 'INSTALLING':
+ if self.severity == 'ERROR':
+ self.state = 'ERROR'
+ elif self.percentage >= 1.0:
+ self.state = 'SUCCESSFUL'
+ self.percentage = 1.0
+ if self.state == 'SUCCESSFUL':
+ self.percentage = 1.0
+ super(StateMixin, self).update()
+
+
+class LogHistoryMixin(TimestampMixin, HelperMixin):
+ """Provides general fields and functions for LogHistory related tables."""
+ position = Column(Integer, default=0)
+ partial_line = Column(Text, default='')
+ percentage = Column(Float, default=0.0)
+ message = Column(Text, default='')
+ severity = Column(
+ Enum('ERROR', 'WARNING', 'INFO'),
+ ColumnDefault('INFO')
+ )
+ line_matcher_name = Column(
+ String(80), default='start'
+ )
+
+ def validate(self):
+ # TODO(xicheng): some validation can be moved to column.
+ if not self.filename:
+ raise exception.InvalidParameter(
+ 'filename is not set in %s' % self.id
+ )
+
+
+class HostNetwork(BASE, TimestampMixin, HelperMixin):
+ """Host network table."""
+ __tablename__ = 'host_network'
+
+ id = Column(Integer, primary_key=True)
+ host_id = Column(
+ Integer,
+ ForeignKey('host.id', onupdate='CASCADE', ondelete='CASCADE')
+ )
+ interface = Column(
+ String(80), nullable=False)
+ subnet_id = Column(
+ Integer,
+ ForeignKey('subnet.id', onupdate='CASCADE', ondelete='CASCADE')
+ )
+ user_id = Column(Integer, ForeignKey('user.id'))
+ ip_int = Column(BigInteger, nullable=False)
+ is_mgmt = Column(Boolean, default=False)
+ is_promiscuous = Column(Boolean, default=False)
+
+ __table_args__ = (
+ UniqueConstraint('host_id', 'interface', name='interface_constraint'),
+ UniqueConstraint('ip_int', 'user_id', name='ip_constraint')
+ )
+
+ def __init__(self, host_id, interface, user_id, **kwargs):
+ self.host_id = host_id
+ self.interface = interface
+ self.user_id = user_id
+ super(HostNetwork, self).__init__(**kwargs)
+
+ def __str__(self):
+ return 'HostNetwork[%s=%s]' % (self.interface, self.ip)
+
+ @property
+ def ip(self):
+ return str(netaddr.IPAddress(self.ip_int))
+
+ @ip.setter
+ def ip(self, value):
+ self.ip_int = int(netaddr.IPAddress(value))
+
+ @property
+ def netmask(self):
+ return str(netaddr.IPNetwork(self.subnet.subnet).netmask)
+
+ def update(self):
+ self.host.config_validated = False
+
+ def validate(self):
+ # TODO(xicheng): some validation can be moved to column.
+ super(HostNetwork, self).validate()
+ if not self.subnet:
+ raise exception.InvalidParameter(
+ 'subnet is not set in %s interface %s' % (
+ self.host_id, self.interface
+ )
+ )
+ if not self.ip_int:
+ raise exception.InvalidParameter(
+ 'ip is not set in %s interface %s' % (
+ self.host_id, self.interface
+ )
+ )
+ ip = netaddr.IPAddress(self.ip_int)
+ subnet = netaddr.IPNetwork(self.subnet.subnet)
+ if ip not in subnet:
+ raise exception.InvalidParameter(
+ 'ip %s is not in subnet %s' % (
+ str(ip), str(subnet)
+ )
+ )
+
+ def to_dict(self):
+ dict_info = super(HostNetwork, self).to_dict()
+ dict_info['ip'] = self.ip
+ dict_info['interface'] = self.interface
+ dict_info['netmask'] = self.netmask
+ dict_info['subnet'] = self.subnet.subnet
+ dict_info['user_id'] = self.user_id
+ return dict_info
+
+
+class ClusterHostLogHistory(BASE, LogHistoryMixin):
+ """clusterhost installing log history for each file.
+
+ """
+ __tablename__ = 'clusterhost_log_history'
+
+ clusterhost_id = Column(
+ 'id', Integer,
+ ForeignKey('clusterhost.id', onupdate='CASCADE', ondelete='CASCADE'),
+ primary_key=True
+ )
+ filename = Column(String(80), primary_key=True, nullable=False)
+ cluster_id = Column(
+ Integer,
+ ForeignKey('cluster.id')
+ )
+ host_id = Column(
+ Integer,
+ ForeignKey('host.id')
+ )
+
+ def __init__(self, clusterhost_id, filename, **kwargs):
+ self.clusterhost_id = clusterhost_id
+ self.filename = filename
+ super(ClusterHostLogHistory, self).__init__(**kwargs)
+
+ def __str__(self):
+ return 'ClusterHostLogHistory[%s:%s]' % (
+ self.clusterhost_id, self.filename
+ )
+
+ def initialize(self):
+ self.cluster_id = self.clusterhost.cluster_id
+ self.host_id = self.clusterhost.host_id
+ super(ClusterHostLogHistory, self).initialize()
+
+
+class HostLogHistory(BASE, LogHistoryMixin):
+ """host installing log history for each file.
+
+ """
+ __tablename__ = 'host_log_history'
+
+ id = Column(
+ Integer,
+ ForeignKey('host.id', onupdate='CASCADE', ondelete='CASCADE'),
+ primary_key=True)
+ filename = Column(String(80), primary_key=True, nullable=False)
+
+ def __init__(self, id, filename, **kwargs):
+ self.id = id
+ self.filename = filename
+ super(HostLogHistory, self).__init__(**kwargs)
+
+ def __str__(self):
+ return 'HostLogHistory[%s:%s]' % (self.id, self.filename)
+
+
+class ClusterHostState(BASE, StateMixin):
+ """ClusterHost state table."""
+ __tablename__ = 'clusterhost_state'
+
+ id = Column(
+ Integer,
+ ForeignKey(
+ 'clusterhost.id',
+ onupdate='CASCADE', ondelete='CASCADE'
+ ),
+ primary_key=True
+ )
+
+ def __str__(self):
+ return 'ClusterHostState[%s state %s percentage %s]' % (
+ self.id, self.state, self.percentage
+ )
+
+ def update(self):
+ """Update clusterhost state.
+
+ When clusterhost state is updated, the underlying host state
+ may be updated accordingly.
+ """
+ super(ClusterHostState, self).update()
+ host_state = self.clusterhost.host.state
+ if self.state == 'INITIALIZED':
+ if host_state.state in ['UNINITIALIZED', 'UPDATE_PREPARING']:
+ host_state.state = 'INITIALIZED'
+ host_state.update()
+ elif self.state == 'INSTALLING':
+ if host_state.state in [
+ 'UNINITIALIZED', 'UPDATE_PREPARING', 'INITIALIZED'
+ ]:
+ host_state.state = 'INSTALLING'
+ host_state.update()
+ elif self.state == 'SUCCESSFUL':
+ if host_state.state != 'SUCCESSFUL':
+ host_state.state = 'SUCCESSFUL'
+ host_state.update()
+
+
+class ClusterHost(BASE, TimestampMixin, HelperMixin):
+ """ClusterHost table."""
+ __tablename__ = 'clusterhost'
+
+ clusterhost_id = Column('id', Integer, primary_key=True)
+ cluster_id = Column(
+ Integer,
+ ForeignKey('cluster.id', onupdate='CASCADE', ondelete='CASCADE')
+ )
+ host_id = Column(
+ Integer,
+ ForeignKey('host.id', onupdate='CASCADE', ondelete='CASCADE')
+ )
+ # the list of role names.
+ _roles = Column('roles', JSONEncoded, default=[])
+ _patched_roles = Column('patched_roles', JSONEncoded, default=[])
+ config_step = Column(String(80), default='')
+ package_config = Column(JSONEncoded, default={})
+ config_validated = Column(Boolean, default=False)
+ deployed_package_config = Column(JSONEncoded, default={})
+
+ log_histories = relationship(
+ ClusterHostLogHistory,
+ passive_deletes=True, passive_updates=True,
+ cascade='all, delete-orphan',
+ backref=backref('clusterhost')
+ )
+
+ __table_args__ = (
+ UniqueConstraint('cluster_id', 'host_id', name='constraint'),
+ )
+
+ state = relationship(
+ ClusterHostState,
+ uselist=False,
+ passive_deletes=True, passive_updates=True,
+ cascade='all, delete-orphan',
+ backref=backref('clusterhost')
+ )
+
+ def __init__(self, cluster_id, host_id, **kwargs):
+ self.cluster_id = cluster_id
+ self.host_id = host_id
+ self.state = ClusterHostState()
+ super(ClusterHost, self).__init__(**kwargs)
+
+ def __str__(self):
+ return 'ClusterHost[%s:%s]' % (self.clusterhost_id, self.name)
+
+ def update(self):
+ if self.host.reinstall_os:
+ if self.state in ['SUCCESSFUL', 'ERROR']:
+ if self.config_validated:
+ self.state.state = 'INITIALIZED'
+ else:
+ self.state.state = 'UNINITIALIZED'
+ self.cluster.update()
+ self.host.update()
+ self.state.update()
+ super(ClusterHost, self).update()
+
+ @property
+ def name(self):
+ return '%s.%s' % (self.host.name, self.cluster.name)
+
+ @property
+ def patched_package_config(self):
+ return self.package_config
+
+ @patched_package_config.setter
+ def patched_package_config(self, value):
+ package_config = copy.deepcopy(self.package_config)
+ self.package_config = util.merge_dict(package_config, value)
+ logging.debug(
+ 'patch clusterhost %s package_config: %s',
+ self.clusterhost_id, value
+ )
+ self.config_validated = False
+
+ @property
+ def put_package_config(self):
+ return self.package_config
+
+ @put_package_config.setter
+ def put_package_config(self, value):
+ package_config = copy.deepcopy(self.package_config)
+ package_config.update(value)
+ self.package_config = package_config
+ logging.debug(
+ 'put clusterhost %s package_config: %s',
+ self.clusterhost_id, value
+ )
+ self.config_validated = False
+
+ @property
+ def patched_os_config(self):
+ return self.host.os_config
+
+ @patched_os_config.setter
+ def patched_os_config(self, value):
+ host = self.host
+ host.patched_os_config = value
+
+ @property
+ def put_os_config(self):
+ return self.host.os_config
+
+ @put_os_config.setter
+ def put_os_config(self, value):
+ host = self.host
+ host.put_os_config = value
+
+ @property
+ def deployed_os_config(self):
+ return self.host.deployed_os_config
+
+ @deployed_os_config.setter
+ def deployed_os_config(self, value):
+ host = self.host
+ host.deployed_os_config = value
+
+ @hybrid_property
+ def os_name(self):
+ return self.host.os_name
+
+ @os_name.expression
+ def os_name(cls):
+ return cls.host.os_name
+
+ @hybrid_property
+ def clustername(self):
+ return self.cluster.name
+
+ @clustername.expression
+ def clustername(cls):
+ return cls.cluster.name
+
+ @hybrid_property
+ def hostname(self):
+ return self.host.hostname
+
+ @hostname.expression
+ def hostname(cls):
+ return Host.hostname
+
+ @property
+ def distributed_system_installed(self):
+ return self.state.state == 'SUCCESSFUL'
+
+ @property
+ def resintall_os(self):
+ return self.host.reinstall_os
+
+ @property
+ def reinstall_distributed_system(self):
+ return self.cluster.reinstall_distributed_system
+
+ @property
+ def os_installed(self):
+ return self.host.os_installed
+
+ @property
+ def roles(self):
+ # only the role exists in flavor roles will be returned.
+ # the role will be sorted as the order defined in flavor
+ # roles.
+ # duplicate role names will be removed.
+ # The returned value is a list of dict like
+ # [{'name': 'allinone', 'optional': False}]
+ role_names = list(self._roles)
+ if not role_names:
+ return []
+ cluster_roles = self.cluster.flavor['roles']
+ if not cluster_roles:
+ return []
+ roles = []
+ for cluster_role in cluster_roles:
+ if cluster_role['name'] in role_names:
+ roles.append(cluster_role)
+ return roles
+
+ @roles.setter
+ def roles(self, value):
+ """value should be a list of role name."""
+ self._roles = list(value)
+ self.config_validated = False
+
+ @property
+ def patched_roles(self):
+ patched_role_names = list(self._patched_roles)
+ if not patched_role_names:
+ return []
+ cluster_roles = self.cluster.flavor['roles']
+ if not cluster_roles:
+ return []
+ roles = []
+ for cluster_role in cluster_roles:
+ if cluster_role['name'] in patched_role_names:
+ roles.append(cluster_role)
+ return roles
+
+ @patched_roles.setter
+ def patched_roles(self, value):
+ """value should be a list of role name."""
+ # if value is an empty list, we empty the field
+ if value:
+ roles = list(self._roles)
+ roles.extend(value)
+ self._roles = roles
+ patched_roles = list(self._patched_roles)
+ patched_roles.extend(value)
+ self._patched_roles = patched_roles
+ self.config_validated = False
+ else:
+ self._patched_roles = list(value)
+ self.config_validated = False
+
+ @hybrid_property
+ def owner(self):
+ return self.cluster.owner
+
+ @owner.expression
+ def owner(cls):
+ return cls.cluster.owner
+
+ def state_dict(self):
+ """Get clusterhost state dict.
+
+ The clusterhost state_dict is different from
+ clusterhost.state.to_dict. The main difference is state_dict
+ show the progress of both installing os on host and installing
+ distributed system on clusterhost. While clusterhost.state.to_dict
+ only shows the progress of installing distributed system on
+ clusterhost.
+ """
+ cluster = self.cluster
+ host = self.host
+ host_state = host.state_dict()
+ if not cluster.flavor_name:
+ return host_state
+ clusterhost_state = self.state.to_dict()
+ if clusterhost_state['state'] in ['ERROR', 'SUCCESSFUL']:
+ return clusterhost_state
+ if (
+ clusterhost_state['state'] in 'INSTALLING' and
+ clusterhost_state['percentage'] > 0
+ ):
+ clusterhost_state['percentage'] = min(
+ 1.0, (
+ 0.5 + clusterhost_state['percentage'] / 2
+ )
+ )
+ return clusterhost_state
+
+ host_state['percentage'] = host_state['percentage'] / 2
+ if host_state['state'] == 'SUCCESSFUL':
+ host_state['state'] = 'INSTALLING'
+ return host_state
+
+ def to_dict(self):
+ dict_info = self.host.to_dict()
+ dict_info.update(super(ClusterHost, self).to_dict())
+ state_dict = self.state_dict()
+ dict_info.update({
+ 'distributed_system_installed': self.distributed_system_installed,
+ 'reinstall_distributed_system': self.reinstall_distributed_system,
+ 'owner': self.owner,
+ 'clustername': self.clustername,
+ 'name': self.name,
+ 'state': state_dict['state']
+ })
+ dict_info['roles'] = self.roles
+ dict_info['patched_roles'] = self.patched_roles
+ return dict_info
+
+
+class HostState(BASE, StateMixin):
+ """Host state table."""
+ __tablename__ = 'host_state'
+
+ id = Column(
+ Integer,
+ ForeignKey('host.id', onupdate='CASCADE', ondelete='CASCADE'),
+ primary_key=True
+ )
+
+ def __str__(self):
+ return 'HostState[%s state %s percentage %s]' % (
+ self.id, self.state, self.percentage
+ )
+
+ def update(self):
+ """Update host state.
+
+ When host state is updated, all clusterhosts on the
+ host will update their state if necessary.
+ """
+ super(HostState, self).update()
+ host = self.host
+ if self.state == 'INSTALLING':
+ host.reinstall_os = False
+ for clusterhost in self.host.clusterhosts:
+ if clusterhost.state in [
+ 'SUCCESSFUL', 'ERROR'
+ ]:
+ clusterhost.state = 'INSTALLING'
+ clusterhost.state.update()
+ elif self.state == 'UNINITIALIZED':
+ for clusterhost in self.host.clusterhosts:
+ if clusterhost.state in [
+ 'INITIALIZED', 'INSTALLING', 'SUCCESSFUL', 'ERROR'
+ ]:
+ clusterhost.state = 'UNINITIALIZED'
+ clusterhost.state.update()
+ elif self.state == 'UPDATE_PREPARING':
+ for clusterhost in self.host.clusterhosts:
+ if clusterhost.state in [
+ 'INITIALIZED', 'INSTALLING', 'SUCCESSFUL', 'ERROR'
+ ]:
+ clusterhost.state = 'UPDATE_PREPARING'
+ clusterhost.state.update()
+ elif self.state == 'INITIALIZED':
+ for clusterhost in self.host.clusterhosts:
+ if clusterhost.state in [
+ 'INSTALLING', 'SUCCESSFUL', 'ERROR'
+ ]:
+ clusterhost.state = 'INITIALIZED'
+ clusterhost.state.update()
+
+
+class Host(BASE, TimestampMixin, HelperMixin):
+ """Host table."""
+ __tablename__ = 'host'
+
+ name = Column(String(80), nullable=True)
+ config_step = Column(String(80), default='')
+ os_config = Column(JSONEncoded, default={})
+ config_validated = Column(Boolean, default=False)
+ deployed_os_config = Column(JSONEncoded, default={})
+ os_name = Column(String(80))
+ creator_id = Column(Integer, ForeignKey('user.id'))
+ owner = Column(String(80))
+ os_installer = Column(JSONEncoded, default={})
+
+ __table_args__ = (
+ UniqueConstraint('name', 'owner', name='constraint'),
+ )
+
+ id = Column(
+ Integer,
+ ForeignKey('machine.id', onupdate='CASCADE', ondelete='CASCADE'),
+ primary_key=True
+ )
+ reinstall_os = Column(Boolean, default=True)
+
+ host_networks = relationship(
+ HostNetwork,
+ passive_deletes=True, passive_updates=True,
+ cascade='all, delete-orphan',
+ backref=backref('host')
+ )
+ clusterhosts = relationship(
+ ClusterHost,
+ passive_deletes=True, passive_updates=True,
+ cascade='all, delete-orphan',
+ backref=backref('host')
+ )
+ state = relationship(
+ HostState,
+ uselist=False,
+ passive_deletes=True, passive_updates=True,
+ cascade='all, delete-orphan',
+ backref=backref('host')
+ )
+ log_histories = relationship(
+ HostLogHistory,
+ passive_deletes=True, passive_updates=True,
+ cascade='all, delete-orphan',
+ backref=backref('host')
+ )
+
+ def __str__(self):
+ return 'Host[%s:%s]' % (self.id, self.name)
+
+ @hybrid_property
+ def mac(self):
+ machine = self.machine
+ if machine:
+ return machine.mac
+ else:
+ return None
+
+ @property
+ def os_id(self):
+ return self.os_name
+
+ @os_id.setter
+ def os_id(self, value):
+ self.os_name = value
+
+ @hybrid_property
+ def hostname(self):
+ return self.name
+
+ @hostname.expression
+ def hostname(cls):
+ return cls.name
+
+ @property
+ def patched_os_config(self):
+ return self.os_config
+
+ @patched_os_config.setter
+ def patched_os_config(self, value):
+ os_config = copy.deepcopy(self.os_config)
+ self.os_config = util.merge_dict(os_config, value)
+ logging.debug('patch host os config in %s: %s', self.id, value)
+ self.config_validated = False
+
+ @property
+ def put_os_config(self):
+ return self.os_config
+
+ @put_os_config.setter
+ def put_os_config(self, value):
+ os_config = copy.deepcopy(self.os_config)
+ os_config.update(value)
+ self.os_config = os_config
+ logging.debug('put host os config in %s: %s', self.id, value)
+ self.config_validated = False
+
+ def __init__(self, id, **kwargs):
+ self.id = id
+ self.state = HostState()
+ super(Host, self).__init__(**kwargs)
+
+ def update(self):
+ creator = self.creator
+ if creator:
+ self.owner = creator.email
+ if self.reinstall_os:
+ if self.state in ['SUCCESSFUL', 'ERROR']:
+ if self.config_validated:
+ self.state.state = 'INITIALIZED'
+ else:
+ self.state.state = 'UNINITIALIZED'
+ self.state.update()
+ self.state.update()
+ super(Host, self).update()
+
+ def validate(self):
+ # TODO(xicheng): some validation can be moved to the column in future.
+ super(Host, self).validate()
+ creator = self.creator
+ if not creator:
+ raise exception.InvalidParameter(
+ 'creator is not set in host %s' % self.id
+ )
+ os_name = self.os_name
+ if not os_name:
+ raise exception.InvalidParameter(
+ 'os is not set in host %s' % self.id
+ )
+ os_installer = self.os_installer
+ if not os_installer:
+ raise exception.Invalidparameter(
+ 'os_installer is not set in host %s' % self.id
+ )
+
+ @property
+ def os_installed(self):
+ return self.state.state == 'SUCCESSFUL'
+
+ @property
+ def clusters(self):
+ return [clusterhost.cluster for clusterhost in self.clusterhosts]
+
+ def state_dict(self):
+ return self.state.to_dict()
+
+ def to_dict(self):
+ """Host dict contains its underlying machine dict."""
+ dict_info = self.machine.to_dict()
+ dict_info.update(super(Host, self).to_dict())
+ state_dict = self.state_dict()
+ ip = None
+ for host_network in self.host_networks:
+ if host_network.is_mgmt:
+ ip = host_network.ip
+ dict_info.update({
+ 'machine_id': self.machine.id,
+ 'os_installed': self.os_installed,
+ 'hostname': self.hostname,
+ 'ip': ip,
+ 'networks': [
+ host_network.to_dict()
+ for host_network in self.host_networks
+ ],
+ 'os_id': self.os_id,
+ 'clusters': [cluster.to_dict() for cluster in self.clusters],
+ 'state': state_dict['state']
+ })
+ return dict_info
+
+
+class ClusterState(BASE, StateMixin):
+ """Cluster state table."""
+ __tablename__ = 'cluster_state'
+
+ id = Column(
+ Integer,
+ ForeignKey('cluster.id', onupdate='CASCADE', ondelete='CASCADE'),
+ primary_key=True
+ )
+ total_hosts = Column(
+ Integer,
+ default=0
+ )
+ installing_hosts = Column(
+ Integer,
+ default=0
+ )
+ completed_hosts = Column(
+ Integer,
+ default=0
+ )
+ failed_hosts = Column(
+ Integer,
+ default=0
+ )
+
+ def __init__(self, **kwargs):
+ super(ClusterState, self).__init__(**kwargs)
+
+ def __str__(self):
+ return 'ClusterState[%s state %s percentage %s]' % (
+ self.id, self.state, self.percentage
+ )
+
+ def to_dict(self):
+ dict_info = super(ClusterState, self).to_dict()
+ dict_info['status'] = {
+ 'total_hosts': self.total_hosts,
+ 'installing_hosts': self.installing_hosts,
+ 'completed_hosts': self.completed_hosts,
+ 'failed_hosts': self.failed_hosts
+ }
+ return dict_info
+
+ def update(self):
+ # all fields of cluster state should be calculated by
+ # its each underlying clusterhost state.
+ cluster = self.cluster
+ clusterhosts = cluster.clusterhosts
+ self.total_hosts = len(clusterhosts)
+ self.installing_hosts = 0
+ self.failed_hosts = 0
+ self.completed_hosts = 0
+ if not cluster.flavor_name:
+ for clusterhost in clusterhosts:
+ host = clusterhost.host
+ host_state = host.state.state
+ if host_state == 'INSTALLING':
+ self.installing_hosts += 1
+ elif host_state == 'ERROR':
+ self.failed_hosts += 1
+ elif host_state == 'SUCCESSFUL':
+ self.completed_hosts += 1
+ else:
+ for clusterhost in clusterhosts:
+ clusterhost_state = clusterhost.state.state
+ if clusterhost_state == 'INSTALLING':
+ self.installing_hosts += 1
+ elif clusterhost_state == 'ERROR':
+ self.failed_hosts += 1
+ elif clusterhost_state == 'SUCCESSFUL':
+ self.completed_hosts += 1
+ if self.total_hosts:
+ if self.completed_hosts == self.total_hosts:
+ self.percentage = 1.0
+ else:
+ self.percentage = (
+ float(self.completed_hosts)
+ /
+ float(self.total_hosts)
+ )
+ if self.state == 'SUCCESSFUL':
+ self.state = 'INSTALLING'
+ self.ready = False
+ self.message = (
+ 'total %s, installing %s, completed: %s, error %s'
+ ) % (
+ self.total_hosts, self.installing_hosts,
+ self.completed_hosts, self.failed_hosts
+ )
+ if self.failed_hosts:
+ self.severity = 'ERROR'
+
+ super(ClusterState, self).update()
+ if self.state == 'INSTALLING':
+ cluster.reinstall_distributed_system = False
+
+
+class Cluster(BASE, TimestampMixin, HelperMixin):
+ """Cluster table."""
+ __tablename__ = 'cluster'
+
+ id = Column(Integer, primary_key=True)
+ name = Column(String(80), nullable=False)
+ reinstall_distributed_system = Column(Boolean, default=True)
+ config_step = Column(String(80), default='')
+ os_name = Column(String(80))
+ flavor_name = Column(String(80), nullable=True)
+ # flavor dict got from flavor id.
+ flavor = Column(JSONEncoded, default={})
+ os_config = Column(JSONEncoded, default={})
+ package_config = Column(JSONEncoded, default={})
+ deployed_os_config = Column(JSONEncoded, default={})
+ deployed_package_config = Column(JSONEncoded, default={})
+ config_validated = Column(Boolean, default=False)
+ adapter_name = Column(String(80))
+ creator_id = Column(Integer, ForeignKey('user.id'))
+ owner = Column(String(80))
+ clusterhosts = relationship(
+ ClusterHost,
+ passive_deletes=True, passive_updates=True,
+ cascade='all, delete-orphan',
+ backref=backref('cluster')
+ )
+ state = relationship(
+ ClusterState,
+ uselist=False,
+ passive_deletes=True, passive_updates=True,
+ cascade='all, delete-orphan',
+ backref=backref('cluster')
+ )
+ __table_args__ = (
+ UniqueConstraint('name', 'creator_id', name='constraint'),
+ )
+
+ def __init__(self, name, creator_id, **kwargs):
+ self.name = name
+ self.creator_id = creator_id
+ self.state = ClusterState()
+ super(Cluster, self).__init__(**kwargs)
+
+ def __str__(self):
+ return 'Cluster[%s:%s]' % (self.id, self.name)
+
+ def update(self):
+ creator = self.creator
+ if creator:
+ self.owner = creator.email
+ if self.reinstall_distributed_system:
+ if self.state in ['SUCCESSFUL', 'ERROR']:
+ if self.config_validated:
+ self.state.state = 'INITIALIZED'
+ else:
+ self.state.state = 'UNINITIALIZED'
+ self.state.update()
+ self.state.update()
+ super(Cluster, self).update()
+
+ def validate(self):
+ # TODO(xicheng): some validation can be moved to column.
+ super(Cluster, self).validate()
+ creator = self.creator
+ if not creator:
+ raise exception.InvalidParameter(
+ 'creator is not set in cluster %s' % self.id
+ )
+ os_name = self.os_name
+ if not os_name:
+ raise exception.InvalidParameter(
+ 'os is not set in cluster %s' % self.id
+ )
+ adapter_name = self.adapter_name
+ if not adapter_name:
+ raise exception.InvalidParameter(
+ 'adapter is not set in cluster %s' % self.id
+ )
+ flavor_name = self.flavor_name
+ if flavor_name:
+ if 'name' not in self.flavor:
+ raise exception.InvalidParameter(
+ 'key name does not exist in flavor %s' % (
+ self.flavor
+ )
+ )
+ if flavor_name != self.flavor['name']:
+ raise exception.InvalidParameter(
+ 'flavor name %s is not match '
+ 'the name key in flavor %s' % (
+ flavor_name, self.flavor
+ )
+ )
+ else:
+ if self.flavor:
+ raise exception.InvalidParameter(
+ 'flavor %s is not empty' % self.flavor
+ )
+
+ @property
+ def os_id(self):
+ return self.os_name
+
+ @os_id.setter
+ def os_id(self, value):
+ self.os_name = value
+
+ @property
+ def adapter_id(self):
+ return self.adapter_name
+
+ @adapter_id.setter
+ def adapter_id(self, value):
+ self.adapter_name = value
+
+ @property
+ def flavor_id(self):
+ if self.flavor_name:
+ return '%s:%s' % (self.adapter_name, self.flavor_name)
+ else:
+ return None
+
+ @flavor_id.setter
+ def flavor_id(self, value):
+ if value:
+ _, flavor_name = value.split(':', 1)
+ self.flavor_name = flavor_name
+ else:
+ self.flavor_name = value
+
+ @property
+ def patched_os_config(self):
+ return self.os_config
+
+ @patched_os_config.setter
+ def patched_os_config(self, value):
+ os_config = copy.deepcopy(self.os_config)
+ self.os_config = util.merge_dict(os_config, value)
+ logging.debug('patch cluster %s os config: %s', self.id, value)
+ self.config_validated = False
+
+ @property
+ def put_os_config(self):
+ return self.os_config
+
+ @put_os_config.setter