From 8646b8d62cf4ca7b6bccae537a0c9e72ba45eab3 Mon Sep 17 00:00:00 2001 From: Harry Huang Date: Fri, 17 Nov 2017 14:53:44 +0800 Subject: Merge compass-tasks-osa and compass-tasks-k8s JIRA: COMPASS-568 rename compass-tasks to compass-tasks-base. add both osa and k8s support in compass-tasks Change-Id: I438f5b17e509d4cb751ced0ffe640ec70899882f Signed-off-by: Harry Huang --- compass-tasks-base/Dockerfile | 12 + compass-tasks-base/README.md | 1 + compass-tasks-base/actions/__init__.py | 13 + compass-tasks-base/actions/clean.py | 195 ++ compass-tasks-base/actions/cli.py | 179 ++ compass-tasks-base/actions/delete.py | 148 ++ compass-tasks-base/actions/deploy.py | 182 ++ .../actions/health_check/__init__.py | 13 + compass-tasks-base/actions/health_check/base.py | 57 + compass-tasks-base/actions/health_check/check.py | 96 + .../actions/health_check/check_apache.py | 89 + .../actions/health_check/check_celery.py | 115 + .../actions/health_check/check_dhcp.py | 184 ++ .../actions/health_check/check_dns.py | 139 ++ .../actions/health_check/check_hds.py | 97 + .../actions/health_check/check_misc.py | 219 ++ .../actions/health_check/check_os_installer.py | 151 ++ .../health_check/check_package_installer.py | 68 + .../actions/health_check/check_squid.py | 128 + .../actions/health_check/check_tftp.py | 96 + compass-tasks-base/actions/health_check/utils.py | 114 + compass-tasks-base/actions/install_callback.py | 181 ++ compass-tasks-base/actions/patch.py | 69 + compass-tasks-base/actions/poll_switch.py | 162 ++ compass-tasks-base/actions/reinstall.py | 38 + compass-tasks-base/actions/search.py | 46 + compass-tasks-base/actions/update_progress.py | 298 +++ compass-tasks-base/actions/util.py | 342 +++ compass-tasks-base/apiclient/__init__.py | 0 compass-tasks-base/apiclient/example.py | 463 ++++ compass-tasks-base/apiclient/restful.py | 1102 +++++++++ compass-tasks-base/apiclient/v1/__init__.py | 0 compass-tasks-base/apiclient/v1/example.py | 305 +++ compass-tasks-base/apiclient/v1/restful.py | 655 ++++++ compass-tasks-base/bin/switch_virtualenv.py | 29 + compass-tasks-base/build.sh | 58 + compass-tasks-base/db/__init__.py | 13 + compass-tasks-base/db/api/__init__.py | 13 + compass-tasks-base/db/api/adapter.py | 313 +++ compass-tasks-base/db/api/adapter_holder.py | 155 ++ compass-tasks-base/db/api/cluster.py | 2444 ++++++++++++++++++++ compass-tasks-base/db/api/database.py | 264 +++ compass-tasks-base/db/api/health_check_report.py | 190 ++ compass-tasks-base/db/api/host.py | 1120 +++++++++ compass-tasks-base/db/api/machine.py | 317 +++ compass-tasks-base/db/api/metadata.py | 517 +++++ compass-tasks-base/db/api/metadata_holder.py | 731 ++++++ compass-tasks-base/db/api/network.py | 160 ++ compass-tasks-base/db/api/permission.py | 357 +++ compass-tasks-base/db/api/switch.py | 1213 ++++++++++ compass-tasks-base/db/api/user.py | 553 +++++ compass-tasks-base/db/api/user_log.py | 82 + compass-tasks-base/db/api/utils.py | 1286 ++++++++++ compass-tasks-base/db/callback.py | 204 ++ .../db/config_validation/__init__.py | 0 .../db/config_validation/default_validator.py | 131 ++ .../db/config_validation/extension/__init__.py | 0 .../db/config_validation/extension/openstack.py | 18 + compass-tasks-base/db/exception.py | 116 + compass-tasks-base/db/models.py | 1924 +++++++++++++++ compass-tasks-base/db/v1/model.py | 724 ++++++ compass-tasks-base/db/validator.py | 195 ++ compass-tasks-base/deployment/__init__.py | 15 + compass-tasks-base/deployment/deploy_manager.py | 237 ++ .../deployment/installers/__init__.py | 21 + .../deployment/installers/config_manager.py | 527 +++++ .../deployment/installers/installer.py | 291 +++ .../installers/os_installers/__init__.py | 13 + .../installers/os_installers/cobbler/__init__.py | 13 + .../installers/os_installers/cobbler/cobbler.py | 449 ++++ .../installers/pk_installers/__init__.py | 13 + .../pk_installers/ansible_installer/__init__.py | 0 .../ansible_installer/ansible_installer.py | 441 ++++ compass-tasks-base/deployment/utils/__init__.py | 15 + compass-tasks-base/deployment/utils/constants.py | 84 + compass-tasks-base/hdsdiscovery/SNMP_CONFIG.md | 33 + compass-tasks-base/hdsdiscovery/__init__.py | 13 + compass-tasks-base/hdsdiscovery/base.py | 185 ++ compass-tasks-base/hdsdiscovery/error.py | 26 + compass-tasks-base/hdsdiscovery/hdmanager.py | 171 ++ compass-tasks-base/hdsdiscovery/utils.py | 289 +++ .../hdsdiscovery/vendors/__init__.py | 13 + .../hdsdiscovery/vendors/appliance/__init__.py | 0 .../hdsdiscovery/vendors/appliance/appliance.py | 34 + .../vendors/appliance/plugins/__init__.py | 0 .../hdsdiscovery/vendors/appliance/plugins/mac.py | 48 + .../hdsdiscovery/vendors/arista/__init__.py | 13 + .../hdsdiscovery/vendors/arista/arista.py | 33 + .../vendors/arista/plugins/__init__.py | 13 + .../hdsdiscovery/vendors/arista/plugins/mac.py | 24 + .../hdsdiscovery/vendors/hp/__init__.py | 13 + compass-tasks-base/hdsdiscovery/vendors/hp/hp.py | 33 + .../hdsdiscovery/vendors/hp/plugins/__init__.py | 13 + .../hdsdiscovery/vendors/hp/plugins/mac.py | 23 + .../hdsdiscovery/vendors/huawei/__init__.py | 13 + .../hdsdiscovery/vendors/huawei/huawei.py | 33 + .../vendors/huawei/plugins/__init__.py | 13 + .../hdsdiscovery/vendors/huawei/plugins/mac.py | 63 + .../hdsdiscovery/vendors/ovswitch/__init__.py | 13 + .../hdsdiscovery/vendors/ovswitch/ovswitch.py | 76 + .../vendors/ovswitch/plugins/__init__.py | 13 + .../hdsdiscovery/vendors/ovswitch/plugins/mac.py | 87 + .../hdsdiscovery/vendors/pica8/__init__.py | 13 + .../hdsdiscovery/vendors/pica8/pica8.py | 33 + .../hdsdiscovery/vendors/pica8/plugins/__init__.py | 13 + .../hdsdiscovery/vendors/pica8/plugins/mac.py | 24 + compass-tasks-base/log_analyzor/__init__.py | 13 + compass-tasks-base/log_analyzor/adapter_matcher.py | 126 + compass-tasks-base/log_analyzor/environment.py | 29 + compass-tasks-base/log_analyzor/file_matcher.py | 252 ++ compass-tasks-base/log_analyzor/line_matcher.py | 206 ++ .../log_analyzor/progress_calculator.py | 208 ++ compass-tasks-base/misc/Dockerfile | 53 + compass-tasks-base/misc/compass_install.repo | 5 + compass-tasks-base/requirements.txt | 22 + compass-tasks-base/setup.py | 97 + compass-tasks-base/start.sh | 4 + compass-tasks-base/supervisord.conf | 135 ++ compass-tasks-base/tasks/__init__.py | 13 + compass-tasks-base/tasks/client.py | 33 + compass-tasks-base/tasks/tasks.py | 326 +++ compass-tasks-base/utils/__init__.py | 13 + compass-tasks-base/utils/celeryconfig_wrapper.py | 44 + compass-tasks-base/utils/flags.py | 91 + compass-tasks-base/utils/logsetting.py | 108 + compass-tasks-base/utils/setting_wrapper.py | 175 ++ compass-tasks-base/utils/util.py | 395 ++++ compass-tasks-k8s/Dockerfile | 8 - compass-tasks-k8s/README.md | 1 - compass-tasks-k8s/run.sh | 25 - compass-tasks-osa/Dockerfile | 8 - compass-tasks-osa/run.sh | 44 - .../tacker_conf/ansible-role-requirements.yml | 192 -- compass-tasks-osa/tacker_conf/haproxy_config.yml | 285 --- .../tacker_conf/openstack_services.yml | 228 -- .../tacker_conf/os-tacker-install.yml | 63 - compass-tasks-osa/tacker_conf/setup-openstack.yml | 43 - compass-tasks-osa/tacker_conf/tacker.yml | 36 - compass-tasks-osa/tacker_conf/tacker_all.yml | 46 - compass-tasks-osa/tacker_conf/user_secrets.yml | 155 -- compass-tasks/Dockerfile | 18 +- compass-tasks/README.md | 1 - compass-tasks/actions/__init__.py | 13 - compass-tasks/actions/clean.py | 195 -- compass-tasks/actions/cli.py | 179 -- compass-tasks/actions/delete.py | 148 -- compass-tasks/actions/deploy.py | 182 -- compass-tasks/actions/health_check/__init__.py | 13 - compass-tasks/actions/health_check/base.py | 57 - compass-tasks/actions/health_check/check.py | 96 - compass-tasks/actions/health_check/check_apache.py | 89 - compass-tasks/actions/health_check/check_celery.py | 115 - compass-tasks/actions/health_check/check_dhcp.py | 184 -- compass-tasks/actions/health_check/check_dns.py | 139 -- compass-tasks/actions/health_check/check_hds.py | 97 - compass-tasks/actions/health_check/check_misc.py | 219 -- .../actions/health_check/check_os_installer.py | 151 -- .../health_check/check_package_installer.py | 68 - compass-tasks/actions/health_check/check_squid.py | 128 - compass-tasks/actions/health_check/check_tftp.py | 96 - compass-tasks/actions/health_check/utils.py | 114 - compass-tasks/actions/install_callback.py | 181 -- compass-tasks/actions/patch.py | 69 - compass-tasks/actions/poll_switch.py | 162 -- compass-tasks/actions/reinstall.py | 38 - compass-tasks/actions/search.py | 46 - compass-tasks/actions/update_progress.py | 298 --- compass-tasks/actions/util.py | 342 --- compass-tasks/apiclient/__init__.py | 0 compass-tasks/apiclient/example.py | 463 ---- compass-tasks/apiclient/restful.py | 1102 --------- compass-tasks/apiclient/v1/__init__.py | 0 compass-tasks/apiclient/v1/example.py | 305 --- compass-tasks/apiclient/v1/restful.py | 655 ------ compass-tasks/build.sh | 51 - compass-tasks/db/__init__.py | 13 - compass-tasks/db/api/__init__.py | 13 - compass-tasks/db/api/adapter.py | 313 --- compass-tasks/db/api/adapter_holder.py | 155 -- compass-tasks/db/api/cluster.py | 2444 -------------------- compass-tasks/db/api/database.py | 264 --- compass-tasks/db/api/health_check_report.py | 190 -- compass-tasks/db/api/host.py | 1120 --------- compass-tasks/db/api/machine.py | 317 --- compass-tasks/db/api/metadata.py | 517 ----- compass-tasks/db/api/metadata_holder.py | 731 ------ compass-tasks/db/api/network.py | 160 -- compass-tasks/db/api/permission.py | 357 --- compass-tasks/db/api/switch.py | 1213 ---------- compass-tasks/db/api/user.py | 553 ----- compass-tasks/db/api/user_log.py | 82 - compass-tasks/db/api/utils.py | 1286 ---------- compass-tasks/db/callback.py | 204 -- compass-tasks/db/config_validation/__init__.py | 0 .../db/config_validation/default_validator.py | 131 -- .../db/config_validation/extension/__init__.py | 0 .../db/config_validation/extension/openstack.py | 18 - compass-tasks/db/exception.py | 116 - compass-tasks/db/models.py | 1924 --------------- compass-tasks/db/v1/model.py | 724 ------ compass-tasks/db/validator.py | 195 -- compass-tasks/deployment/__init__.py | 15 - compass-tasks/deployment/deploy_manager.py | 237 -- compass-tasks/deployment/installers/__init__.py | 21 - .../deployment/installers/config_manager.py | 527 ----- compass-tasks/deployment/installers/installer.py | 291 --- .../installers/os_installers/__init__.py | 13 - .../installers/os_installers/cobbler/__init__.py | 13 - .../installers/os_installers/cobbler/cobbler.py | 449 ---- .../installers/pk_installers/__init__.py | 13 - .../pk_installers/ansible_installer/__init__.py | 0 .../ansible_installer/ansible_installer.py | 441 ---- compass-tasks/deployment/utils/__init__.py | 15 - compass-tasks/deployment/utils/constants.py | 84 - compass-tasks/hdsdiscovery/SNMP_CONFIG.md | 33 - compass-tasks/hdsdiscovery/__init__.py | 13 - compass-tasks/hdsdiscovery/base.py | 185 -- compass-tasks/hdsdiscovery/error.py | 26 - compass-tasks/hdsdiscovery/hdmanager.py | 171 -- compass-tasks/hdsdiscovery/utils.py | 289 --- compass-tasks/hdsdiscovery/vendors/__init__.py | 13 - .../hdsdiscovery/vendors/appliance/__init__.py | 0 .../hdsdiscovery/vendors/appliance/appliance.py | 34 - .../vendors/appliance/plugins/__init__.py | 0 .../hdsdiscovery/vendors/appliance/plugins/mac.py | 48 - .../hdsdiscovery/vendors/arista/__init__.py | 13 - .../hdsdiscovery/vendors/arista/arista.py | 33 - .../vendors/arista/plugins/__init__.py | 13 - .../hdsdiscovery/vendors/arista/plugins/mac.py | 24 - compass-tasks/hdsdiscovery/vendors/hp/__init__.py | 13 - compass-tasks/hdsdiscovery/vendors/hp/hp.py | 33 - .../hdsdiscovery/vendors/hp/plugins/__init__.py | 13 - .../hdsdiscovery/vendors/hp/plugins/mac.py | 23 - .../hdsdiscovery/vendors/huawei/__init__.py | 13 - .../hdsdiscovery/vendors/huawei/huawei.py | 33 - .../vendors/huawei/plugins/__init__.py | 13 - .../hdsdiscovery/vendors/huawei/plugins/mac.py | 63 - .../hdsdiscovery/vendors/ovswitch/__init__.py | 13 - .../hdsdiscovery/vendors/ovswitch/ovswitch.py | 76 - .../vendors/ovswitch/plugins/__init__.py | 13 - .../hdsdiscovery/vendors/ovswitch/plugins/mac.py | 87 - .../hdsdiscovery/vendors/pica8/__init__.py | 13 - compass-tasks/hdsdiscovery/vendors/pica8/pica8.py | 33 - .../hdsdiscovery/vendors/pica8/plugins/__init__.py | 13 - .../hdsdiscovery/vendors/pica8/plugins/mac.py | 24 - compass-tasks/log_analyzor/__init__.py | 13 - compass-tasks/log_analyzor/adapter_matcher.py | 126 - compass-tasks/log_analyzor/environment.py | 29 - compass-tasks/log_analyzor/file_matcher.py | 252 -- compass-tasks/log_analyzor/line_matcher.py | 206 -- compass-tasks/log_analyzor/progress_calculator.py | 208 -- compass-tasks/misc/Dockerfile | 53 - compass-tasks/misc/compass_install.repo | 5 - compass-tasks/requirements.txt | 23 - compass-tasks/run.sh | 63 + compass-tasks/setup-complete.yml | 7 + compass-tasks/setup.py | 97 - compass-tasks/start.sh | 4 - compass-tasks/supervisord.conf | 135 -- .../tacker_conf/ansible-role-requirements.yml | 192 ++ compass-tasks/tacker_conf/haproxy_config.yml | 285 +++ compass-tasks/tacker_conf/openstack_services.yml | 228 ++ compass-tasks/tacker_conf/os-tacker-install.yml | 63 + compass-tasks/tacker_conf/setup-openstack.yml | 43 + compass-tasks/tacker_conf/tacker.yml | 36 + compass-tasks/tacker_conf/tacker_all.yml | 46 + compass-tasks/tacker_conf/user_secrets.yml | 155 ++ compass-tasks/tasks/__init__.py | 13 - compass-tasks/tasks/client.py | 33 - compass-tasks/tasks/tasks.py | 326 --- compass-tasks/utils/__init__.py | 13 - compass-tasks/utils/celeryconfig_wrapper.py | 44 - compass-tasks/utils/flags.py | 91 - compass-tasks/utils/logsetting.py | 108 - compass-tasks/utils/setting_wrapper.py | 175 -- compass-tasks/utils/util.py | 395 ---- 276 files changed, 26046 insertions(+), 26021 deletions(-) create mode 100644 compass-tasks-base/Dockerfile create mode 100644 compass-tasks-base/README.md create mode 100644 compass-tasks-base/actions/__init__.py create mode 100644 compass-tasks-base/actions/clean.py create mode 100644 compass-tasks-base/actions/cli.py create mode 100644 compass-tasks-base/actions/delete.py create mode 100644 compass-tasks-base/actions/deploy.py create mode 100644 compass-tasks-base/actions/health_check/__init__.py create mode 100644 compass-tasks-base/actions/health_check/base.py create mode 100644 compass-tasks-base/actions/health_check/check.py create mode 100644 compass-tasks-base/actions/health_check/check_apache.py create mode 100644 compass-tasks-base/actions/health_check/check_celery.py create mode 100644 compass-tasks-base/actions/health_check/check_dhcp.py create mode 100644 compass-tasks-base/actions/health_check/check_dns.py create mode 100644 compass-tasks-base/actions/health_check/check_hds.py create mode 100644 compass-tasks-base/actions/health_check/check_misc.py create mode 100644 compass-tasks-base/actions/health_check/check_os_installer.py create mode 100644 compass-tasks-base/actions/health_check/check_package_installer.py create mode 100644 compass-tasks-base/actions/health_check/check_squid.py create mode 100644 compass-tasks-base/actions/health_check/check_tftp.py create mode 100644 compass-tasks-base/actions/health_check/utils.py create mode 100644 compass-tasks-base/actions/install_callback.py create mode 100644 compass-tasks-base/actions/patch.py create mode 100644 compass-tasks-base/actions/poll_switch.py create mode 100644 compass-tasks-base/actions/reinstall.py create mode 100644 compass-tasks-base/actions/search.py create mode 100644 compass-tasks-base/actions/update_progress.py create mode 100644 compass-tasks-base/actions/util.py create mode 100644 compass-tasks-base/apiclient/__init__.py create mode 100755 compass-tasks-base/apiclient/example.py create mode 100644 compass-tasks-base/apiclient/restful.py create mode 100644 compass-tasks-base/apiclient/v1/__init__.py create mode 100755 compass-tasks-base/apiclient/v1/example.py create mode 100644 compass-tasks-base/apiclient/v1/restful.py create mode 100644 compass-tasks-base/bin/switch_virtualenv.py create mode 100755 compass-tasks-base/build.sh create mode 100644 compass-tasks-base/db/__init__.py create mode 100644 compass-tasks-base/db/api/__init__.py create mode 100644 compass-tasks-base/db/api/adapter.py create mode 100644 compass-tasks-base/db/api/adapter_holder.py create mode 100644 compass-tasks-base/db/api/cluster.py create mode 100644 compass-tasks-base/db/api/database.py create mode 100644 compass-tasks-base/db/api/health_check_report.py create mode 100644 compass-tasks-base/db/api/host.py create mode 100644 compass-tasks-base/db/api/machine.py create mode 100644 compass-tasks-base/db/api/metadata.py create mode 100644 compass-tasks-base/db/api/metadata_holder.py create mode 100644 compass-tasks-base/db/api/network.py create mode 100644 compass-tasks-base/db/api/permission.py create mode 100644 compass-tasks-base/db/api/switch.py create mode 100644 compass-tasks-base/db/api/user.py create mode 100644 compass-tasks-base/db/api/user_log.py create mode 100644 compass-tasks-base/db/api/utils.py create mode 100644 compass-tasks-base/db/callback.py create mode 100644 compass-tasks-base/db/config_validation/__init__.py create mode 100644 compass-tasks-base/db/config_validation/default_validator.py create mode 100644 compass-tasks-base/db/config_validation/extension/__init__.py create mode 100644 compass-tasks-base/db/config_validation/extension/openstack.py create mode 100644 compass-tasks-base/db/exception.py create mode 100644 compass-tasks-base/db/models.py create mode 100644 compass-tasks-base/db/v1/model.py create mode 100644 compass-tasks-base/db/validator.py create mode 100644 compass-tasks-base/deployment/__init__.py create mode 100644 compass-tasks-base/deployment/deploy_manager.py create mode 100644 compass-tasks-base/deployment/installers/__init__.py create mode 100644 compass-tasks-base/deployment/installers/config_manager.py create mode 100644 compass-tasks-base/deployment/installers/installer.py create mode 100644 compass-tasks-base/deployment/installers/os_installers/__init__.py create mode 100644 compass-tasks-base/deployment/installers/os_installers/cobbler/__init__.py create mode 100644 compass-tasks-base/deployment/installers/os_installers/cobbler/cobbler.py create mode 100644 compass-tasks-base/deployment/installers/pk_installers/__init__.py create mode 100644 compass-tasks-base/deployment/installers/pk_installers/ansible_installer/__init__.py create mode 100644 compass-tasks-base/deployment/installers/pk_installers/ansible_installer/ansible_installer.py create mode 100644 compass-tasks-base/deployment/utils/__init__.py create mode 100644 compass-tasks-base/deployment/utils/constants.py create mode 100644 compass-tasks-base/hdsdiscovery/SNMP_CONFIG.md create mode 100644 compass-tasks-base/hdsdiscovery/__init__.py create mode 100644 compass-tasks-base/hdsdiscovery/base.py create mode 100644 compass-tasks-base/hdsdiscovery/error.py create mode 100644 compass-tasks-base/hdsdiscovery/hdmanager.py create mode 100644 compass-tasks-base/hdsdiscovery/utils.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/__init__.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/appliance/__init__.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/appliance/appliance.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/appliance/plugins/__init__.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/appliance/plugins/mac.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/arista/__init__.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/arista/arista.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/arista/plugins/__init__.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/arista/plugins/mac.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/hp/__init__.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/hp/hp.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/hp/plugins/__init__.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/hp/plugins/mac.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/huawei/__init__.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/huawei/huawei.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/huawei/plugins/__init__.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/huawei/plugins/mac.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/ovswitch/__init__.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/ovswitch/ovswitch.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/ovswitch/plugins/__init__.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/ovswitch/plugins/mac.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/pica8/__init__.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/pica8/pica8.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/pica8/plugins/__init__.py create mode 100644 compass-tasks-base/hdsdiscovery/vendors/pica8/plugins/mac.py create mode 100644 compass-tasks-base/log_analyzor/__init__.py create mode 100644 compass-tasks-base/log_analyzor/adapter_matcher.py create mode 100644 compass-tasks-base/log_analyzor/environment.py create mode 100644 compass-tasks-base/log_analyzor/file_matcher.py create mode 100644 compass-tasks-base/log_analyzor/line_matcher.py create mode 100644 compass-tasks-base/log_analyzor/progress_calculator.py create mode 100644 compass-tasks-base/misc/Dockerfile create mode 100644 compass-tasks-base/misc/compass_install.repo create mode 100644 compass-tasks-base/requirements.txt create mode 100644 compass-tasks-base/setup.py create mode 100755 compass-tasks-base/start.sh create mode 100644 compass-tasks-base/supervisord.conf create mode 100644 compass-tasks-base/tasks/__init__.py create mode 100644 compass-tasks-base/tasks/client.py create mode 100644 compass-tasks-base/tasks/tasks.py create mode 100644 compass-tasks-base/utils/__init__.py create mode 100644 compass-tasks-base/utils/celeryconfig_wrapper.py create mode 100644 compass-tasks-base/utils/flags.py create mode 100644 compass-tasks-base/utils/logsetting.py create mode 100644 compass-tasks-base/utils/setting_wrapper.py create mode 100644 compass-tasks-base/utils/util.py delete mode 100644 compass-tasks-k8s/Dockerfile delete mode 100644 compass-tasks-k8s/README.md delete mode 100644 compass-tasks-k8s/run.sh delete mode 100644 compass-tasks-osa/Dockerfile delete mode 100644 compass-tasks-osa/run.sh delete mode 100644 compass-tasks-osa/tacker_conf/ansible-role-requirements.yml delete mode 100644 compass-tasks-osa/tacker_conf/haproxy_config.yml delete mode 100644 compass-tasks-osa/tacker_conf/openstack_services.yml delete mode 100644 compass-tasks-osa/tacker_conf/os-tacker-install.yml delete mode 100644 compass-tasks-osa/tacker_conf/setup-openstack.yml delete mode 100644 compass-tasks-osa/tacker_conf/tacker.yml delete mode 100644 compass-tasks-osa/tacker_conf/tacker_all.yml delete mode 100644 compass-tasks-osa/tacker_conf/user_secrets.yml delete mode 100644 compass-tasks/README.md delete mode 100644 compass-tasks/actions/__init__.py delete mode 100644 compass-tasks/actions/clean.py delete mode 100644 compass-tasks/actions/cli.py delete mode 100644 compass-tasks/actions/delete.py delete mode 100644 compass-tasks/actions/deploy.py delete mode 100644 compass-tasks/actions/health_check/__init__.py delete mode 100644 compass-tasks/actions/health_check/base.py delete mode 100644 compass-tasks/actions/health_check/check.py delete mode 100644 compass-tasks/actions/health_check/check_apache.py delete mode 100644 compass-tasks/actions/health_check/check_celery.py delete mode 100644 compass-tasks/actions/health_check/check_dhcp.py delete mode 100644 compass-tasks/actions/health_check/check_dns.py delete mode 100644 compass-tasks/actions/health_check/check_hds.py delete mode 100644 compass-tasks/actions/health_check/check_misc.py delete mode 100644 compass-tasks/actions/health_check/check_os_installer.py delete mode 100644 compass-tasks/actions/health_check/check_package_installer.py delete mode 100644 compass-tasks/actions/health_check/check_squid.py delete mode 100644 compass-tasks/actions/health_check/check_tftp.py delete mode 100644 compass-tasks/actions/health_check/utils.py delete mode 100644 compass-tasks/actions/install_callback.py delete mode 100644 compass-tasks/actions/patch.py delete mode 100644 compass-tasks/actions/poll_switch.py delete mode 100644 compass-tasks/actions/reinstall.py delete mode 100644 compass-tasks/actions/search.py delete mode 100644 compass-tasks/actions/update_progress.py delete mode 100644 compass-tasks/actions/util.py delete mode 100644 compass-tasks/apiclient/__init__.py delete mode 100755 compass-tasks/apiclient/example.py delete mode 100644 compass-tasks/apiclient/restful.py delete mode 100644 compass-tasks/apiclient/v1/__init__.py delete mode 100755 compass-tasks/apiclient/v1/example.py delete mode 100644 compass-tasks/apiclient/v1/restful.py delete mode 100755 compass-tasks/build.sh delete mode 100644 compass-tasks/db/__init__.py delete mode 100644 compass-tasks/db/api/__init__.py delete mode 100644 compass-tasks/db/api/adapter.py delete mode 100644 compass-tasks/db/api/adapter_holder.py delete mode 100644 compass-tasks/db/api/cluster.py delete mode 100644 compass-tasks/db/api/database.py delete mode 100644 compass-tasks/db/api/health_check_report.py delete mode 100644 compass-tasks/db/api/host.py delete mode 100644 compass-tasks/db/api/machine.py delete mode 100644 compass-tasks/db/api/metadata.py delete mode 100644 compass-tasks/db/api/metadata_holder.py delete mode 100644 compass-tasks/db/api/network.py delete mode 100644 compass-tasks/db/api/permission.py delete mode 100644 compass-tasks/db/api/switch.py delete mode 100644 compass-tasks/db/api/user.py delete mode 100644 compass-tasks/db/api/user_log.py delete mode 100644 compass-tasks/db/api/utils.py delete mode 100644 compass-tasks/db/callback.py delete mode 100644 compass-tasks/db/config_validation/__init__.py delete mode 100644 compass-tasks/db/config_validation/default_validator.py delete mode 100644 compass-tasks/db/config_validation/extension/__init__.py delete mode 100644 compass-tasks/db/config_validation/extension/openstack.py delete mode 100644 compass-tasks/db/exception.py delete mode 100644 compass-tasks/db/models.py delete mode 100644 compass-tasks/db/v1/model.py delete mode 100644 compass-tasks/db/validator.py delete mode 100644 compass-tasks/deployment/__init__.py delete mode 100644 compass-tasks/deployment/deploy_manager.py delete mode 100644 compass-tasks/deployment/installers/__init__.py delete mode 100644 compass-tasks/deployment/installers/config_manager.py delete mode 100644 compass-tasks/deployment/installers/installer.py delete mode 100644 compass-tasks/deployment/installers/os_installers/__init__.py delete mode 100644 compass-tasks/deployment/installers/os_installers/cobbler/__init__.py delete mode 100644 compass-tasks/deployment/installers/os_installers/cobbler/cobbler.py delete mode 100644 compass-tasks/deployment/installers/pk_installers/__init__.py delete mode 100644 compass-tasks/deployment/installers/pk_installers/ansible_installer/__init__.py delete mode 100644 compass-tasks/deployment/installers/pk_installers/ansible_installer/ansible_installer.py delete mode 100644 compass-tasks/deployment/utils/__init__.py delete mode 100644 compass-tasks/deployment/utils/constants.py delete mode 100644 compass-tasks/hdsdiscovery/SNMP_CONFIG.md delete mode 100644 compass-tasks/hdsdiscovery/__init__.py delete mode 100644 compass-tasks/hdsdiscovery/base.py delete mode 100644 compass-tasks/hdsdiscovery/error.py delete mode 100644 compass-tasks/hdsdiscovery/hdmanager.py delete mode 100644 compass-tasks/hdsdiscovery/utils.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/__init__.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/appliance/__init__.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/appliance/appliance.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/appliance/plugins/__init__.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/appliance/plugins/mac.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/arista/__init__.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/arista/arista.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/arista/plugins/__init__.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/arista/plugins/mac.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/hp/__init__.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/hp/hp.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/hp/plugins/__init__.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/hp/plugins/mac.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/huawei/__init__.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/huawei/huawei.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/huawei/plugins/__init__.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/huawei/plugins/mac.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/ovswitch/__init__.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/ovswitch/ovswitch.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/ovswitch/plugins/__init__.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/ovswitch/plugins/mac.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/pica8/__init__.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/pica8/pica8.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/pica8/plugins/__init__.py delete mode 100644 compass-tasks/hdsdiscovery/vendors/pica8/plugins/mac.py delete mode 100644 compass-tasks/log_analyzor/__init__.py delete mode 100644 compass-tasks/log_analyzor/adapter_matcher.py delete mode 100644 compass-tasks/log_analyzor/environment.py delete mode 100644 compass-tasks/log_analyzor/file_matcher.py delete mode 100644 compass-tasks/log_analyzor/line_matcher.py delete mode 100644 compass-tasks/log_analyzor/progress_calculator.py delete mode 100644 compass-tasks/misc/Dockerfile delete mode 100644 compass-tasks/misc/compass_install.repo delete mode 100644 compass-tasks/requirements.txt create mode 100644 compass-tasks/run.sh create mode 100644 compass-tasks/setup-complete.yml delete mode 100644 compass-tasks/setup.py delete mode 100755 compass-tasks/start.sh delete mode 100644 compass-tasks/supervisord.conf create mode 100644 compass-tasks/tacker_conf/ansible-role-requirements.yml create mode 100644 compass-tasks/tacker_conf/haproxy_config.yml create mode 100644 compass-tasks/tacker_conf/openstack_services.yml create mode 100644 compass-tasks/tacker_conf/os-tacker-install.yml create mode 100644 compass-tasks/tacker_conf/setup-openstack.yml create mode 100644 compass-tasks/tacker_conf/tacker.yml create mode 100644 compass-tasks/tacker_conf/tacker_all.yml create mode 100644 compass-tasks/tacker_conf/user_secrets.yml delete mode 100644 compass-tasks/tasks/__init__.py delete mode 100644 compass-tasks/tasks/client.py delete mode 100644 compass-tasks/tasks/tasks.py delete mode 100644 compass-tasks/utils/__init__.py delete mode 100644 compass-tasks/utils/celeryconfig_wrapper.py delete mode 100644 compass-tasks/utils/flags.py delete mode 100644 compass-tasks/utils/logsetting.py delete mode 100644 compass-tasks/utils/setting_wrapper.py delete mode 100644 compass-tasks/utils/util.py diff --git a/compass-tasks-base/Dockerfile b/compass-tasks-base/Dockerfile new file mode 100644 index 0000000..3beaf57 --- /dev/null +++ b/compass-tasks-base/Dockerfile @@ -0,0 +1,12 @@ +FROM centos:7 + +ADD . /root/compass-tasks + +RUN /root/compass-tasks/build.sh + +EXPOSE 6379 + +VOLUME ["/var/ansible", "/etc/compass/machine_list", "/etc/compass/switch_list"] + +ENTRYPOINT ["/bin/bash", "-c"] +CMD ["/usr/local/bin/start.sh"] diff --git a/compass-tasks-base/README.md b/compass-tasks-base/README.md new file mode 100644 index 0000000..952f4b5 --- /dev/null +++ b/compass-tasks-base/README.md @@ -0,0 +1 @@ +# compass-tasks \ No newline at end of file diff --git a/compass-tasks-base/actions/__init__.py b/compass-tasks-base/actions/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/actions/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/actions/clean.py b/compass-tasks-base/actions/clean.py new file mode 100644 index 0000000..a4e9bc9 --- /dev/null +++ b/compass-tasks-base/actions/clean.py @@ -0,0 +1,195 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to clean installers +""" +import logging +import xmlrpclib + +from compass.actions import util + +try: + import chef +except ImportError: + pass + +class CobblerInstaller(object): + """cobbler installer""" + CREDENTIALS = "credentials" + USERNAME = 'username' + PASSWORD = 'password' + + INSTALLER_URL = "cobbler_url" + + def __init__(self, settings): + username = settings[self.CREDENTIALS][self.USERNAME] + password = settings[self.CREDENTIALS][self.PASSWORD] + cobbler_url = settings[self.INSTALLER_URL] + try: + self.remote = xmlrpclib.Server(cobbler_url) + self.token = self.remote.login(username, password) + logging.info('cobbler %s client created', cobbler_url) + except Exception as error: + logging.error( + 'failed to login %s with (%s, %s)', + cobbler_url, username, password + ) + logging.exception(error) + + def clean(self): + systems = self.remote.get_systems() + for system in systems: + system_name = system['name'] + try: + self.remote.remove_system(system_name, self.token) + logging.info('system %s is removed', system_name) + except Exception as error: + logging.error( + 'failed to remove system %s', system_name + ) + logging.exception(error) + + +class AnsibleInstaller(object): + + def __init__(self, settings): + return + + def clean(self): + pass + + +class ChefInstaller(object): + DATABAGS = "databags" + CHEFSERVER_URL = "chef_url" + CHEFSERVER_DNS = "chef_server_dns" + CHEFSERVER_IP = "chef_server_ip" + KEY_DIR = "key_dir" + CLIENT = "client_name" + + def __init__(self, settings): + installer_url = settings.get(self.CHEFSERVER_URL, None) + key_dir = settings.get(self.KEY_DIR, None) + client = settings.get(self.CLIENT, None) + try: + if installer_url and key_dir and client: + self.api = chef.ChefAPI(installer_url, key_dir, client) + else: + self.api = chef.autoconfigure() + logging.info( + 'chef client created %s(%s, %s)', + installer_url, key_dir, client + ) + except Exception as error: + logging.error( + 'failed to create chef client %s(%s, %s)', + installer_url, key_dir, client + ) + logging.exception(error) + + def clean(self): + try: + for node_name in chef.Node.list(api=self.api): + node = chef.Node(node_name, api=self.api) + node.delete() + logging.info('delete node %s', node_name) + except Exception as error: + logging.error('failed to delete some nodes') + logging.exception(error) + + try: + for client_name in chef.Client.list(api=self.api): + if client_name in ['chef-webui', 'chef-validator']: + continue + client = chef.Client(client_name, api=self.api) + client.delete() + logging.info('delete client %s', client_name) + except Exception as error: + logging.error('failed to delete some clients') + logging.exception(error) + + try: + for env_name in chef.Environment.list(api=self.api): + if env_name == '_default': + continue + env = chef.Environment(env_name, api=self.api) + env.delete() + logging.info('delete env %s', env_name) + except Exception as error: + logging.error('failed to delete some envs') + logging.exception(error) + + try: + for databag_name in chef.DataBag.list(api=self.api): + databag = chef.DataBag(databag_name, api=self.api) + for item_name, item in databag.items(): + item.delete() + logging.info( + 'delete item %s from databag %s', + item_name, databag_name + ) + except Exception as error: + logging.error('failed to delete some databag items') + logging.exception(error) + + +OS_INSTALLERS = { + 'cobbler': CobblerInstaller +} +PK_INSTALLERS = { + 'chef_installer': ChefInstaller, + 'ansible_installer': AnsibleInstaller +} + + +def clean_os_installer( + os_installer_name, os_installer_settings +): + with util.lock('serialized_action', timeout=100) as lock: + if not lock: + raise Exception( + 'failed to acquire lock to clean os installer' + ) + + if os_installer_name not in OS_INSTALLERS: + logging.error( + '%s not found in os_installers', + os_installer_name + ) + + os_installer = OS_INSTALLERS[os_installer_name]( + os_installer_settings + ) + os_installer.clean() + + +def clean_package_installer( + package_installer_name, package_installer_settings +): + with util.lock('serialized_action', timeout=100) as lock: + if not lock: + raise Exception( + 'failed to acquire lock to clean package installer' + ) + + if package_installer_name not in PK_INSTALLERS: + logging.error( + '%s not found in os_installers', + package_installer_name + ) + + package_installer = PK_INSTALLERS[package_installer_name]( + package_installer_settings + ) + package_installer.clean() diff --git a/compass-tasks-base/actions/cli.py b/compass-tasks-base/actions/cli.py new file mode 100644 index 0000000..c9058ed --- /dev/null +++ b/compass-tasks-base/actions/cli.py @@ -0,0 +1,179 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compass Command Line Interface""" +import logging +import subprocess +import sys + +from compass.actions.health_check import check +from compass.db.api import database + +from compass.utils import flags +from compass.utils import logsetting +from compass.utils import setting_wrapper as setting +from compass.utils.util import pretty_print + + +ACTION_MAP = { + "check": "apache celery dhcp dns hds misc os_installer " + "package_installer squid tftp".split(" "), + "refresh": "db sync".split(" "), +} + + +class BootCLI(object): + """CLI to do compass check.""" + + def __init__(self): + return + + def run(self, args): + """cli takes the commands and calls respective modules.""" + action = self.get_action(args) + if action is None: + self.print_help() + else: + module = self.get_module(action, args) + if module == "invalid": + self.print_help(action) + else: + method = "self.run_" + action + "(module)" + eval(method) + + @classmethod + def get_action(cls, args): + """This method returns an action type. + + .. note:: + For 'compass check dhcp' command, it will return 'check'. + """ + if len(args) == 1: + return None + elif args[1] in ACTION_MAP.keys(): + return args[1] + return None + + @classmethod + def get_module(cls, action, args): + """This method returns a module. + + .. note:: + For 'compass check dhcp' command, it will return 'dhcp'. + """ + if len(args) <= 2: + return None + elif args[2] in ACTION_MAP[action]: + return args[2] + return "invalid" + + def run_check(self, module=None): + """This provides a flexible sanity check. + + .. note:: + param module default set to None. + if parameter module is none. Compass checks all modules. + If module specified, Compass will only check such module. + """ + if module is None: + pretty_print("Starting: Compass Health Check", + "==============================") + chk = check.BootCheck() + res = chk.run() + self.output_check_result(res) + + else: + pretty_print("Checking Module: %s" % module, + "============================") + chk = check.BootCheck() + method = "chk._check_" + module + "()" + res = eval(method) + print "\n".join(msg for msg in res[1]) + + @classmethod + def output_check_result(cls, result): + """output check result.""" + if result == {}: + return + pretty_print("\n", + "===============================", + "* Compass Health Check Report *", + "===============================") + successful = True + for key in result.keys(): + if result[key][0] == 0: + successful = False + print "%s" % "\n".join(item for item in result[key][1]) + + print "====================" + if successful is True: + print "Compass Check completes. No problems found, all systems go" + sys.exit(0) + else: + print ( + "Compass has ERRORS shown above. Please fix them before " + "deploying!") + sys.exit(1) + + @classmethod + def run_refresh(cls, action=None): + """Run refresh.""" + # TODO(xicheng): replace refresh.sh with refresh.py + if action is None: + pretty_print("Refreshing Compass...", + "=================") + subprocess.Popen( + ['/opt/compass/bin/refresh.sh'], shell=True) + elif action == "db": + pretty_print("Refreshing Compass Database...", + "===================") + subprocess.Popen( + ['/opt/compass/bin/manage_db.py createdb'], shell=True) + else: + pretty_print("Syncing with Installers...", + "================") + subprocess.Popen( + ['/opt/compass/bin/manage_db.py sync_from_installers'], + shell=True + ) + + @classmethod + def print_help(cls, module_help=""): + """print help.""" + if module_help == "": + pretty_print("usage\n=====", + "compass ", + "type 'compass {action} --help' for detailed " + "command list") + + elif module_help == "refresh": + pretty_print("usage\n=====", + "compass refresh [%s]" % + "|".join(action for action in ACTION_MAP['refresh'])) + + else: + pretty_print("usage\n=====", + "compass check [%s]" % + "|".join(action for action in ACTION_MAP['check'])) + sys.exit(2) + + +def main(): + """Compass cli entry point.""" + flags.init() + logsetting.init() + database.init() + cli = BootCLI() + output = cli.run(sys.argv) + return sys.exit(output) diff --git a/compass-tasks-base/actions/delete.py b/compass-tasks-base/actions/delete.py new file mode 100644 index 0000000..d89994d --- /dev/null +++ b/compass-tasks-base/actions/delete.py @@ -0,0 +1,148 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to delete a given cluster +""" +import logging + +from compass.actions import util +from compass.db.api import cluster as cluster_api +from compass.db.api import host as host_api +from compass.db.api import user as user_db +from compass.deployment.deploy_manager import DeployManager +from compass.deployment.utils import constants as const + + +def delete_cluster( + cluster_id, host_id_list, + username=None, delete_underlying_host=False +): + """Delete cluster and all clusterhosts on it. + + :param cluster_id: id of the cluster. + :type cluster_id: int + :param host_id_list: list of host id. + :type host_id_list: list of int. + + If delete_underlying_host is set, all underlying hosts will + be deleted. + + .. note:: + The function should be called out of database session. + """ + with util.lock('serialized_action', timeout=100) as lock: + if not lock: + raise Exception('failed to acquire lock to delete cluster') + + user = user_db.get_user_object(username) + + cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) + adapter_id = cluster_info[const.ADAPTER_ID] + + adapter_info = util.ActionHelper.get_adapter_info( + adapter_id, cluster_id, user) + hosts_info = util.ActionHelper.get_hosts_info( + cluster_id, host_id_list, user) + + deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) + + deploy_manager.remove_hosts( + package_only=not delete_underlying_host, + delete_cluster=True + ) + util.ActionHelper.delete_cluster( + cluster_id, host_id_list, user, + delete_underlying_host + ) + + +def delete_cluster_host( + cluster_id, host_id, + username=None, delete_underlying_host=False +): + """Delete clusterhost. + + :param cluster_id: id of the cluster. + :type cluster_id: int + :param host_id: id of the host. + :type host_id: int + + If delete_underlying_host is set, the underlying host + will be deleted too. + + .. note:: + The function should be called out of database session. + """ + with util.lock('serialized_action', timeout=100) as lock: + if not lock: + raise Exception('failed to acquire lock to delete clusterhost') + + user = user_db.get_user_object(username) + cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) + adapter_id = cluster_info[const.ADAPTER_ID] + + adapter_info = util.ActionHelper.get_adapter_info( + adapter_id, cluster_id, user) + hosts_info = util.ActionHelper.get_hosts_info( + cluster_id, [host_id], user) + + deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) + + deploy_manager.remove_hosts( + package_only=not delete_underlying_host, + delete_cluster=False + ) + util.ActionHelper.delete_cluster_host( + cluster_id, host_id, user, + delete_underlying_host + ) + + +def delete_host( + host_id, cluster_id_list, username=None +): + """Delete host and all clusterhosts on it. + + :param host_id: id of the host. + :type host_id: int + + .. note:: + The function should be called out of database session. + """ + with util.lock('serialized_action', timeout=100) as lock: + if not lock: + raise Exception('failed to acquire lock to delete host') + + user = user_db.get_user_object(username) + for cluster_id in cluster_id_list: + cluster_info = util.ActionHelper.get_cluster_info( + cluster_id, user) + adapter_id = cluster_info[const.ADAPTER_ID] + + adapter_info = util.ActionHelper.get_adapter_info( + adapter_id, cluster_id, user) + hosts_info = util.ActionHelper.get_hosts_info( + cluster_id, [host_id], user) + + deploy_manager = DeployManager( + adapter_info, cluster_info, hosts_info) + + deploy_manager.remove_hosts( + package_only=True, + delete_cluster=False + ) + + util.ActionHelper.delete_host( + host_id, user + ) diff --git a/compass-tasks-base/actions/deploy.py b/compass-tasks-base/actions/deploy.py new file mode 100644 index 0000000..53179f5 --- /dev/null +++ b/compass-tasks-base/actions/deploy.py @@ -0,0 +1,182 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to deploy a given cluster +""" +import logging + +from compass.actions import util +from compass.db.api import cluster as cluster_db +from compass.db.api import health_check_report as health_check_db +from compass.db.api import user as user_db +from compass.deployment.deploy_manager import DeployManager +from compass.deployment.utils import constants as const + + +def deploy(cluster_id, hosts_id_list, username=None): + """Deploy clusters. + + :param cluster_hosts: clusters and hosts in each cluster to deploy. + :type cluster_hosts: dict of int or str to list of int or str + + .. note:: + The function should be called out of database session. + """ + with util.lock('serialized_action', timeout=1000) as lock: + if not lock: + raise Exception('failed to acquire lock to deploy') + + user = user_db.get_user_object(username) + + cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) + adapter_id = cluster_info[const.ADAPTER_ID] + + adapter_info = util.ActionHelper.get_adapter_info( + adapter_id, cluster_id, user) + hosts_info = util.ActionHelper.get_hosts_info( + cluster_id, hosts_id_list, user) + + deploy_successful = True + try: + deploy_manager = DeployManager( + adapter_info, cluster_info, hosts_info) + # deploy_manager.prepare_for_deploy() + logging.debug('Created deploy manager with %s %s %s' + % (adapter_info, cluster_info, hosts_info)) + deployed_config = deploy_manager.deploy() + except Exception as error: + logging.exception(error) + deploy_successful = False + + if deploy_successful: + util.ActionHelper.save_deployed_config(deployed_config, user) + util.ActionHelper.update_state( + cluster_id, hosts_id_list, user, state='INSTALLING' + ) + else: + util.ActionHelper.update_state( + cluster_id, hosts_id_list, user, state='ERROR', + message='failed to start deployment', severity='ERROR' + ) + + +def redeploy(cluster_id, username=None): + """Deploy clusters. + + :param cluster_hosts: clusters and hosts in each cluster to deploy. + :type cluster_hosts: dict of int or str to list of int or str + """ + with util.lock('serialized_action') as lock: + if not lock: + raise Exception('failed to acquire lock to deploy') + + user = user_db.get_user_object(username) + cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) + adapter_id = cluster_info[const.ADAPTER_ID] + + adapter_info = util.ActionHelper.get_adapter_info( + adapter_id, cluster_id, user) + + cluster_hosts = cluster_db.list_cluster_hosts(cluster_id, user) + hosts_id_list = [host['id'] for host in cluster_hosts] + + hosts_info = util.ActionHelper.get_hosts_info( + cluster_id, hosts_id_list, user) + + deploy_successful = True + try: + deploy_manager = DeployManager( + adapter_info, cluster_info, hosts_info) + # deploy_manager.prepare_for_deploy() + deploy_manager.redeploy() + except Exception as error: + logging.exception(error) + deploy_successful = False + if deploy_successful: + util.ActionHelper.update_state( + cluster_id, hosts_id_list, user, state='INSTALLING', + ) + else: + util.ActionHelper.update_state( + cluster_id, hosts_id_list, user, state='ERROR', + message='failed to start redeployment', severity='ERROR' + ) + + +def health_check(cluster_id, report_uri, username): + with util.lock('cluster_health_check') as lock: + if not lock: + raise Exception('failed to acquire lock to check health') + + user = user_db.get_user_object(username) + cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) + adapter_id = cluster_info[const.ADAPTER_ID] + + adapter_info = util.ActionHelper.get_adapter_info( + adapter_id, cluster_id, user + ) + + deploy_manager = DeployManager(adapter_info, cluster_info, None) + try: + deploy_manager.check_cluster_health(report_uri) + except Exception as exc: + logging.error("health_check exception: ============= %s" % exc) + data = {'state': 'error', 'error_message': str(exc), 'report': {}} + reports = health_check_db.list_health_reports( + cluster_id, user=user) + if not reports: + # Exception before executing command remotely for health check. + # No reports names sending back yet. Create a report + name = 'pre_remote_health_check' + health_check_db.add_report_record( + cluster_id, name, user=user, **data + ) + + health_check_db.update_multi_reports(cluster_id, user=user, **data) + + +class ServerPowerMgmt(object): + """Power management for bare-metal machines by IPMI command.""" + @staticmethod + def poweron(machine_id, user): + """Power on the specified machine.""" + pass + + @staticmethod + def poweroff(machine_id, user): + pass + + @staticmethod + def reset(machine_id, user): + pass + + +class HostPowerMgmt(object): + """Power management for hosts installed OS by OS installer. OS installer + + will poweron/poweroff/reset host. + + """ + @staticmethod + def poweron(host_id, user): + """Power on the specified host.""" + pass + + @staticmethod + def poweroff(host_id, user): + pass + + @staticmethod + def reset(host_id, user): + pass diff --git a/compass-tasks-base/actions/health_check/__init__.py b/compass-tasks-base/actions/health_check/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/actions/health_check/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/actions/health_check/base.py b/compass-tasks-base/actions/health_check/base.py new file mode 100644 index 0000000..22b6fae --- /dev/null +++ b/compass-tasks-base/actions/health_check/base.py @@ -0,0 +1,57 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Base class for Compass Health Check.""" +from compass.actions.health_check import utils as health_check_utils +from compass.db.api import adapter as adapter_api +from compass.utils import setting_wrapper as setting + + +class BaseCheck(object): + """health check base class.""" + + def __init__(self): + self.config = setting + self.code = 1 + self.messages = [] + self.dist, self.version, self.release = health_check_utils.get_dist() + adapter_api.load_adapters_internal() + self.os_installer = self._get_os_installer() + self.package_installer = self._get_package_installer() + + def _get_os_installer(self): + installer = adapter_api.OS_INSTALLERS.values()[0] + os_installer = {} + os_installer['name'] = health_check_utils.strip_name( + installer['name']) + os_installer.update(installer['settings']) + return os_installer + + def _get_package_installer(self): + package_installer = {} + installer = adapter_api.PACKAGE_INSTALLERS.values()[0] + package_installer = {} + package_installer['name'] = health_check_utils.strip_name( + installer['name']) + package_installer.update(installer['settings']) + return package_installer + + def _set_status(self, code, message): + """set status.""" + self.code = code + self.messages.append(message) + + def get_status(self): + """get status.""" + return (self.code, self.messages) diff --git a/compass-tasks-base/actions/health_check/check.py b/compass-tasks-base/actions/health_check/check.py new file mode 100644 index 0000000..c1adbc6 --- /dev/null +++ b/compass-tasks-base/actions/health_check/check.py @@ -0,0 +1,96 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Main Entry Point of Compass Health Check.""" +from compass.actions.health_check import base +from compass.actions.health_check import check_apache +from compass.actions.health_check import check_celery +from compass.actions.health_check import check_dhcp +from compass.actions.health_check import check_dns +from compass.actions.health_check import check_hds +from compass.actions.health_check import check_misc +from compass.actions.health_check import check_os_installer +from compass.actions.health_check import check_package_installer +from compass.actions.health_check import check_squid +from compass.actions.health_check import check_tftp + + +class BootCheck(base.BaseCheck): + """health check for all components.""" + + def run(self): + """do health check.""" + status = {} + status['apache'] = self._check_apache() + status['celery'] = self._check_celery() + status['dhcp'] = self._check_dhcp() + status['dns'] = self._check_dns() + status['hds'] = self._check_hds() + status['os_installer'] = self._check_os_installer() + status['package_installer'] = self._check_package_installer() + status['squid'] = self._check_squid() + status['tftp'] = self._check_tftp() + status['other'] = self._check_misc() + + return status + + def _check_apache(self): + """do apache health check.""" + checker = check_apache.ApacheCheck() + return checker.run() + + def _check_celery(self): + """do celery health check.""" + checker = check_celery.CeleryCheck() + return checker.run() + + def _check_dhcp(self): + """do dhcp health check.""" + checker = check_dhcp.DhcpCheck() + return checker.run() + + def _check_dns(self): + """do dns health check.""" + checker = check_dns.DnsCheck() + return checker.run() + + def _check_hds(self): + """do hds health check.""" + checker = check_hds.HdsCheck() + return checker.run() + + def _check_os_installer(self): + """do os installer health check.""" + checker = check_os_installer.OsInstallerCheck() + return checker.run() + + def _check_package_installer(self): + """do package installer health check.""" + checker = check_package_installer.PackageInstallerCheck() + return checker.run() + + def _check_squid(self): + """do squid health check.""" + checker = check_squid.SquidCheck() + return checker.run() + + def _check_tftp(self): + """do tftp health check.""" + checker = check_tftp.TftpCheck() + return checker.run() + + def _check_misc(self): + """do misc health check.""" + checker = check_misc.MiscCheck() + return checker.run() diff --git a/compass-tasks-base/actions/health_check/check_apache.py b/compass-tasks-base/actions/health_check/check_apache.py new file mode 100644 index 0000000..294d6f9 --- /dev/null +++ b/compass-tasks-base/actions/health_check/check_apache.py @@ -0,0 +1,89 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Health Check module for Apache service.""" + +import socket +import urllib2 + +from compass.actions.health_check import base +from compass.actions.health_check import utils as health_check_utils + + +class ApacheCheck(base.BaseCheck): + """apache server health check class.""" + NAME = "Apache Check" + + def run(self): + """do the healthcheck.""" + if self.dist in ("centos", "redhat", "fedora", "scientific linux"): + apache_service = 'httpd' + else: + apache_service = 'apache2' + self.check_apache_conf(apache_service) + print "[Done]" + self.check_apache_running(apache_service) + print "[Done]" + if self.code == 1: + self.messages.append( + "[%s]Info: Apache health check has completed. " + "No problems found, all systems go." % self.NAME) + return (self.code, self.messages) + + def check_apache_conf(self, apache_service): + """Validates if Apache settings. + + :param apache_service : service type of apache, os dependent. + e.g. httpd or apache2 + :type apache_service : string + + """ + print "Checking Apache Config......", + conf_err_msg = health_check_utils.check_path( + self.NAME, + "/etc/%s/conf.d/ods-server.conf" % apache_service) + if not conf_err_msg == "": + self._set_status(0, conf_err_msg) + + wsgi_err_msg = health_check_utils.check_path( + self.NAME, + '/var/www/compass/compass.wsgi') + if not wsgi_err_msg == "": + self._set_status(0, wsgi_err_msg) + + return True + + def check_apache_running(self, apache_service): + """Checks if Apache service is running on port 80.""" + + print "Checking Apache service......", + serv_err_msg = health_check_utils.check_service_running(self.NAME, + apache_service) + if not serv_err_msg == "": + self._set_status(0, serv_err_msg) + if 'http' != socket.getservbyport(80): + self._set_status( + 0, + "[%s]Error: Apache is not listening on port 80." + % self.NAME) + try: + html = urllib2.urlopen('http://localhost') + html.geturl() + except Exception: + self._set_status( + 0, + "[%s]Error: Apache is not listening on port 80." + % self.NAME) + + return True diff --git a/compass-tasks-base/actions/health_check/check_celery.py b/compass-tasks-base/actions/health_check/check_celery.py new file mode 100644 index 0000000..2d8d27c --- /dev/null +++ b/compass-tasks-base/actions/health_check/check_celery.py @@ -0,0 +1,115 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Health Check module for Celery.""" +import commands +import os + +from celery.task.control import inspect + +from compass.actions.health_check import base +from compass.actions.health_check import utils as health_check_utils + + +class CeleryCheck(base.BaseCheck): + """celery health check class.""" + NAME = "Celery Check." + + def run(self): + """do health check.""" + self.check_compass_celery_setting() + print "[Done]" + self.check_celery_backend() + print "[Done]" + if self.code == 1: + self.messages.append("[%s]Info: Celery health check " + "has completed. No problems found, " + "all systems go." % self.NAME) + return (self.code, self.messages) + + def check_compass_celery_setting(self): + """Validates Celery settings.""" + + print "Checking Celery setting......", + setting_map = { + 'logfile': 'CELERY_LOGFILE', + 'configdir': 'CELERYCONFIG_DIR', + 'configfile': 'CELERYCONFIG_FILE', + } + unset = [] + res = health_check_utils.validate_setting('Celery', + self.config, + 'CELERY_LOGFILE') + if res is False: + unset.append(setting_map["logfile"]) + self._set_status(0, res) + + res = health_check_utils.validate_setting('Celery', + self.config, + 'CELERYCONFIG_DIR') + if res is False: + unset.append(setting_map["configdir"]) + self._set_status(0, res) + + res = health_check_utils.validate_setting('Celery', + self.config, + 'CELERYCONFIG_FILE') + if res is False: + unset.append(setting_map["configdir"]) + self._set_status(0, res) + + if len(unset) != 0: + self._set_status(0, + "[%s]Error: Unset celery settings: %s" + " in /etc/compass/setting" + % (self.NAME, ', '.join(item for item in unset))) + return True + + def check_celery_backend(self): + """Checks if Celery backend is running and configured properly.""" + + print "Checking Celery Backend......", + if 'celery worker' not in commands.getoutput('ps -ef'): + self._set_status(0, "[%s]Error: celery is not running" % self.NAME) + return True + + if not os.path.exists('/etc/compass/celeryconfig'): + self._set_status( + 0, + "[%s]Error: No celery config file found for Compass" + % self.NAME) + return True + + try: + insp = inspect() + celery_stats = inspect.stats(insp) + if not celery_stats: + self._set_status( + 0, + "[%s]Error: No running Celery workers were found." + % self.NAME) + except IOError as error: + self._set_status( + 0, + "[%s]Error: Failed to connect to the backend: %s" + % (self.NAME, str(error))) + from errno import errorcode + if ( + len(error.args) > 0 and + errorcode.get(error.args[0]) == 'ECONNREFUSED' + ): + self.messages.append( + "[%s]Error: RabbitMQ server isn't running" + % self.NAME) + return True diff --git a/compass-tasks-base/actions/health_check/check_dhcp.py b/compass-tasks-base/actions/health_check/check_dhcp.py new file mode 100644 index 0000000..e3bae1e --- /dev/null +++ b/compass-tasks-base/actions/health_check/check_dhcp.py @@ -0,0 +1,184 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Health Check module for DHCP service.""" +import commands +import os +import re +import socket +import xmlrpclib + +from compass.actions.health_check import base + + +class DhcpCheck(base.BaseCheck): + """dhcp health check class.""" + + NAME = "DHCP Check" + + def run(self): + """do health check.""" + method_name = "self.check_" + self.os_installer['name'] + "_dhcp()" + return eval(method_name) + + def check_cobbler_dhcp(self): + """Checks if Cobbler has taken over DHCP service.""" + + try: + remote = xmlrpclib.Server( + self.os_installer['cobbler_url'], + allow_none=True) + credentials = self.os_installer['credentials'] + remote.login( + credentials['username'], credentials['password']) + except Exception: + self._set_status( + 0, + "[%s]Error: Cannot login to Cobbler with " + "the tokens provided in the config file" % self.NAME) + return (self.code, self.messages) + + cobbler_settings = remote.get_settings() + if cobbler_settings['manage_dhcp'] == 0: + self.messages.append( + "[%s]Info: DHCP service is " + "not managed by Compass" % self.NAME) + self.code = 0 + return (self.code, self.messages) + + self.check_cobbler_dhcp_template() + print "[Done]" + self.check_dhcp_service() + self.check_dhcp_netmask() + print "[Done]" + if self.code == 1: + self.messages.append( + "[%s]Info: DHCP health check has completed. " + "No problems found, all systems go." % self.NAME) + + return (self.code, self.messages) + + def check_cobbler_dhcp_template(self): + """Validates Cobbler's DHCP template file.""" + print "Checking DHCP template......", + if os.path.exists("/etc/cobbler/dhcp.template"): + var_map = { + "match_next_server": False, + "match_subnet": False, + "match_filename": False, + "match_range": False, + } + + ip_regex = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') + + dhcp_template = open("/etc/cobbler/dhcp.template") + for line in dhcp_template.readlines(): + if line.find("next_server") != -1: + elmlist = line.split(" ") + for elm in elmlist: + if ";" in elm: + elm = elm[:-2] + + if "$next_server" in elm or ip_regex.match(elm): + var_map["match_next_server"] = True + + elif line.find("subnet") != -1 and line.find("{") != -1: + elmlist = line.split(" ") + for elm in elmlist: + if ip_regex.match(elm): + if elm[-1] == "0" and "255" not in elm: + var_map["match_subnet"] = True + elif elm[-1] != "0": + self.messages.append( + "[%s]Error: Subnet should be set " + "in the form of 192.168.0.0 in" + "/etc/cobbler/dhcp.template" % self.NAME) + + elif line.find("filename") != -1: + var_map["match_filename"] = True + elif line.find("range dynamic-bootp") != -1: + elmlist = line.split(" ") + ip_count = 0 + for elm in elmlist: + if ";" in elm and "\n" in elm: + elm = elm[:-2] + + if ip_regex.match(elm): + ip_count += 1 + + if ip_count != 2: + self.messages.append( + "[%s]Error: DHCP range should be set " + "between two IP addresses in " + "/etc/cobbler/dhcp.template" % self.NAME) + else: + var_map["match_range"] = True + + dhcp_template.close() + fails = [] + for var in var_map.keys(): + if var_map[var] is False: + fails.append(var) + + if len(fails) != 0: + self._set_status( + 0, + "[%s]Info: DHCP template file " + "failed components: %s" % ( + self.NAME, ' '.join(failed for failed in fails))) + + else: + self._set_status( + 0, + "[%s]Error: DHCP template file doesn't exist, " + "health check failed." % self.NAME) + + return True + + def check_dhcp_netmask(self): + with open('/etc/dhcp/dhcpd.conf') as conf_reader: + lines = conf_reader.readlines() + for line in lines: + if re.search('^subnet', line): + elm_list = line.split(' ') + break + subnet_ip = elm_list[1] + netmask = elm_list[-2] + subnet_ip_elm = subnet_ip.split('.') + netmask_elm = netmask.split('.') + for index, digit in enumerate(subnet_ip_elm): + if int(digit) & int(netmask_elm[index]) != int(digit): + self._set_status( + 0, + "[%s]Info: DHCP subnet IP and " + "netmask do not match" % self.NAME) + break + return True + + def check_dhcp_service(self): + """Checks if DHCP is running on port 67.""" + print "Checking DHCP service......", + if not commands.getoutput('pgrep dhcp'): + self._set_status( + 0, + "[%s]Error: dhcp service does not " + "seem to be running" % self.NAME) + + if socket.getservbyport(67) != 'bootps': + self._set_status( + 0, + "[%s]Error: bootps is not listening " + "on port 67" % self.NAME) + + return True diff --git a/compass-tasks-base/actions/health_check/check_dns.py b/compass-tasks-base/actions/health_check/check_dns.py new file mode 100644 index 0000000..843d7e2 --- /dev/null +++ b/compass-tasks-base/actions/health_check/check_dns.py @@ -0,0 +1,139 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Health Check module for DNS service.""" + +import commands +import os +import socket +import xmlrpclib + +from compass.actions.health_check import base + + +class DnsCheck(base.BaseCheck): + """dns health check class.""" + NAME = "DNS Check" + + def run(self): + """do health check.""" + method_name = "self.check_" + self.os_installer['name'] + "_dns()" + return eval(method_name) + + def check_cobbler_dns(self): + """Checks if Cobbler has taken over DNS service.""" + try: + remote = xmlrpclib.Server( + self.os_installer['cobbler_url'], + allow_none=True) + credentials = self.os_installer['credentials'] + remote.login( + credentials['username'], credentials['password']) + except Exception: + self._set_status(0, + "[%s]Error: Cannot login to Cobbler " + "with the tokens provided in the config file" + % self.NAME) + return (self.code, self.messages) + + cobbler_settings = remote.get_settings() + if cobbler_settings['manage_dns'] == 0: + self.messages.append('[DNS]Info: DNS is not managed by Compass') + return (0, self.messages) + self.check_cobbler_dns_template() + print "[Done]" + self.check_dns_service() + print "[Done]" + if self.code == 1: + self.messages.append( + "[%s]Info: DNS health check has complated. " + "No problems found, all systems go." % self.NAME) + return (self.code, self.messages) + + def check_cobbler_dns_template(self): + """Validates Cobbler's DNS template file.""" + + print "Checking DNS template......", + if os.path.exists("/etc/cobbler/named.template"): + var_map = { + "match_port": False, + "match_allow_query": False, + } + named_template = open("/etc/cobbler/named.template") + host_ip = socket.gethostbyname(socket.gethostname()) + missing_query = [] + for line in named_template.readlines(): + if "listen-on port 53" in line and host_ip in line: + var_map["match_port"] = True + + if "allow-query" in line: + for subnet in ["127.0.0.0/8"]: + if subnet not in line: + missing_query.append(subnet) + + named_template.close() + + if var_map["match_port"] is False: + self.messages.append( + "[%s]Error: named service port " + "and/or IP is misconfigured in " + "/etc/cobbler/named.template" % self.NAME) + + if len(missing_query) != 0: + self.messages.append( + "[%s]Error: Missing allow_query values in " + "/etc/cobbler/named.template:%s" % ( + self.NAME, + ', '.join(subnet for subnet in missing_query))) + else: + var_map["match_allow_query"] = True + + fails = [] + for var in var_map.keys(): + if var_map[var] is False: + fails.append(var) + + if len(fails) != 0: + self._set_status( + 0, + "[%s]Info: DNS template failed components: " + "%s" % ( + self.NAME, + ' '.join(failed for failed in fails))) + + else: + self._set_status( + 0, + "[%s]Error: named template file doesn't exist, " + "health check failed." % self.NAME) + + return True + + def check_dns_service(self): + """Checks if DNS is running on port 53.""" + + print "Checking DNS service......", + if 'named' not in commands.getoutput('ps -ef'): + self._set_status( + 0, + "[%s]Error: named service does not seem to be " + "running" % self.NAME) + + if socket.getservbyport(53) != 'domain': + self._set_status( + 0, + "[%s]Error: domain service is not listening on port " + "53" % self.NAME) + + return None diff --git a/compass-tasks-base/actions/health_check/check_hds.py b/compass-tasks-base/actions/health_check/check_hds.py new file mode 100644 index 0000000..d176f1f --- /dev/null +++ b/compass-tasks-base/actions/health_check/check_hds.py @@ -0,0 +1,97 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Health Check module for Hardware Discovery.""" +import logging + +from compass.actions.health_check import base +from compass.actions.health_check import utils as health_check_utils + + +class HdsCheck(base.BaseCheck): + """hds health check class.""" + NAME = "HDS Check" + + def run(self): + """do health check.""" + if self.dist in ("centos", "redhat", "fedora", "scientific linux"): + pkg_type = "yum" + else: + pkg_type = "apt" + + try: + pkg_module = __import__(pkg_type) + except Exception: + self._set_status( + 0, "[%s]Error: No module named %s please install it first." + % (self.NAME, pkg_type) + ) + return (self.code, self.messages) + + logging.info('import %s: %s', pkg_type, pkg_module) + method_name = 'self.check_' + pkg_type + '_snmp(pkg_module)' + eval(method_name) + print "[Done]" + self.check_snmp_mibs() + print "[Done]" + if self.code == 1: + self.messages.append("[%s]Info: hds health check has complated. " + "No problems found, all systems go." + % self.NAME) + + return (self.code, self.messages) + + def check_yum_snmp(self, pkg_module): + """Check if SNMP yum dependencies are installed + + :param pkg_module : python yum library + :type pkg_module : python module + + """ + print "Checking SNMP Packages......", + yum_base = pkg_module.YumBase() + uninstalled = [] + for package in ['net-snmp-utils', 'net-snmp', 'net-snmp-python']: + if len(yum_base.rpmdb.searchNevra(name=package)) == 0: + self.messages.append("[%s]Error: %s package is required " + "for HDS" % (self.NAME, package)) + uninstalled.append(package) + + if len(uninstalled) != 0: + self._set_status(0, "[%s]Info: Uninstalled packages: %s" + % (self.NAME, + ', '.join(item for item in uninstalled))) + + return True + + def check_apt_snmp(self, pkg_module): + """do apt health check.""" + return None + + def check_snmp_mibs(self): + """Checks if SNMP MIB files are properly placed.""" + + print "Checking SNMP MIBs......", + conf_err_msg = health_check_utils.check_path(self.NAME, + '/etc/snmp/snmp.conf') + if not conf_err_msg == "": + self._set_status(0, conf_err_msg) + + mibs_err_msg = health_check_utils.check_path( + self.NAME, + '/usr/local/share/snmp/mibs') + if not mibs_err_msg == "": + self._set_status(0, mibs_err_msg) + + return True diff --git a/compass-tasks-base/actions/health_check/check_misc.py b/compass-tasks-base/actions/health_check/check_misc.py new file mode 100644 index 0000000..b8beb1b --- /dev/null +++ b/compass-tasks-base/actions/health_check/check_misc.py @@ -0,0 +1,219 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Miscellaneous Health Check for Compass.""" +import logging + +from compass.actions.health_check import base +from compass.actions.health_check import utils as health_check_utils + + +class MiscCheck(base.BaseCheck): + """health check for misc.""" + NAME = "Miscellaneous Check" + + MISC_MAPPING = { + "yum": "rsyslog ntp iproute openssh-clients python git wget " + "python-setuptools " + "amqp mod_wsgi httpd squid " + "dhcp bind rsync yum-utils xinetd tftp-server gcc " + "net-snmp-utils net-snmp".split(" "), + "pip": "netaddr flask flask_script flask_restful amqplib " + "flask_sqlalchemy paramiko mock celery six discover daemon " + "unittest2 chef".split(" "), + "disable": "iptables ip6tables".split(" "), + "enable": "httpd squid xinetd dhcpd named sshd rsyslog cobblerd " + "ntpd compass-celeryd compass-progress-updated".split(" "), + } + + def run(self): + """do health check.""" + self.check_linux_dependencies() + print "[Done]" + self.check_pip_dependencies() + print "[Done]" + self.check_ntp() + print "[Done]" + self.check_rsyslogd() + print "[Done]" + self.check_chkconfig() + print "[Done]" + self.check_selinux() + print "[Done]" + + if self.code == 1: + self.messages.append( + "[%s]Info: Miscellaneous check has completed " + "No problems found, all systems go." % self.NAME) + return (self.code, self.messages) + + def check_linux_dependencies(self): + """Checks if dependencies are installed.""" + print "Checking Linux dependencies....", + if self.dist in ("centos", "redhat", "fedora", "scientific linux"): + pkg_type = "yum" + else: + pkg_type = "apt" + + try: + pkg_module = __import__(pkg_type) + except Exception: + self._set_status( + 0, + "[%s]Error: No module named %s, " + "please install it first." % (self.NAME, pkg_type)) + return True + + logging.info('import %s: %s', pkg_type, pkg_module) + method_name = 'self.check_' + pkg_type + '_dependencies(pkg_module)' + eval(method_name) + + def check_yum_dependencies(self, pkg_module): + """Checks if yum dependencies are installed. + + :param pkg_module : python yum library + :type pkg_module : python module + + """ + print "Checking Yum dependencies......", + yum_base = pkg_module.YumBase() + uninstalled = [] + for package in self.MISC_MAPPING["yum"]: + if len(yum_base.rpmdb.searchNevra(name=package)) == 0: + self._set_status( + 0, + "[%s]Error: %s package is required" + % (self.NAME, package)) + uninstalled.append(package) + + if len(uninstalled) != 0: + self._set_status( + 0, + "[%s]Info: Uninstalled yum packages: %s" + % (self.NAME, ', '.join(item for item in uninstalled))) + + return True + + def check_pip_dependencies(self): + """Checks if required pip packages are installed.""" + print "Checking pip dependencies......", + uninstalled = [] + for module in self.MISC_MAPPING['pip']: + try: + __import__(module) + except Exception: + self._set_status( + 0, + "[%s]Error: pip package %s is requred" + % (self.NAME, module)) + uninstalled.append(module) + + if len(uninstalled) != 0: + self._set_status( + 0, + "[%s]Info: Uninstalled pip packages: %s" + % (self.NAME, ', '.join(item for item in uninstalled))) + + return True + + def check_ntp(self): + """Validates ntp configuration and service.""" + + print "Checking NTP......", + conf_err_msg = health_check_utils.check_path(self.NAME, + '/etc/ntp.conf') + if not conf_err_msg == "": + self._set_status(0, conf_err_msg) + + serv_err_msg = health_check_utils.check_service_running(self.NAME, + 'ntpd') + if not serv_err_msg == "": + self._set_status(0, serv_err_msg) + + return True + + def check_rsyslogd(self): + """Validates rsyslogd configuration and service.""" + + print "Checking rsyslog......", + conf_err_msg = health_check_utils.check_path(self.NAME, + '/etc/rsyslog.conf') + if not conf_err_msg == "": + self._set_status(0, conf_err_msg) + + dir_err_msg = health_check_utils.check_path(self.NAME, + '/etc/rsyslog.d/') + if not dir_err_msg == "": + self._set_status(0, dir_err_msg) + + serv_err_msg = health_check_utils.check_service_running(self.NAME, + 'rsyslogd') + if not serv_err_msg == "": + self._set_status(0, serv_err_msg) + + return True + + def check_chkconfig(self): + """Check if required services are enabled on the start up.""" + + print "Checking chkconfig......", + serv_to_disable = [] + for serv in self.MISC_MAPPING["disable"]: + if health_check_utils.check_chkconfig(serv) is True: + self._set_status( + 0, + "[%s]Error: %s is not disabled" + % (self.NAME, serv)) + serv_to_disable.append(serv) + + if len(serv_to_disable) != 0: + self._set_status( + 0, + "[%s]Info: You need to disable these services " + "on system start-up: %s" + % (self.NAME, + ", ".join(item for item in serv_to_disable))) + + serv_to_enable = [] + for serv in self.MISC_MAPPING["enable"]: + if health_check_utils.check_chkconfig(serv) is False: + self._set_status( + 0, "[%s]Error: %s is disabled" % (self.NAME, serv)) + serv_to_enable.append(serv) + + if len(serv_to_enable) != 0: + self._set_status(0, "[%s]Info: You need to enable these " + "services on system start-up: %s" + % (self.NAME, + ", ".join(item for item in serv_to_enable))) + + return True + + def check_selinux(self): + """Check if SELinux is disabled.""" + print "Checking Selinux......", + disabled = False + with open("/etc/selinux/config") as selinux: + for line in selinux: + if "SELINUX=disabled" in line: + disabled = True + break + + if disabled is False: + self._set_status( + 0, + "[%s]Selinux is not disabled, " + "please disable it in /etc/selinux/config." % self.NAME) + + return True diff --git a/compass-tasks-base/actions/health_check/check_os_installer.py b/compass-tasks-base/actions/health_check/check_os_installer.py new file mode 100644 index 0000000..6ef9818 --- /dev/null +++ b/compass-tasks-base/actions/health_check/check_os_installer.py @@ -0,0 +1,151 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compass Health Check module for OS Installer.""" + +import os +import xmlrpclib + +from compass.actions.health_check import base + + +class OsInstallerCheck(base.BaseCheck): + """os installer health check.""" + NAME = "OS Installer Check" + + def run(self): + """do health check.""" + method_name = 'self.' + self.os_installer['name'] + '_check()' + return eval(method_name) + + def cobbler_check(self): + """Runs cobbler check from xmlrpc client.""" + try: + remote = xmlrpclib.Server( + self.os_installer['cobbler_url'], + allow_none=True) + credentials = self.os_installer['credentials'] + token = remote.login( + credentials['username'], credentials['password']) + except Exception: + self.code = 0 + self.messages.append( + "[%s]Error: Cannot login to Cobbler with " + "the tokens provided in the config file" + % self.NAME) + self.messages.append( + "[%s]Error: Failed to connect to Cobbler " + "API, please check if /etc/cobbler/setting " + "is properly configured" % self.NAME) + return (self.code, self.messages) + + check_result = remote.check(token) + + for index, message in enumerate(check_result): + if "SELinux" in message: + check_result.pop(index) + + if len(check_result) != 0: + self.code = 0 + for error_msg in check_result: + self.messages.append("[%s]Error: " % self.NAME + error_msg) + + if len(remote.get_distros()) == 0: + self._set_status(0, + "[%s]Error: No Cobbler distros found" % self.NAME) + + if len(remote.get_profiles()) == 0: + self._set_status(0, + "[%s]Error: No Cobbler profiles found" + % self.NAME) + + found_ppa = False + if len(remote.get_repos()) != 0: + for repo in remote.get_repos(): + if 'ppa_repo' in repo['mirror']: + found_ppa = True + break + + if found_ppa is False: + self._set_status(0, + "[%s]Error: No repository ppa_repo found" + % self.NAME) + + path_map = { + 'match_kickstart': ( + '/var/lib/cobbler/kickstarts/', + ['default.ks', 'default.seed'] + ), + 'match_snippets': ( + '/var/lib/cobbler/snippets/', + [ + 'kickstart_done', + 'kickstart_start', + 'kickstart_pre_partition_disks', + 'kickstart_partition_disks', + 'kickstart_pre_anamon', + 'kickstart_post_anamon', + 'kickstart_pre_install_network_config', + 'kickstart_network_config', + 'kickstart_post_install_network_config', + 'kickstart_chef', + 'kickstart_ntp', + 'kickstart_yum_repo_config', + 'preseed_pre_partition_disks', + 'preseed_partition_disks', + 'preseed_pre_anamon', + 'preseed_post_anamon', + 'preseed_pre_install_network_config', + 'preseed_network_config', + 'preseed_post_install_network_config', + 'preseed_chef', + 'preseed_ntp', + 'preseed_apt_repo_config', + ] + ), + 'match_ks_mirror': ( + '/var/www/cobbler/', + ['ks_mirror'] + ), + 'match_repo_mirror': ( + '/var/www/cobbler/', + ['repo_mirror'] + ), + 'match_iso': ( + '/var/lib/cobbler/', + ['iso'] + ), + } + not_exists = [] + for key in path_map.keys(): + for path in path_map[key][1]: + if not os.path.exists(path_map[key][0] + path): + not_exists.append(path_map[key][0] + path) + + if len(not_exists) != 0: + self._set_status( + 0, + "[%s]Error: These locations do not exist: " + "%s" % ( + self.NAME, + ', '.join(item for item in not_exists) + ) + ) + + if self.code == 1: + self.messages.append( + "[%s]Info: OS Installer health check has completed." + " No problems found, all systems go." % self.NAME) + + return (self.code, self.messages) diff --git a/compass-tasks-base/actions/health_check/check_package_installer.py b/compass-tasks-base/actions/health_check/check_package_installer.py new file mode 100644 index 0000000..efcd8e8 --- /dev/null +++ b/compass-tasks-base/actions/health_check/check_package_installer.py @@ -0,0 +1,68 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Health Check module for Package Installer.""" +import logging +import os +import requests + +from compass.actions.health_check import base +from compass.actions.health_check import utils as health_check_utils + + +class PackageInstallerCheck(base.BaseCheck): + """package installer health check class.""" + NAME = "Package Installer Check" + + def run(self): + """do health check.""" + method_name = "self." + self.package_installer['name'] + "_check()" + return eval(method_name) + + def chef_check(self): + """Checks chef setting, cookbooks and roles.""" + self.check_chef_config_dir() + print "[Done]" + if self.code == 1: + self.messages.append( + "[%s]Info: Package installer health check " + "has completed. No problems found, all systems " + "go." % self.NAME) + + return (self.code, self.messages) + + def check_chef_config_dir(self): + """Validates chef configuration directories.""" + + print "Checking Chef configurations......", + message = health_check_utils.check_path(self.NAME, '/etc/chef-server/') + if not message == "": + self._set_status(0, message) + + message = health_check_utils.check_path(self.NAME, '/opt/chef-server/') + if not message == "": + self._set_status(0, message) + + return None + + def ansible_check(self): + """Placeholder for ansible check.""" + print "Checking ansible......" + print ("[Done]") + self.code == 1 + self.messages.append( + "[%s]Info: Package installer health check " + "has completed. No problems found, all systems " + "go." % self.NAME) + return (self.code, self.messages) diff --git a/compass-tasks-base/actions/health_check/check_squid.py b/compass-tasks-base/actions/health_check/check_squid.py new file mode 100644 index 0000000..5628a63 --- /dev/null +++ b/compass-tasks-base/actions/health_check/check_squid.py @@ -0,0 +1,128 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Health Check module for Squid service.""" +import commands +import os +import pwd +import socket + +from compass.actions.health_check import base +from compass.actions.health_check import utils as health_check_utils + + +class SquidCheck(base.BaseCheck): + """Squid health check class.""" + NAME = "Squid Check" + + def run(self): + """do health check.""" + self.check_squid_files() + print "[Done]" + self.check_squid_service() + print "[Done]" + if self.code == 1: + self.messages.append( + "[%s]Info: Squid health check has completed. " + "No problems found, all systems go." % self.NAME) + return (self.code, self.messages) + + def check_squid_files(self): + """Validates squid config, cache directory and ownership.""" + print "Checking Squid Files......", + var_map = { + 'match_squid_conf': False, + 'match_squid_cache': False, + 'match_squid_ownership': False, + } + + conf_err_msg = health_check_utils.check_path( + self.NAME, + "/etc/squid/squid.conf") + if not conf_err_msg == "": + self._set_status(0, conf_err_msg) + elif int(oct(os.stat('/etc/squid/squid.conf').st_mode)) < 100644: + self._set_status( + 0, + "[%s]Error: squid.conf has incorrect " + "file permissions" % self.NAME) + else: + var_map['match_squid_conf'] = True + + squid_path_err_msg = health_check_utils.check_path( + self.NAME, '/var/squid/') + if not squid_path_err_msg == "": + self._set_status(0, squid_path_err_msg) + elif health_check_utils.check_path( + self.NAME, + '/var/squid/cache' + ) != "": + self._set_status( + 0, + health_check_utils.check_path( + self.NAME, + '/var/squid/cache' + ) + ) + else: + var_map['match_squid_cache'] = True + uid = os.stat('/var/squid/').st_uid + gid = os.stat('/var/squid/').st_gid + if uid != gid or pwd.getpwuid(23).pw_name != 'squid': + self._set_status( + 0, + "[%s]Error: /var/squid directory ownership " + "misconfigured" % self.NAME) + else: + var_map['match_squid_ownership'] = True + + fails = [] + for key in var_map.keys(): + if var_map[key] is False: + fails.append(key) + + if len(fails) != 0: + self.messages.append( + "[%s]Info: Failed components for squid config: " + "%s" % ( + self.NAME, + ', '.join(item for item in fails) + ) + ) + return True + + def check_squid_service(self): + """Checks if squid is running on port 3128.""" + + print "Checking Squid service......", + if 'squid' not in commands.getoutput('ps -ef'): + self._set_status( + 0, + "[%s]Error: squid service does not seem " + "running" % self.NAME) + + try: + if 'squid' != socket.getservbyport(3128): + self._set_status( + 0, + "[%s]Error: squid is not listening on " + "3128" % self.NAME) + + except Exception: + self._set_status( + 0, + "[%s]Error: No service is listening on 3128, " + "squid failed" % self.NAME) + + return True diff --git a/compass-tasks-base/actions/health_check/check_tftp.py b/compass-tasks-base/actions/health_check/check_tftp.py new file mode 100644 index 0000000..7ca6405 --- /dev/null +++ b/compass-tasks-base/actions/health_check/check_tftp.py @@ -0,0 +1,96 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Health Check module for TFTP service.""" +import os +import socket +import xmlrpclib + +from compass.actions.health_check import base +from compass.actions.health_check import utils as health_check_utils + + +class TftpCheck(base.BaseCheck): + """tftp health check class.""" + NAME = "TFTP Check" + + def run(self): + """do health check.""" + method_name = "self.check_" + self.os_installer['name'] + "_tftp()" + return eval(method_name) + + def check_cobbler_tftp(self): + """Checks if Cobbler manages TFTP service. + + :note: we assume TFTP service is running at the + same machine where this health check runs at + """ + + try: + remote = xmlrpclib.Server( + self.os_installer['cobbler_url'], + allow_none=True) + credentials = self.os_installer['credentials'] + remote.login( + credentials['username'], credentials['password']) + except Exception: + self._set_status( + 0, + "[%s]Error: Cannot login to Cobbler with the tokens " + " provided in the config file" % self.NAME) + return (self.code, self.messages) + + cobbler_settings = remote.get_settings() + if cobbler_settings['manage_tftp'] == 0: + self.messages.append( + '[TFTP]Info: tftp service is not managed by Compass') + return (0, self.messages) + self.check_tftp_dir() + print "[Done]" + self.check_tftp_service() + print "[Done]" + if self.code == 1: + self.messages.append( + "[%s]Info: tftp service health check has completed. " + "No problems found, all systems go." % self.NAME) + + return (self.code, self.messages) + + def check_tftp_dir(self): + """Validates TFTP directories and configurations.""" + print "Checking TFTP directories......", + if not os.path.exists('/var/lib/tftpboot/'): + self._set_status( + 0, + "[%s]Error: No tftp-boot libraries found, " + "please check if tftp server is properly " + "installed/managed" % self.NAME) + + return True + + def check_tftp_service(self): + """Checks if TFTP is running on port 69.""" + print "Checking TFTP services......", + serv_err_msg = health_check_utils.check_service_running(self.NAME, + 'xinetd') + if not serv_err_msg == "": + self._set_status(0, serv_err_msg) + + if 'tftp' != socket.getservbyport(69): + self._set_status( + 0, + "[%s]Error: tftp doesn't seem to be listening " + "on Port 60." % self.NAME) + + return True diff --git a/compass-tasks-base/actions/health_check/utils.py b/compass-tasks-base/actions/health_check/utils.py new file mode 100644 index 0000000..369c5b6 --- /dev/null +++ b/compass-tasks-base/actions/health_check/utils.py @@ -0,0 +1,114 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compass Health Check heavy-lifting utilities""" +import commands +import os +import platform +import re + + +def validate_setting(module, setting, param): + """Checks if a Compass setting exists in the config file. + + :param module : module name to be checked + :type module : string + :param setting : compass setting wrapper + :type setting : python module + :param param : settings defined in compass config file + :type param : string + + """ + if hasattr(setting, param): + return True + else: + err_msg = "[%s]Error: no %s defined" % (module, param) + return err_msg + + +def get_dist(): + """Returns the operating system related information.""" + + os_version, version, release = platform.linux_distribution() + return (os_version.lower().strip(), version, release.lower().strip()) + + +def check_path(module_name, path): + """Checks if a directory or file exisits. + + :param module_name : module name to be checked + :type module_name : string + :param path : path of the directory of file + :type path : string + + """ + err_msg = "" + if not os.path.exists(path): + err_msg = ( + "[%s]Error: %s does not exist, " + "please check your configurations.") % (module_name, path) + return err_msg + + +def check_service_running(module_name, service_name): + """Checks if a certain service is running. + + :param module_name : module name to be checked + :type module_name : string + :param service_name : service name to be checked + :type service_name : string + + """ + err_msg = "" + if service_name not in commands.getoutput('ps -ef'): + err_msg = "[%s]Error: %s is not running." % ( + module_name, service_name) + + return err_msg + + +def check_chkconfig(service_name): + """Checks if a service is enabled at the start up. + + :param service_name : service name to be checked + :type service_name : string + + """ + chk_on = False + for service in os.listdir('/etc/rc3.d/'): + if service_name in service and 'S' in service: + chk_on = True + break + + return chk_on + + +def strip_name(name): + """Reformats names.""" + if not any([s in name for s in "(,),-,_".split(',')]): + return name + + paren_regex = re.compile("(.*?)\s*\(") + dash_regex = re.compile("(.*?)\s*\-") + under_dash_regex = re.compile("(.*?)\s*\_") + + r1 = paren_regex.match(name) + r2 = dash_regex.match(name) + r3 = under_dash_regex.match(name) + shortest = 'AVeryLongStringForDefualt' + for r in [r1, r2, r3]: + if r and len(r.group(1)) < len(shortest): + shortest = r.group(1) + + return shortest diff --git a/compass-tasks-base/actions/install_callback.py b/compass-tasks-base/actions/install_callback.py new file mode 100644 index 0000000..14d2639 --- /dev/null +++ b/compass-tasks-base/actions/install_callback.py @@ -0,0 +1,181 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to receive installation callback. + + .. moduleauthor:: Xiaodong Wang +""" +import logging + +from compass.actions import util +from compass.db.api import cluster as cluster_api +from compass.db.api import host as host_api +from compass.db.api import user as user_db +from compass.deployment.deploy_manager import DeployManager +from compass.deployment.utils import constants as const + + +def os_installed( + host_id, clusterhosts_ready, clusters_os_ready, + username=None +): + """Callback when os is installed. + + :param host_id: host that os is installed. + :type host_id: integer + :param clusterhosts_ready: the clusterhosts that should trigger ready. + :param clusters_os_ready: the cluster that should trigger os ready. + + .. note:: + The function should be called out of database session. + """ + with util.lock('serialized_action') as lock: + if not lock: + raise Exception( + 'failed to acquire lock to ' + 'do the post action after os installation' + ) + logging.info( + 'os installed on host %s ' + 'with cluster host ready %s cluster os ready %s', + host_id, clusterhosts_ready, clusters_os_ready + ) + if username: + user = user_db.get_user_object(username) + else: + user = None + os_installed_triggered = False + for cluster_id, clusterhost_ready in clusterhosts_ready.items(): + if not clusterhost_ready and os_installed_triggered: + continue + cluster_id = int(cluster_id) + cluster_info = util.ActionHelper.get_cluster_info( + cluster_id, user) + adapter_id = cluster_info[const.ADAPTER_ID] + + adapter_info = util.ActionHelper.get_adapter_info( + adapter_id, cluster_id, user) + hosts_info = util.ActionHelper.get_hosts_info( + cluster_id, [host_id], user) + + deploy_manager = DeployManager( + adapter_info, cluster_info, hosts_info) + + if not os_installed_triggered: + deploy_manager.os_installed() + util.ActionHelper.host_ready(host_id, True, user) + os_installed_triggered = True + + if clusterhost_ready: + # deploy_manager.cluster_os_installed() + util.ActionHelper.cluster_host_ready( + cluster_id, host_id, False, user + ) + + if util.ActionHelper.is_cluster_os_ready(cluster_id, user): + logging.info("deploy_manager begin cluster_os_installed") + deploy_manager.cluster_os_installed() + + +def package_installed( + cluster_id, host_id, cluster_ready, + host_ready, username=None +): + """Callback when package is installed. + + :param cluster_id: cluster id. + :param host_id: host id. + :param cluster_ready: if the cluster should trigger ready. + :param host_ready: if the host should trigger ready. + + .. note:: + The function should be called out of database session. + """ + with util.lock('serialized_action') as lock: + if not lock: + raise Exception( + 'failed to acquire lock to ' + 'do the post action after package installation' + ) + logging.info( + 'package installed on cluster %s host %s ' + 'with cluster ready %s host ready %s', + cluster_id, host_id, cluster_ready, host_ready + ) + + if username: + user = user_db.get_user_object(username) + else: + user = None + cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) + adapter_id = cluster_info[const.ADAPTER_ID] + + adapter_info = util.ActionHelper.get_adapter_info( + adapter_id, cluster_id, user) + hosts_info = util.ActionHelper.get_hosts_info( + cluster_id, [host_id], user) + + deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) + + deploy_manager.package_installed() + util.ActionHelper.cluster_host_ready(cluster_id, host_id, True, user) + if cluster_ready: + util.ActionHelper.cluster_ready(cluster_id, False, user) + if host_ready: + util.ActionHelper.host_ready(host_id, False, user) + + +def cluster_installed( + cluster_id, clusterhosts_ready, + username=None +): + """Callback when cluster is installed. + + :param cluster_id: cluster id + :param clusterhosts_ready: clusterhosts that should trigger ready. + + .. note:: + The function should be called out of database session. + """ + with util.lock('serialized_action') as lock: + if not lock: + raise Exception( + 'failed to acquire lock to ' + 'do the post action after cluster installation' + ) + logging.info( + 'package installed on cluster %s with clusterhosts ready %s', + cluster_id, clusterhosts_ready + ) + if username: + user = user_db.get_user_object(username) + else: + user = None + cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) + adapter_id = cluster_info[const.ADAPTER_ID] + + adapter_info = util.ActionHelper.get_adapter_info( + adapter_id, cluster_id, user) + hosts_info = util.ActionHelper.get_hosts_info( + cluster_id, clusterhosts_ready.keys(), user) + + deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) + + deploy_manager.cluster_installed() + util.ActionHelper.cluster_ready(cluster_id, True, user) + for host_id, clusterhost_ready in clusterhosts_ready.items(): + if clusterhost_ready: + util.ActionHelper.cluster_host_ready( + cluster_id, host_id, False, user + ) diff --git a/compass-tasks-base/actions/patch.py b/compass-tasks-base/actions/patch.py new file mode 100644 index 0000000..6d29be6 --- /dev/null +++ b/compass-tasks-base/actions/patch.py @@ -0,0 +1,69 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to patch an existing cluster +""" +import logging +import simplejson as json + +from compass.actions import util +from compass.db.api import cluster as cluster_db +from compass.db.api import user as user_db +from compass.deployment.deploy_manager import Patcher +from compass.deployment.utils import constants as const + + +def patch(cluster_id, username=None): + """Patch cluster. + + :param cluster_id: id of the cluster + :type cluster_id: int + + .. note:: + The function should be called out of database session. + """ + with util.lock('serialized_action', timeout=1000) as lock: + if not lock: + raise Exception('failed to acquire lock to deploy') + + user = user_db.get_user_object(username) + cluster_hosts = cluster_db.list_cluster_hosts(cluster_id, user) + hosts_id_list = [host['id'] for host in cluster_hosts] + cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) + adapter_id = cluster_info[const.ADAPTER_ID] + + adapter_info = util.ActionHelper.get_adapter_info( + adapter_id, cluster_id, user) + hosts_info = util.ActionHelper.get_hosts_info( + cluster_id, hosts_id_list, user) + patch_successful = True + try: + patcher = Patcher( + adapter_info, cluster_info, hosts_info, cluster_hosts) + patched_config = patcher.patch() + except Exception as error: + logging.exception(error) + patch_successful = False + + if patch_successful: + clean_payload = '{"patched_roles": []}' + clean_payload = json.loads(clean_payload) + for cluster_host in cluster_hosts: + cluster_db.update_cluster_host( + cluster_id, cluster_host['id'], user, **clean_payload) + logging.info( + "cleaning up patched roles for host id: %s", + cluster_host['id'] + ) + logging.info("Patch successful: %s", patched_config) diff --git a/compass-tasks-base/actions/poll_switch.py b/compass-tasks-base/actions/poll_switch.py new file mode 100644 index 0000000..5c29b01 --- /dev/null +++ b/compass-tasks-base/actions/poll_switch.py @@ -0,0 +1,162 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to provider function to poll switch.""" +import logging +import netaddr + +from compass.actions import util +from compass.db.api import database +from compass.db.api import switch as switch_api +from compass.db.api import user as user_api +from compass.hdsdiscovery.hdmanager import HDManager + + +def _poll_switch(ip_addr, credentials, req_obj='mac', oper="SCAN"): + """Poll switch by ip addr. + + + Args: + ip_addr: ip addr of the switch. + credentials: credentials of the switch. + + Returns: switch attributes dict and list of machine attributes dict. + """ + under_monitoring = 'under_monitoring' + unreachable = 'unreachable' + polling_error = 'error' + hdmanager = HDManager() + vendor, state, err_msg = hdmanager.get_vendor(ip_addr, credentials) + if not vendor: + logging.info("*****error_msg: %s****", err_msg) + logging.error('no vendor found or match switch %s', ip_addr) + return ( + { + 'vendor': vendor, 'state': state, 'err_msg': err_msg + }, { + } + ) + + logging.debug( + 'hdmanager learn switch from %s', ip_addr + ) + results = [] + try: + results = hdmanager.learn( + ip_addr, credentials, vendor, req_obj, oper + ) + except Exception as error: + logging.exception(error) + state = unreachable + err_msg = ( + 'SNMP walk for querying MAC addresses timedout' + ) + return ( + { + 'vendor': vendor, 'state': state, 'err_msg': err_msg + }, { + } + ) + + logging.info("pollswitch %s result: %s", ip_addr, results) + if not results: + logging.error( + 'no result learned from %s', ip_addr + ) + state = polling_error + err_msg = 'No result learned from SNMP walk' + return ( + {'vendor': vendor, 'state': state, 'err_msg': err_msg}, + {} + ) + + logging.info('poll switch result: %s' % str(results)) + machine_dicts = {} + for machine in results: + mac = machine['mac'] + port = machine['port'] + vlan = int(machine['vlan']) + if vlan: + vlans = [vlan] + else: + vlans = [] + if mac not in machine_dicts: + machine_dicts[mac] = {'mac': mac, 'port': port, 'vlans': vlans} + else: + machine_dicts[mac]['port'] = port + machine_dicts[mac]['vlans'].extend(vlans) + + logging.debug('update switch %s state to under monitoring', ip_addr) + state = under_monitoring + return ( + {'vendor': vendor, 'state': state, 'err_msg': err_msg}, + machine_dicts.values() + ) + + +def poll_switch(poller_email, ip_addr, credentials, + req_obj='mac', oper="SCAN"): + """Query switch and update switch machines. + + .. note:: + When polling switch succeeds, for each mac it got from polling switch, + A Machine record associated with the switch is added to the database. + + :param ip_addr: switch ip address. + :type ip_addr: str + :param credentials: switch crednetials. + :type credentials: dict + :param req_obj: the object requested to query from switch. + :type req_obj: str + :param oper: the operation to query the switch. + :type oper: str, should be one of ['SCAN', 'GET', 'SET'] + + .. note:: + The function should be called out of database session scope. + """ + poller = user_api.get_user_object(poller_email) + ip_int = long(netaddr.IPAddress(ip_addr)) + with util.lock('poll switch %s' % ip_addr, timeout=120) as lock: + if not lock: + raise Exception( + 'failed to acquire lock to poll switch %s' % ip_addr + ) + + # TODO(grace): before repoll the switch, set the state to repolling. + # and when the poll switch is timeout, set the state to error. + # the frontend should only consider some main state like INTIALIZED, + # ERROR and SUCCESSFUL, REPOLLING is as an intermediate state to + # indicate the switch is in learning the mac of the machines connected + # to it. + logging.debug('poll switch: %s', ip_addr) + switch_dict, machine_dicts = _poll_switch( + ip_addr, credentials, req_obj=req_obj, oper=oper + ) + switches = switch_api.list_switches(ip_int=ip_int, user=poller) + if not switches: + logging.error('no switch found for %s', ip_addr) + return + + for switch in switches: + for machine_dict in machine_dicts: + logging.info('add machine: %s', machine_dict) + machine_dict['owner_id'] = poller.id + switch_api.add_switch_machine( + switch['id'], False, user=poller, **machine_dict + ) + switch_api.update_switch( + switch['id'], + user=poller, + **switch_dict + ) diff --git a/compass-tasks-base/actions/reinstall.py b/compass-tasks-base/actions/reinstall.py new file mode 100644 index 0000000..62d1bcb --- /dev/null +++ b/compass-tasks-base/actions/reinstall.py @@ -0,0 +1,38 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to reinstall a given cluster + + .. moduleauthor:: Xiaodong Wang +""" +import logging + +from compass.actions import util +from compass.db.api import database + + +def reinstall(cluster_hosts): + """Reinstall clusters. + + :param cluster_hosts: clusters and hosts in each cluster to reinstall. + :type cluster_hosts: dict of int or str to list of int or str + + .. note:: + The function should be called out of database session. + """ + with util.lock('serialized_action') as lock: + if not lock: + raise Exception( + 'failed to acquire lock to reinstall') + logging.debug('reinstall cluster_hosts: %s', cluster_hosts) diff --git a/compass-tasks-base/actions/search.py b/compass-tasks-base/actions/search.py new file mode 100644 index 0000000..73ce1d9 --- /dev/null +++ b/compass-tasks-base/actions/search.py @@ -0,0 +1,46 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to search configs of given clusters + + .. moduleauthor:: Xiaodong Wang +""" +import logging + +from compass.actions import util +from compass.config_management.utils.config_manager import ConfigManager +from compass.db.api import database + + +def search(cluster_hosts, cluster_propreties_match, + cluster_properties_name, host_properties_match, + host_properties_name): + """search clusters. + + :param cluster_hosts: clusters and hosts in each cluster to search. + :type cluster_hosts: dict of int or str to list of int or str + + .. note:: + The function should be called out of database session. + """ + logging.debug('search cluster_hosts: %s', cluster_hosts) + with database.session(): + cluster_hosts, os_versions, target_systems = ( + util.update_cluster_hosts(cluster_hosts)) + manager = ConfigManager() + return manager.filter_cluster_and_hosts( + cluster_hosts, os_versions, + target_systems, cluster_propreties_match, + cluster_properties_name, host_properties_match, + host_properties_name) diff --git a/compass-tasks-base/actions/update_progress.py b/compass-tasks-base/actions/update_progress.py new file mode 100644 index 0000000..67a9963 --- /dev/null +++ b/compass-tasks-base/actions/update_progress.py @@ -0,0 +1,298 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to update status and installing progress of the given cluster. + + .. moduleauthor:: Xiaodong Wang +""" +import logging + +from compass.actions import util +from compass.db.api import adapter_holder as adapter_api +from compass.db.api import cluster as cluster_api +from compass.db.api import host as host_api +from compass.db.api import user as user_api +from compass.log_analyzor import progress_calculator +from compass.utils import setting_wrapper as setting + + +def update_progress(): + """Update status and installing progress of the given cluster. + + :param cluster_hosts: clusters and hosts in each cluster to update. + :type cluster_hosts: dict of int or str to list of int or str + + .. note:: + The function should be called out of the database session scope. + In the function, it will update the database cluster_state and + host_state table for the deploying cluster and hosts. + + The function will also query log_progressing_history table to get + the lastest installing progress and the position of log it has + processed in the last run. The function uses these information to + avoid recalculate the progress from the beginning of the log file. + After the progress got updated, these information will be stored back + to the log_progressing_history for next time run. + """ + with util.lock('log_progressing', timeout=60, blocking=False) as lock: + if not lock: + logging.error( + 'failed to acquire lock to calculate installation progress' + ) + return + + logging.info('update installing progress') + + user = user_api.get_user_object(setting.COMPASS_ADMIN_EMAIL) + hosts = host_api.list_hosts(user=user) + host_mapping = {} + for host in hosts: + if 'id' not in host: + logging.error('id is not in host %s', host) + continue + host_id = host['id'] + if 'os_name' not in host: + logging.error('os_name is not in host %s', host) + continue + if 'os_installer' not in host: + logging.error('os_installer is not in host %s', host) + continue + host_dirname = setting.HOST_INSTALLATION_LOGDIR_NAME + if host_dirname not in host: + logging.error( + '%s is not in host %s', host_dirname, host + ) + continue + host_state = host_api.get_host_state(host_id, user=user) + if 'state' not in host_state: + logging.error('state is not in host state %s', host_state) + continue + if host_state['state'] == 'INSTALLING': + host_log_histories = host_api.get_host_log_histories( + host_id, user=user + ) + host_log_history_mapping = {} + for host_log_history in host_log_histories: + if 'filename' not in host_log_history: + logging.error( + 'filename is not in host log history %s', + host_log_history + ) + continue + host_log_history_mapping[ + host_log_history['filename'] + ] = host_log_history + host_mapping[host_id] = ( + host, host_state, host_log_history_mapping + ) + else: + logging.info( + 'ignore host state %s since it is not in installing', + host_state + ) + adapters = adapter_api.list_adapters(user=user) + adapter_mapping = {} + for adapter in adapters: + if 'id' not in adapter: + logging.error( + 'id not in adapter %s', adapter + ) + continue + if 'package_installer' not in adapter: + logging.info( + 'package_installer not in adapter %s', adapter + ) + continue + adapter_id = adapter['id'] + adapter_mapping[adapter_id] = adapter + clusters = cluster_api.list_clusters(user=user) + cluster_mapping = {} + for cluster in clusters: + if 'id' not in cluster: + logging.error('id not in cluster %s', cluster) + continue + cluster_id = cluster['id'] + if 'adapter_id' not in cluster: + logging.error( + 'adapter_id not in cluster %s', + cluster + ) + continue + cluster_state = cluster_api.get_cluster_state( + cluster_id, + user=user + ) + if 'state' not in cluster_state: + logging.error('state not in cluster state %s', cluster_state) + continue + cluster_mapping[cluster_id] = (cluster, cluster_state) + clusterhosts = cluster_api.list_clusterhosts(user=user) + clusterhost_mapping = {} + for clusterhost in clusterhosts: + if 'clusterhost_id' not in clusterhost: + logging.error( + 'clusterhost_id not in clusterhost %s', + clusterhost + ) + continue + clusterhost_id = clusterhost['clusterhost_id'] + if 'cluster_id' not in clusterhost: + logging.error( + 'cluster_id not in clusterhost %s', + clusterhost + ) + continue + cluster_id = clusterhost['cluster_id'] + if cluster_id not in cluster_mapping: + logging.info( + 'ignore clusterhost %s ' + 'since the cluster_id ' + 'is not in cluster_mapping %s', + clusterhost, cluster_mapping + ) + continue + cluster, _ = cluster_mapping[cluster_id] + if 'flavor_name' not in cluster: + logging.error( + 'flavor_name is not in clusterhost %s related cluster', + clusterhost + ) + continue + clusterhost_dirname = setting.CLUSTERHOST_INATALLATION_LOGDIR_NAME + if clusterhost_dirname not in clusterhost: + logging.error( + '%s is not in clusterhost %s', + clusterhost_dirname, clusterhost + ) + continue + adapter_id = cluster['adapter_id'] + if adapter_id not in adapter_mapping: + logging.info( + 'ignore clusterhost %s ' + 'since the adapter_id %s ' + 'is not in adaper_mapping %s', + clusterhost, adapter_id, adapter_mapping + ) + continue + adapter = adapter_mapping[adapter_id] + if 'package_installer' not in adapter: + logging.info( + 'ignore clusterhost %s ' + 'since the package_installer is not define ' + 'in adapter %s', + clusterhost, adapter + ) + continue + package_installer = adapter['package_installer'] + clusterhost['package_installer'] = package_installer + clusterhost['adapter_name'] = adapter['name'] + clusterhost_state = cluster_api.get_clusterhost_self_state( + clusterhost_id, user=user + ) + if 'state' not in clusterhost_state: + logging.error( + 'state not in clusterhost_state %s', + clusterhost_state + ) + continue + if clusterhost_state['state'] == 'INSTALLING': + clusterhost_log_histories = ( + cluster_api.get_clusterhost_log_histories( + clusterhost_id, user=user + ) + ) + clusterhost_log_history_mapping = {} + for clusterhost_log_history in clusterhost_log_histories: + if 'filename' not in clusterhost_log_history: + logging.error( + 'filename not in clusterhost_log_history %s', + clusterhost_log_history + ) + continue + clusterhost_log_history_mapping[ + clusterhost_log_history['filename'] + ] = clusterhost_log_history + clusterhost_mapping[clusterhost_id] = ( + clusterhost, clusterhost_state, + clusterhost_log_history_mapping + ) + else: + logging.info( + 'ignore clusterhost state %s ' + 'since it is not in installing', + clusterhost_state + ) + + progress_calculator.update_host_progress( + host_mapping) + for host_id, (host, host_state, host_log_history_mapping) in ( + host_mapping.items() + ): + host_api.update_host_state( + host_id, user=user, + percentage=host_state.get('percentage', 0), + message=host_state.get('message', ''), + severity=host_state.get('severity', 'INFO') + ) + for filename, host_log_history in ( + host_log_history_mapping.items() + ): + host_api.add_host_log_history( + host_id, filename=filename, user=user, + position=host_log_history.get('position', 0), + percentage=host_log_history.get('percentage', 0), + partial_line=host_log_history.get('partial_line', ''), + message=host_log_history.get('message', ''), + severity=host_log_history.get('severity', 'INFO'), + line_matcher_name=host_log_history.get( + 'line_matcher_name', 'start' + ) + ) + progress_calculator.update_clusterhost_progress( + clusterhost_mapping) + for ( + clusterhost_id, + (clusterhost, clusterhost_state, clusterhost_log_history_mapping) + ) in ( + clusterhost_mapping.items() + ): + cluster_api.update_clusterhost_state( + clusterhost_id, user=user, + percentage=clusterhost_state.get('percentage', 0), + message=clusterhost_state.get('message', ''), + severity=clusterhost_state.get('severity', 'INFO') + ) + for filename, clusterhost_log_history in ( + clusterhost_log_history_mapping.items() + ): + cluster_api.add_clusterhost_log_history( + clusterhost_id, user=user, filename=filename, + position=clusterhost_log_history.get('position', 0), + percentage=clusterhost_log_history.get('percentage', 0), + partial_line=clusterhost_log_history.get( + 'partial_line', ''), + message=clusterhost_log_history.get('message', ''), + severity=clusterhost_log_history.get('severity', 'INFO'), + line_matcher_name=( + clusterhost_log_history.get( + 'line_matcher_name', 'start' + ) + ) + ) + progress_calculator.update_cluster_progress( + cluster_mapping) + for cluster_id, (cluster, cluster_state) in cluster_mapping.items(): + cluster_api.update_cluster_state( + cluster_id, user=user + ) diff --git a/compass-tasks-base/actions/util.py b/compass-tasks-base/actions/util.py new file mode 100644 index 0000000..4d9f855 --- /dev/null +++ b/compass-tasks-base/actions/util.py @@ -0,0 +1,342 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to provide util for actions + + .. moduleauthor:: Xiaodong Wang ,xiaodongwang@huawei.com> +""" +import logging +import redis + +from contextlib import contextmanager + +from compass.db.api import adapter_holder as adapter_db +from compass.db.api import cluster as cluster_db +from compass.db.api import host as host_db +from compass.db.api import machine as machine_db +from compass.deployment.utils import constants as const + + +@contextmanager +def lock(lock_name, blocking=True, timeout=10): + """acquire a lock to do some actions. + + The lock is acquired by lock_name among the whole distributed + systems. + """ + # TODO(xicheng): in future we should explicitly told which redis + # server we want to talk to make the lock works on distributed + # systems. + redis_instance = redis.Redis() + instance_lock = redis_instance.lock(lock_name, timeout=timeout) + owned = False + try: + locked = instance_lock.acquire(blocking=blocking) + if locked: + owned = True + logging.debug('acquired lock %s', lock_name) + yield instance_lock + else: + logging.info('lock %s is already hold', lock_name) + yield None + + except Exception as error: + logging.info( + 'redis fails to acquire the lock %s', lock_name) + logging.exception(error) + yield None + + finally: + if owned: + instance_lock.acquired_until = 0 + instance_lock.release() + logging.debug('released lock %s', lock_name) + else: + logging.debug('nothing to release %s', lock_name) + + +class ActionHelper(object): + + @staticmethod + def get_adapter_info(adapter_id, cluster_id, user): + """Get adapter information. Return a dictionary as below, + + { + "id": 1, + "name": "xxx", + "flavors": [ + { + "flavor_name": "xxx", + "roles": ['xxx', 'yyy', ...], + "template": "xxx.tmpl" + }, + ... + ], + "metadata": { + "os_config": { + ... + }, + "package_config": { + ... + } + }, + "os_installer": { + "name": "cobbler", + "settings": {....} + }, + "pk_installer": { + "name": "chef", + "settings": {....} + }, + ... + } + To view a complete output, please refer to backend doc. + """ + + adapter_info = adapter_db.get_adapter(adapter_id, user=user) + metadata = cluster_db.get_cluster_metadata(cluster_id, user=user) + adapter_info.update({const.METADATA: metadata}) + + for flavor_info in adapter_info[const.FLAVORS]: + roles = flavor_info[const.ROLES] + flavor_info[const.ROLES] = ActionHelper._get_role_names(roles) + + return adapter_info + + @staticmethod + def _get_role_names(roles): + return [role[const.NAME] for role in roles] + + @staticmethod + def get_cluster_info(cluster_id, user): + """Get cluster information.Return a dictionary as below, + + { + "id": 1, + "adapter_id": 1, + "os_version": "CentOS-6.5-x86_64", + "name": "cluster_01", + "flavor": { + "flavor_name": "zzz", + "template": "xx.tmpl", + "roles": [...] + } + "os_config": {..}, + "package_config": {...}, + "deployed_os_config": {}, + "deployed_package_config": {}, + "owner": "xxx" + } + """ + + cluster_info = cluster_db.get_cluster(cluster_id, user=user) + + # convert roles retrieved from db into a list of role names + roles_info = cluster_info.setdefault( + const.FLAVOR, {}).setdefault(const.ROLES, []) + cluster_info[const.FLAVOR][const.ROLES] = \ + ActionHelper._get_role_names(roles_info) + + # get cluster config info + cluster_config = cluster_db.get_cluster_config(cluster_id, user=user) + cluster_info.update(cluster_config) + + deploy_config = cluster_db.get_cluster_deployed_config(cluster_id, + user=user) + cluster_info.update(deploy_config) + + return cluster_info + + @staticmethod + def get_hosts_info(cluster_id, hosts_id_list, user): + """Get hosts information. Return a dictionary as below, + + { + "hosts": { + 1($host_id): { + "reinstall_os": True, + "mac": "xxx", + "name": "xxx", + "roles": [xxx, yyy] + }, + "networks": { + "eth0": { + "ip": "192.168.1.1", + "netmask": "255.255.255.0", + "is_mgmt": True, + "is_promiscuous": False, + "subnet": "192.168.1.0/24" + }, + "eth1": {...} + }, + "os_config": {}, + "package_config": {}, + "deployed_os_config": {}, + "deployed_package_config": {} + }, + 2: {...}, + .... + } + } + """ + + hosts_info = {} + for host_id in hosts_id_list: + info = cluster_db.get_cluster_host(cluster_id, host_id, user=user) + logging.debug("checking on info %r %r" % (host_id, info)) + + info[const.ROLES] = ActionHelper._get_role_names(info[const.ROLES]) + + # TODO(grace): Is following line necessary?? + info.setdefault(const.ROLES, []) + + config = cluster_db.get_cluster_host_config(cluster_id, + host_id, + user=user) + info.update(config) + + networks = info[const.NETWORKS] + networks_dict = {} + # Convert networks from list to dictionary format + for entry in networks: + nic_info = {} + nic_info = { + entry[const.NIC]: { + const.IP_ADDR: entry[const.IP_ADDR], + const.NETMASK: entry[const.NETMASK], + const.MGMT_NIC_FLAG: entry[const.MGMT_NIC_FLAG], + const.PROMISCUOUS_FLAG: entry[const.PROMISCUOUS_FLAG], + const.SUBNET: entry[const.SUBNET] + } + } + networks_dict.update(nic_info) + + info[const.NETWORKS] = networks_dict + + hosts_info[host_id] = info + + return hosts_info + + @staticmethod + def save_deployed_config(deployed_config, user): + """Save deployed config.""" + cluster_config = deployed_config[const.CLUSTER] + cluster_id = cluster_config[const.ID] + del cluster_config[const.ID] + + cluster_db.update_cluster_deployed_config(cluster_id, user=user, + **cluster_config) + + hosts_id_list = deployed_config[const.HOSTS].keys() + for host_id in hosts_id_list: + config = deployed_config[const.HOSTS][host_id] + cluster_db.update_cluster_host_deployed_config(cluster_id, + host_id, + user=user, + **config) + + @staticmethod + def update_state( + cluster_id, host_id_list, user, **kwargs + ): + # update all clusterhosts state + for host_id in host_id_list: + cluster_db.update_cluster_host_state( + cluster_id, + host_id, + user=user, + **kwargs + ) + + # update cluster state + cluster_db.update_cluster_state( + cluster_id, + user=user, + **kwargs + ) + + @staticmethod + def delete_cluster( + cluster_id, host_id_list, user, delete_underlying_host=False + ): + """Delete cluster. + + If delete_underlying_host is set, underlying hosts will also + be deleted. + """ + if delete_underlying_host: + for host_id in host_id_list: + host_db.del_host( + host_id, True, True, user=user + ) + cluster_db.del_cluster( + cluster_id, True, True, user=user + ) + + @staticmethod + def delete_cluster_host( + cluster_id, host_id, user, delete_underlying_host=False + ): + """Delete clusterhost. + + If delete_underlying_host set, also delete underlying host. + """ + if delete_underlying_host: + host_db.del_host( + host_id, True, True, user=user + ) + cluster_db.del_cluster_host( + cluster_id, host_id, True, True, user=user + ) + + @staticmethod + def delete_host(host_id, user): + host_db.del_host( + host_id, True, True, user=user + ) + + @staticmethod + def host_ready(host_id, from_database_only, user): + """Trigger host ready.""" + host_db.update_host_state_internal( + host_id, from_database_only=from_database_only, + user=user, ready=True + ) + + @staticmethod + def cluster_host_ready( + cluster_id, host_id, from_database_only, user + ): + """Trigger clusterhost ready.""" + cluster_db.update_cluster_host_state_internal( + cluster_id, host_id, from_database_only=from_database_only, + user=user, ready=True + ) + + @staticmethod + def is_cluster_os_ready(cluster_id, user=None): + return cluster_db.is_cluster_os_ready(cluster_id, user=user) + + @staticmethod + def cluster_ready(cluster_id, from_database_only, user): + """Trigger cluster ready.""" + cluster_db.update_cluster_state_internal( + cluster_id, from_database_only=from_database_only, + user=user, ready=True + ) + + @staticmethod + def get_machine_IPMI(machine_id, user): + machine_info = machine_db.get_machine(machine_id, user=user) + return machine_info[const.IPMI_CREDS] diff --git a/compass-tasks-base/apiclient/__init__.py b/compass-tasks-base/apiclient/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/compass-tasks-base/apiclient/example.py b/compass-tasks-base/apiclient/example.py new file mode 100755 index 0000000..4c01b98 --- /dev/null +++ b/compass-tasks-base/apiclient/example.py @@ -0,0 +1,463 @@ +#!/usr/bin/python +# copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example code to deploy a cluster by compass client api.""" +import os +import re +import sys +import time + +# from compass.apiclient.restful import Client +from restful import Client + +COMPASS_SERVER_URL = 'http://localhost/api' +COMPASS_LOGIN_EMAIL = 'admin@huawei.com' +COMPASS_LOGIN_PASSWORD = 'admin' +SWITCH_IP = '172.29.8.40' +SWITCH_SNMP_VERSION = '2c' +SWITCH_SNMP_COMMUNITY = 'public' +CLUSTER_NAME = 'test_cluster' +HOST_NAME_PREFIX = 'host' +SERVICE_USERNAME = 'service' +SERVICE_PASSWORD = 'service' +CONSOLE_USERNAME = 'console' +CONSOLE_PASSWORD = 'console' +HA_VIP = '' + +MANAGEMENT_IP_START = '10.145.88.130' +MANAGEMENT_IP_END = '10.145.88.254' +MANAGEMENT_IP_GATEWAY = '10.145.88.1' +MANAGEMENT_NETMASK = '255.255.255.0' +MANAGEMENT_NIC = 'eth0' +MANAGEMENT_PROMISC = 0 +TENANT_IP_START = '192.168.10.130' +TENANT_IP_END = '192.168.10.255' +TENANT_IP_GATEWAY = '192.168.10.1' +TENANT_NETMASK = '255.255.255.0' +TENANT_NIC = 'eth0' +TENANT_PROMISC = 0 +PUBLIC_IP_START = '12.234.32.130' +PUBLIC_IP_END = '12.234.32.255' +PUBLIC_IP_GATEWAY = '12.234.32.1' +PUBLIC_NETMASK = '255.255.255.0' +PUBLIC_NIC = 'eth1' +PUBLIC_PROMISC = 1 +STORAGE_IP_START = '172.16.100.130' +STORAGE_IP_END = '172.16.100.255' +STORAGE_NETMASK = '255.255.255.0' +STORAGE_IP_GATEWAY = '172.16.100.1' +STORAGE_NIC = 'eth0' +STORAGE_PROMISC = 0 +HOME_PERCENTAGE = 5 +TMP_PERCENTAGE = 5 +VAR_PERCENTAGE = 10 +HOST_OS = 'CentOS-6.5-x86_64' + + +PRESET_VALUES = { + 'LANGUAGE': 'EN', + 'TIMEZONE': 'GMT', + 'HTTPS_PROXY': 'http://10.145.89.100:3128', + 'NO_PROXY': ['127.0.0.1'], + 'DOMAIN': 'ods.com', + 'NAMESERVERS': ['10.145.89.100'], + 'NTP_SERVER': '10.145.89.100', + 'GATEWAY': '10.145.88.1', + 'PROXY': 'http://10.145.89.100:3128', + 'OS_NAME_PATTERN': 'CentOS.*', + 'ADAPTER_NAME': 'openstack_icehouse', + 'FLAVOR_PATTERN': 'allinone.*', + 'ROLES_LIST': ['allinone-compute'], + 'MACHINES_TO_ADD': ['00:0c:29:a7:ea:4b'], + 'BUILD_TIMEOUT': 60, + 'SEARCH_PATH': ['ods.com'], + 'SERVER_USERNAME': 'root', + 'SERVER_PASSWORD': 'root' +} +for v in PRESET_VALUES: + if v in os.environ.keys(): + PRESET_VALUES[v] = os.environ.get(v) + print (v + PRESET_VALUES[v] + " is set by env variables") + else: + print (PRESET_VALUES[v]) + +# instantiate a client +client = Client(COMPASS_SERVER_URL) + +# login +status, response = client.login(COMPASS_LOGIN_EMAIL, COMPASS_LOGIN_PASSWORD) +print '============================================================' +print 'login status: %s response: %s' % (status, response) +if status >= 400: + sys.exit(1) + +# list all switches +status, response = client.list_switches() +print '=============================================================' +print 'get all switches status: %s response: %s' % (status, response) + +# add a switch +status, response = client.add_switch( + SWITCH_IP, + SWITCH_SNMP_VERSION, + SWITCH_SNMP_COMMUNITY +) +print '============================================' +print 'adding a switch..status: %s, response: %s' % (status, response) + +# if switch already exists, get one from all switches +switch = None +if status < 400: + switch = response +else: + status, response = client.list_switches() + print '=========================================' + print 'list switches status %s response %s' % (status, response) + if status >= 400: + sys.exit(1) + for switch_ in response: + if switch_['ip'] == SWITCH_IP: + switch = switch_ + break + +switch_id = switch['id'] +switch_ip = switch['ip'] +print '======================' +print 'switch has been set as %s' % switch_ip + +# wait till switch state becomes under_monitoring +while switch['state'] != 'under_monitoring': + print 'waiting for state to become under_monitoring' + client.poll_switch(switch_id) + status, resp = client.get_switch(switch_id) + print '=====================================' + print 'poll switch status %s response %s' % (status, resp) + switch = resp + print 'switch is in state: %s' % switch['state'] + time.sleep(5) + +print '=========================================' +print 'switch state now is %s' % (switch['state']) + +# create a machine list +machine_macs = {} +machines = {} +for machine in PRESET_VALUES['MACHINES_TO_ADD']: + status, response = client.list_machines(mac=machine) + print '============================================' + print 'list machines status %s response %s' % (status, response) + if status >= 400: + sys.exit(1) + if status == 200 and response != []: + machine_id = response[0]['id'] + machine_macs[machine_id] = response[0]['mac'] + machines = response + +print '=================================' +print 'found machines are : %s' % machines + +machines_to_add = PRESET_VALUES['MACHINES_TO_ADD'] +if set(machine_macs.values()) != set(machines_to_add): + print 'only found macs %s while expected are %s' % ( + machine_macs.values(), machines_to_add) + sys.exit(1) + +# list all adapters +status, response = client.list_adapters() +print '===============================' +print 'all adapters are: %s' % response +if status >= 400: + sys.exit(1) + +adapters = response +adapter_id = None +os_id = None +flavor_id = None +adapter_name = PRESET_VALUES['ADPATER_NAME'] +os_pattern = re.compile(PRESET_VALUES['OS_NAME_PATTERN']) +flavor_pattern = re.compile(PRESET_VALUES['FLAVOR_PATTERN']) +for adapter in adapters: + if adapter_name == adapter['name']: + adapter_id = adapter['id'] + for supported_os in adapter['supported_oses']: + if os_pattern.match(supported_os['name']): + os_id = supported_os['id'] + break + for flavor in adapter['flavors']: + if flavor_pattern.match(flavor['name']): + flavor_id = flavor['id'] + if adapter_id and os_id and flavor_id: + break + +print '=======================================================' +print 'using adapter %s os %s flavor %s to deploy cluster' % ( + adapter_id, os_id, flavor_id +) + +# add a cluster +status, response = client.add_cluster( + CLUSTER_NAME, + adapter_id, + os_id, + flavor_id +) +print '===============================================================' +print 'add cluster %s status %s: %s' % (CLUSTER_NAME, status, response) +if status >= 400: + sys.exit(1) + +status, response = client.list_clusters(name=CLUSTER_NAME) +print '================================================================' +print 'list clusters %s status %s: %s' % (CLUSTER_NAME, status, response) +if status >= 400: + sys.exit(1) + +cluster = response[0] +cluster_id = cluster['id'] + +print '==================' +print 'cluster is %s' % cluster + +# Add hosts to the cluster +machines_dict = {} +machine_id_list = [] +for machine in machines: + id_mapping = {} + id_mapping['machine_id'] = machine['id'] + machine_id_list.append(id_mapping) + +machines_dict['machines'] = machine_id_list + +status, response = client.add_hosts_to_cluster( + cluster_id, machines_dict +) +print '===================================' +print 'add hosts %s to cluster status %s response %s' % ( + machines_dict, status, response) +if status >= 400: + sys.exit(1) + +# Add two subnets +subnet_1 = '10.145.89.0/24' +subnet_2 = '192.168.100.0/24' + +status, response = client.add_subnet(subnet_1) +print '==================' +print 'add subnet %s status %s: %s' % (subnet_1, status, response) +if status >= 400: + sys.exit(1) + +status, response = client.add_subnet(subnet_2) +print '==================' +print 'add subnet %s status %s: %s' % (subnet_2, status, response) +if status >= 400: + sys.exit(1) + +status, subnet1 = client.list_subnets(subnet=subnet_1) +print '===========================================================' +print 'list subnet %s status %s: %s' % (subnet_1, status, subnet1) +if status >= 400: + sys.exit(1) + +status, subnet2 = client.list_subnets(subnet=subnet_2) +print '===========================================================' +print 'list subnet %s status %s: %s' % (subnet_2, status, subnet2) +if status >= 400: + sys.exit(1) + +subnet1_id = subnet1[0]['id'] +subnet2_id = subnet2[0]['id'] +print '========================' +print 'subnet1 has id: %s, subnet is %s' % (subnet1_id, subnet1) +print 'subnet2 has id: %s, subnet is %s' % (subnet2_id, subnet2) + +# Add host network +status, response = client.list_cluster_hosts(cluster_id) +print '================================================' +print 'list cluster hosts status %s: %s' % (status, response) +if status >= 400: + sys.exit(1) + +host = response[0] +host_id = host['id'] +print '==================' +print 'host is: %s' % host + +status, response = client.add_host_network( + host_id, + 'eth0', + '10.145.89.200', + subnet1_id, + is_mgmt=True +) +print '=======================' +print 'add eth0 network status %s: %s' % (status, response) +if status >= 400: + sys.exit(1) + +status, response = client.add_host_network( + host_id, + 'eth1', + '192.168.100.200', + subnet2_id, + is_promiscuous=True +) +print '=======================' +print 'add eth1 network status %s: %s' % (status, response) +if status >= 400: + sys.exit(1) + +# Update os config to cluster +cluster_os_config = { + 'general': { + 'language': PRESET_VALUES['LANGUAGE'], + 'timezone': PRESET_VALUES['TIMEZONE'], + 'http_proxy': PRESET_VALUES['PROXY'], + 'https_proxy': PRESET_VALUES['HTTPS_PROXY'], + 'no_proxy': PRESET_VALUES['NO_PROXY'], + 'ntp_server': PRESET_VALUES['NTP_SERVER'], + 'dns_servers': PRESET_VALUES['NAMESERVERS'], + 'domain': PRESET_VALUES['DOMAIN'], + 'search_path': PRESET_VALUES['SEARCH_PATH'], + 'default_gateway': PRESET_VALUES['GATEWAY'] + }, + 'server_credentials': { + 'username': PRESET_VALUES['SERVER_USERNAME'], + 'password': PRESET_VALUES['SERVER_PASSWORD'] + }, + 'partition': { + '/var': { + 'percentage': VAR_PERCENTAGE, + }, + '/home': { + 'percentage': HOME_PERCENTAGE, + } + } +} + + +cluster_package_config = { + 'security': { + 'service_credentials': { + 'image': { + 'username': SERVICE_USERNAME, + 'password': SERVICE_PASSWORD + }, + 'compute': { + 'username': SERVICE_USERNAME, + 'password': SERVICE_PASSWORD + }, + 'dashboard': { + 'username': SERVICE_USERNAME, + 'password': SERVICE_PASSWORD + }, + 'identity': { + 'username': SERVICE_USERNAME, + 'password': SERVICE_PASSWORD + }, + 'metering': { + 'username': SERVICE_USERNAME, + 'password': SERVICE_PASSWORD + }, + 'rabbitmq': { + 'username': SERVICE_USERNAME, + 'password': SERVICE_PASSWORD + }, + 'volume': { + 'username': SERVICE_USERNAME, + 'password': SERVICE_PASSWORD + }, + 'mysql': { + 'username': SERVICE_USERNAME, + 'password': SERVICE_PASSWORD + } + }, + 'console_credentials': { + 'admin': { + 'username': CONSOLE_USERNAME, + 'password': CONSOLE_PASSWORD + }, + 'compute': { + 'username': CONSOLE_USERNAME, + 'password': CONSOLE_PASSWORD + }, + 'dashboard': { + 'username': CONSOLE_USERNAME, + 'password': CONSOLE_PASSWORD + }, + 'image': { + 'username': CONSOLE_USERNAME, + 'password': CONSOLE_PASSWORD + }, + 'metering': { + 'username': CONSOLE_USERNAME, + 'password': CONSOLE_PASSWORD + }, + 'network': { + 'username': CONSOLE_USERNAME, + 'password': CONSOLE_PASSWORD + }, + 'object-store': { + 'username': CONSOLE_USERNAME, + 'password': CONSOLE_PASSWORD + }, + 'volume': { + 'username': CONSOLE_USERNAME, + 'password': CONSOLE_PASSWORD + } + } + }, + 'network_mapping': { + 'management': MANAGEMENT_NIC, + 'tenant': TENANT_NIC, + 'storage': STORAGE_NIC, + 'public': PUBLIC_NIC + } +} + +status, response = client.update_cluster_config( + cluster_id, + cluster_os_config, + cluster_package_config +) + +print '=======================================' +print 'cluster %s update status %s: %s' % ( + cluster_id, status, response) +if status >= 400: + sys.exit(1) + +status, response = client.update_cluster_host( + cluster_id, host_id, roles=PRESET_VALUES['ROLES_LIST']) +print '=================================================' +print 'update cluster host %s/%s status %s: %s' % ( + cluster_id, host_id, status, response) +if status >= 400: + sys.exit(1) + +# Review and deploy +status, response = client.review_cluster( + cluster_id, review={'hosts': [host_id]}) +print '=======================================' +print 'reviewing cluster status %s: %s' % (status, response) +if status >= 400: + sys.exit(1) + +status, response = client.deploy_cluster( + cluster_id, deploy={'hosts': [host_id]}) +print '=======================================' +print 'deploy cluster status %s: %s' % (status, response) +if status >= 400: + sys.exit(1) diff --git a/compass-tasks-base/apiclient/restful.py b/compass-tasks-base/apiclient/restful.py new file mode 100644 index 0000000..bb82922 --- /dev/null +++ b/compass-tasks-base/apiclient/restful.py @@ -0,0 +1,1102 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compass api client library. +""" + +import json +import logging +import requests + + +class Client(object): + """compass restful api wrapper""" + + def __init__(self, url, headers=None, proxies=None, stream=None): + logging.info('create api client %s', url) + self.url_ = url + self.session_ = requests.Session() + + if headers: + self.session_.headers.update(headers) + self.session_.headers.update({ + 'Accept': 'application/json' + }) + + if proxies is not None: + self.session_.proxies = proxies + + if stream is not None: + self.session_.stream = stream + + def __del__(self): + self.session_.close() + + @classmethod + def _get_response(cls, resp): + response_object = {} + try: + response_object = resp.json() + except Exception as error: + logging.error('failed to load object from %s: %s', + resp.url, resp.content) + logging.exception(error) + response_object['status'] = 'Json Parsing Failed' + response_object['message'] = resp.content + + return resp.status_code, response_object + + def _get(self, req_url, data=None): + url = '%s%s' % (self.url_, req_url) + logging.debug('get %s with data %s', url, data) + if data: + resp = self.session_.get(url, params=data) + else: + resp = self.session_.get(url) + + return self._get_response(resp) + + def _post(self, req_url, data=None): + url = '%s%s' % (self.url_, req_url) + logging.debug('post %s with data %s', url, data) + if data: + resp = self.session_.post(url, json.dumps(data)) + else: + resp = self.session_.post(url) + + return self._get_response(resp) + + def _put(self, req_url, data=None): + """encapsulate put method.""" + url = '%s%s' % (self.url_, req_url) + logging.debug('put %s with data %s', url, data) + if data: + resp = self.session_.put(url, json.dumps(data)) + else: + resp = self.session_.put(url) + + return self._get_response(resp) + + def _patch(self, req_url, data=None): + url = '%s%s' % (self.url_, req_url) + logging.debug('patch %s with data %s', url, data) + if data: + resp = self.session_.patch(url, json.dumps(data)) + else: + resp = self.session_.patch(url) + + return self._get_response(resp) + + def _delete(self, req_url): + url = '%s%s' % (self.url_, req_url) + logging.debug('delete %s', url) + return self._get_response(self.session_.delete(url)) + + def login(self, email, password): + credential = {} + credential['email'] = email + credential['password'] = password + return self._post('/users/login', data=credential) + + def get_token(self, email, password): + credential = {} + credential['email'] = email + credential['password'] = password + status, resp = self._post('/users/token', data=credential) + if status < 400: + self.session_.headers.update({'X-Auth-Token': resp['token']}) + return status, resp + + def get_users(self): + users = self._get('/users') + return users + + def list_switches( + self, + switch_ips=None, + switch_ip_networks=None): + """list switches.""" + params = {} + if switch_ips: + params['switchIp'] = switch_ips + + if switch_ip_networks: + params['switchIpNetwork'] = switch_ip_networks + + switchlist = self._get('/switches', data=params) + return switchlist + + def get_switch(self, switch_id): + return self._get('/switches/%s' % switch_id) + + def add_switch( + self, + switch_ip, + version=None, + community=None, + raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + data['ip'] = switch_ip + data['credentials'] = {} + if version: + data['credentials']['version'] = version + + if community: + data['credentials']['community'] = community + + return self._post('/switches', data=data) + + def update_switch(self, switch_id, state='initialized', + version='2c', community='public', raw_data={}): + data = {} + if raw_data: + data = raw_data + + else: + data['credentials'] = {} + if version: + data['credentials']['version'] = version + + if community: + data['credentials']['community'] = community + + if state: + data['state'] = state + + return self._put('/switches/%s' % switch_id, data=data) + + def delete_switch(self, switch_id): + return self._delete('/switches/%s' % switch_id) + + def list_switch_machines(self, switch_id, port=None, vlans=None, + tag=None, location=None): + data = {} + if port: + data['port'] = port + + if vlans: + data['vlans'] = vlans + + if tag: + data['tag'] = tag + + if location: + data['location'] = location + + return self._get('/switches/%s/machines' % switch_id, data=data) + + def get_switch_machine(self, switch_id, machine_id): + return self._get('/switches/%s/machines/%s' % (switch_id, machine_id)) + + def list_switch_machines_hosts(self, switch_id, port=None, vlans=None, + mac=None, tag=None, location=None, + os_name=None, os_id=None): + + data = {} + if port: + data['port'] = port + + if vlans: + data['vlans'] = vlans + + if mac: + data['mac'] = mac + + if tag: + data['tag'] = tag + + if location: + data['location'] = location + + if os_name: + data['os_name'] = os_name + + if os_id: + data['os_id'] = os_id + + return self._get('/switches/%s/machines-hosts' % switch_id, data=data) + + def add_switch_machine(self, switch_id, mac=None, port=None, + vlans=None, ipmi_credentials=None, + tag=None, location=None, raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if mac: + data['mac'] = mac + + if port: + data['port'] = port + + if vlans: + data['vlans'] = vlans + + if ipmi_credentials: + data['ipmi_credentials'] = ipmi_credentials + + if tag: + data['tag'] = tag + + if location: + data['location'] = location + + return self._post('/switches/%s/machines' % switch_id, data=data) + + def update_switch_machine(self, switch_id, machine_id, port=None, + vlans=None, ipmi_credentials=None, tag=None, + location=None, raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if port: + data['port'] = port + + if vlans: + data['vlans'] = vlans + + if ipmi_credentials: + data['ipmi_credentials'] = ipmi_credentials + + if tag: + data['tag'] = tag + + if location: + data['location'] = location + + return self._put('/switches/%s/machines/%s' % + (switch_id, machine_id), data=data) + + def delete_switch_machine(self, switch_id, machine_id): + return self._delete('/switches/%s/machines/%s' % + (switch_id, machine_id)) + + # test these + def poll_switch(self, switch_id): + data = {} + data['find_machines'] = None + return self._post('/switches/%s/action' % switch_id, data=data) + + def add_group_switch_machines(self, switch_id, group_machine_ids): + data = {} + data['add_machines'] = group_machine_ids + return self._post('/switches/%s/action' % switch_id, data=data) + + def remove_group_switch_machines(self, switch_id, group_machine_ids): + data = {} + data['remove_machines'] = group_machine_ids + return self._post('/switches/%s/action' % switch_id, data=data) + + def update_group_switch_machines(self, switch_id, group_machines): + data = {} + data['set_machines'] = group_machines + return self._post('/switches/%s/action' % switch_id, data=data) + # end + + def list_switchmachines(self, switch_ip_int=None, port=None, vlans=None, + mac=None, tag=None, location=None): + data = {} + if switch_ip_int: + data['switch_ip_int'] = switch_ip_int + + if port: + data['port'] = port + + if vlans: + data['vlans'] = vlans + + if mac: + data['mac'] = mac + + if tag: + data['tag'] = tag + + if location: + data['location'] = location + + return self._get('/switch-machines', data=data) + + def list_switchmachines_hosts(self, switch_ip_int=None, port=None, + vlans=None, mac=None, tag=None, + location=None, os_name=None, os_id=None): + + data = {} + if switch_ip_int: + data['switch_ip_int'] = switch_ip_int + + if port: + data['port'] = port + + if vlans: + data['vlans'] = vlans + + if mac: + data['mac'] = mac + + if tag: + data['tag'] = tag + + if location: + data['location'] = location + + if os_name: + data['os_name'] = os_name + + if os_id: + data['os_id'] = os_id + + return self._get('/switches-machines-hosts', data=data) + + def show_switchmachine(self, switchmachine_id): + return self._get('/switch-machines/%s' % switchmachine_id) + + def update_switchmachine(self, switchmachine_id, + port=None, vlans=None, raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if port: + data['port'] = port + + if vlans: + data['vlans'] = vlans + + return self._put('/switch-machines/%s' % switchmachine_id, data=data) + + def patch_switchmachine(self, switchmachine_id, + vlans=None, raw_data=None): + data = {} + if raw_data: + data = raw_data + elif vlans: + data['vlans'] = vlans + + return self._patch('/switch-machines/%s' % switchmachine_id, data=data) + + def delete_switchmachine(self, switchmachine_id): + return self._delete('/switch-machines/%s' % switchmachine_id) + + def list_machines(self, mac=None, tag=None, location=None): + data = {} + if mac: + data['mac'] = mac + + if tag: + data['tag'] = tag + + if location: + data['location'] = location + + return self._get('/machines', data=data) + + def get_machine(self, machine_id): + data = {} + if id: + data['id'] = id + + return self._get('/machines/%s' % machine_id, data=data) + + def update_machine(self, machine_id, ipmi_credentials=None, tag=None, + location=None, raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if ipmi_credentials: + data['ipmi_credentials'] = ipmi_credentials + + if tag: + data['tag'] = tag + + if location: + data['location'] = location + + return self._put('/machines/%s' % machine_id, data=data) + + def patch_machine(self, machine_id, ipmi_credentials=None, + tag=None, location=None, + raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if ipmi_credentials: + data['ipmi_credentials'] = ipmi_credentials + + if tag: + data['tag'] = tag + + if location: + data['location'] = location + + return self._patch('/machines/%s' % machine_id, data=data) + + def delete_machine(self, machine_id): + return self._delete('machines/%s' % machine_id) + + def list_subnets(self, subnet=None, name=None): + data = {} + if subnet: + data['subnet'] = subnet + + if name: + data['name'] = name + + return self._get('/subnets', data=data) + + def get_subnet(self, subnet_id): + return self._get('/subnets/%s' % subnet_id) + + def add_subnet(self, subnet, name=None, raw_data=None): + data = {} + data['subnet'] = subnet + if raw_data: + data.update(raw_data) + else: + if name: + data['name'] = name + + return self._post('/subnets', data=data) + + def update_subnet(self, subnet_id, subnet=None, + name=None, raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if subnet: + data['subnet'] = subnet + + if name: + data['name'] = name + return self._put('/subnets/%s' % subnet_id, data=data) + + def delete_subnet(self, subnet_id): + return self._delete('/subnets/%s' % subnet_id) + + def list_adapters(self, name=None): + data = {} + if name: + data['name'] = name + + return self._get('/adapters', data=data) + + def get_adapter(self, adapter_id): + return self._get('/adapters/%s' % adapter_id) + + def get_adapter_roles(self, adapter_id): + return self._get('/adapters/%s/roles' % adapter_id) + + def get_adapter_metadata(self, adapter_id): + return self._get('/adapters/%s/metadata' % adapter_id) + + def get_os_metadata(self, os_id): + return self._get('/oses/%s/metadata' % os_id) + + def list_clusters(self, name=None, os_name=None, + owner=None, + adapter_id=None): + data = {} + if name: + data['name'] = name + + if os_name: + data['os_name'] = os_name + + if owner: + data['owner'] = owner + + if adapter_id: + data['adapter_id'] = adapter_id + + return self._get('/clusters', data=data) + + def get_cluster(self, cluster_id): + return self._get('/clusters/%s' % cluster_id) + + def add_cluster(self, name, adapter_id, os_id, + flavor_id=None, raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if flavor_id: + data['flavor_id'] = flavor_id + data['name'] = name + data['adapter_id'] = adapter_id + data['os_id'] = os_id + + return self._post('/clusters', data=data) + + def update_cluster(self, cluster_id, name=None, + reinstall_distributed_system=None, + raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if name: + data['name'] = name + + if reinstall_distributed_system: + data['reinstall_distributed_system'] = ( + reinstall_distributed_system + ) + return self._put('/clusters/%s' % cluster_id, data=data) + + def delete_cluster(self, cluster_id): + return self._delete('/clusters/%s' % cluster_id) + + def get_cluster_config(self, cluster_id): + return self._get('/clusters/%s/config' % cluster_id) + + def get_cluster_metadata(self, cluster_id): + return self._get('/clusters/%s/metadata' % cluster_id) + + def update_cluster_config(self, cluster_id, os_config=None, + package_config=None, config_step=None, + raw_data=None): + data = {} + if raw_data: + data = raw_data + + if os_config: + data['os_config'] = os_config + + if package_config: + data['package_config'] = package_config + + if config_step: + data['config_step'] = config_step + + return self._put('/clusters/%s/config' % cluster_id, data=data) + + def patch_cluster_config(self, cluster_id, os_config=None, + package_config=None, config_step=None, + raw_data=None): + data = {} + if raw_data: + data = raw_data + + if os_config: + data['os_config'] = os_config + + if package_config: + data['package_config'] = package_config + + if config_step: + data['config_step'] = config_step + + return self._patch('/clusters/%s/config' % cluster_id, data=data) + + def delete_cluster_config(self, cluster_id): + return self._delete('/clusters/%s/config' % cluster_id) + + # test these + def add_hosts_to_cluster(self, cluster_id, hosts): + data = {} + data['add_hosts'] = hosts + return self._post('/clusters/%s/action' % cluster_id, data=data) + + def set_hosts_in_cluster(self, cluster_id, hosts): + data = {} + data['set_hosts'] = hosts + return self._post('/clusters/%s/action' % cluster_id, data=data) + + def remove_hosts_from_cluster(self, cluster_id, hosts): + data = {} + data['remove_hosts'] = hosts + return self._post('/clusters/%s/action' % cluster_id, data=data) + + def review_cluster(self, cluster_id, review={}): + data = {} + data['review'] = review + return self._post('/clusters/%s/action' % cluster_id, data=data) + + def deploy_cluster(self, cluster_id, deploy={}): + data = {} + data['deploy'] = deploy + return self._post('/clusters/%s/action' % cluster_id, data=data) + + def redeploy_cluster(self, cluster_id, deploy={}): + data = {} + data['redeploy'] = deploy + return self._post('/clusters/%s/action' % cluster_id, data=data) + + def get_cluster_state(self, cluster_id): + return self._get('/clusters/%s/state' % cluster_id) + + def list_cluster_hosts(self, cluster_id): + return self._get('/clusters/%s/hosts' % cluster_id) + + def list_clusterhosts(self): + return self._get('/clusterhosts') + + def get_cluster_host(self, cluster_id, host_id): + return self._get('/clusters/%s/hosts/%s' % (cluster_id, host_id)) + + def get_clusterhost(self, clusterhost_id): + return self._get('/clusterhosts/%s' % clusterhost_id) + + def add_cluster_host(self, cluster_id, machine_id=None, name=None, + reinstall_os=None, raw_data=None): + data = {} + data['machine_id'] = machine_id + if raw_data: + data.update(raw_data) + else: + if name: + data['name'] = name + + if reinstall_os: + data['reinstall_os'] = reinstall_os + + return self._post('/clusters/%s/hosts' % cluster_id, data=data) + + def delete_cluster_host(self, cluster_id, host_id): + return self._delete('/clusters/%s/hosts/%s' % + (cluster_id, host_id)) + + def delete_clusterhost(self, clusterhost_id): + return self._delete('/clusterhosts/%s' % clusterhost_id) + + def get_cluster_host_config(self, cluster_id, host_id): + return self._get('/clusters/%s/hosts/%s/config' % + (cluster_id, host_id)) + + def get_clusterhost_config(self, clusterhost_id): + return self._get('/clusterhosts/%s/config' % clusterhost_id) + + def update_cluster_host_config(self, cluster_id, host_id, + os_config=None, + package_config=None, + raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if os_config: + data['os_config'] = os_config + + if package_config: + data['package_config'] = package_config + + return self._put('/clusters/%s/hosts/%s/config' % + (cluster_id, host_id), data=data) + + def update_clusterhost_config(self, clusterhost_id, os_config=None, + package_config=None, raw_data=None): + data = {} + if raw_data: + data = raw_data + + else: + if os_config: + data['os_config'] = os_config + + if package_config: + data['package_config'] = package_config + + return self._put('/clusterhosts/%s/config' % clusterhost_id, + data=data) + + def patch_cluster_host_config(self, cluster_id, host_id, + os_config=None, + package_config=None, + raw_data=None): + data = {} + if raw_data: + data = raw_data + + else: + if os_config: + data['os_config'] = os_config + + if package_config: + data['package_config'] = package_config + + return self._patch('/clusters/%s/hosts/%s/config' % + (cluster_id, host_id), data=data) + + def patch_clusterhost_config(self, clusterhost_id, os_config=None, + package_config=None, raw_data=None): + data = {} + if raw_data: + data = raw_data + + else: + if os_config: + data['os_config'] = os_config + + if package_config: + data['package_config'] = package_config + + return self._patch('/clusterhosts/%s' % clusterhost_id, data=data) + + def delete_cluster_host_config(self, cluster_id, host_id): + return self._delete('/clusters/%s/hosts/%s/config' % + (cluster_id, host_id)) + + def delete_clusterhost_config(self, clusterhost_id): + return self._delete('/clusterhosts/%s/config' % clusterhost_id) + + def get_cluster_host_state(self, cluster_id, host_id): + return self._get('/clusters/%s/hosts/%s/state' % + (cluster_id, host_id)) + + def update_cluster_host(self, cluster_id, host_id, + roles=None, raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if roles: + data['roles'] = roles + + return self._put('/clusters/%s/hosts/%s' % + (cluster_id, host_id), data=data) + + def update_clusterhost(self, clusterhost_id, + roles=None, raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if roles: + data['roles'] = roles + + return self._put('/clusterhosts/%s' % clusterhost_id, data=data) + + def patch_cluster_host(self, cluster_id, host_id, + roles=None, raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if roles: + data['roles'] = roles + + return self._patch('/clusters/%s/hosts/%s' % + (cluster_id, host_id), data=data) + + def patch_clusterhost(self, clusterhost_id, + roles=None, raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if roles: + data['roles'] = roles + + return self._patch('/clusterhosts/%s' % clusterhost_id, data=data) + + def get_clusterhost_state(self, clusterhost_id): + return self._get('/clusterhosts/%s/state' % clusterhost_id) + + def update_cluster_host_state(self, cluster_id, host_id, state=None, + percentage=None, message=None, + raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if state: + data['state'] = state + + if percentage: + data['percentage'] = percentage + + if message: + data['message'] = message + + return self._put('/clusters/%s/hosts/%s/state' % (cluster_id, host_id), + data=data) + + def update_clusterhost_state(self, clusterhost_id, state=None, + percentage=None, message=None, + raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if state: + data['state'] = state + + if percentage: + data['percentage'] = percentage + + if message: + data['message'] = message + + return self._put('/clusterhosts/%s/state' % clusterhost_id, data=data) + + def list_hosts(self, name=None, os_name=None, owner=None, mac=None): + data = {} + if name: + data['name'] = name + + if os_name: + data['os_name'] = os_name + + if owner: + data['owner'] = owner + + if mac: + data['mac'] = mac + + return self._get('/hosts', data=data) + + def get_host(self, host_id): + return self._get('/hosts/%s' % host_id) + + def list_machines_or_hosts(self, mac=None, tag=None, + location=None, os_name=None, + os_id=None): + data = {} + if mac: + data['mac'] = mac + + if tag: + data['tag'] = tag + + if location: + data['location'] = location + + if os_name: + data['os_name'] = os_name + + if os_id: + data['os_id'] = os_id + + return self._get('/machines-hosts', data=data) + + def get_machine_or_host(self, host_id): + return self._get('/machines-hosts/%s' % host_id) + + def update_host(self, host_id, name=None, + reinstall_os=None, raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if name: + data['name'] = name + + if reinstall_os: + data['reinstall_os'] = reinstall_os + + return self._put('/hosts/%s' % host_id, data=data) + + def delete_host(self, host_id): + return self._delete('/hosts/%s' % host_id) + + def get_host_clusters(self, host_id): + return self._get('/hosts/%s/clusters' % host_id) + + def get_host_config(self, host_id): + return self._get('/hosts/%s/config' % host_id) + + def update_host_config(self, host_id, os_config, raw_data=None): + data = {} + data['os_config'] = os_config + if raw_data: + data.update(raw_data) + + return self._put('/hosts/%s/config' % host_id, data=data) + + def patch_host_config(self, host_id, os_config, raw_data=None): + data = {} + data['os_config'] = os_config + if raw_data: + data.update(raw_data) + + return self._patch('/hosts/%s/config' % host_id, data=data) + + def delete_host_config(self, host_id): + return self._delete('/hosts/%s/config' % host_id) + + def list_host_networks(self, host_id, interface=None, ip=None, + subnet=None, is_mgmt=None, is_promiscuous=None): + data = {} + if interface: + data['interface'] = interface + + if ip: + data['ip'] = ip + + if subnet: + data['subnet'] = subnet + + if is_mgmt: + data['is_mgmt'] = is_mgmt + + if is_promiscuous: + data['is_promiscuous'] = is_promiscuous + + return self._get('/hosts/%s/networks' % host_id, data=data) + + def list_all_host_networks(self, interface=None, ip=None, subnet=None, + is_mgmt=None, is_promiscuous=None): + data = {} + if interface: + data['interface'] = interface + + if ip: + data['ip'] = ip + + if subnet: + data['subnet'] = subnet + + if is_mgmt: + data['is_mgmt'] = is_mgmt + + if is_promiscuous: + data['is_promiscuous'] = is_promiscuous + + return self._get('/host-networks', data=data) + + def get_host_network(self, host_id, host_network_id): + return self._get('/hosts/%s/networks/%s' % + (host_id, host_network_id)) + + def get_network_for_all_hosts(self, host_network_id): + return self._get('/host-networks/%s' % host_network_id) + + def add_host_network(self, host_id, interface, ip, subnet_id, + is_mgmt=None, is_promiscuous=None, + raw_data=None): + data = {} + data['interface'] = interface + data['ip'] = ip + data['subnet_id'] = subnet_id + if raw_data: + data.update(raw_data) + else: + if is_mgmt: + data['is_mgmt'] = is_mgmt + + if is_promiscuous: + data['is_promiscuous'] = is_promiscuous + + return self._post('/hosts/%s/networks' % host_id, data=data) + + def update_host_network(self, host_id, host_network_id, + ip=None, subnet_id=None, subnet=None, + is_mgmt=None, is_promiscuous=None, + raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if ip: + data['ip'] = ip + + if subnet_id: + data['subnet_id'] = subnet_id + + if subnet: + data['subnet'] = subnet + + if is_mgmt: + data['is_mgmt'] = is_mgmt + + if is_promiscuous: + data['is_promiscuous'] = is_promiscuous + + return self._put('/hosts/%s/networks/%s' % + (host_id, host_network_id), data=data) + + def update_hostnetwork(self, host_network_id, ip=None, + subnet_id=None, subnet=None, + is_mgmt=None, is_promiscuous=None, + raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if ip: + data['ip'] = ip + + if subnet_id: + data['subnet_id'] = subnet_id + + if subnet: + data['subnet'] = subnet + + if is_mgmt: + data['is_mgmt'] = is_mgmt + + if is_promiscuous: + data['is_promiscuous'] = is_promiscuous + + return self._put('/host-networks/%s' % host_network_id, + data=data) + + def delete_host_network(self, host_id, host_network_id): + return self._delete('/hosts/%s/networks/%s', + (host_id, host_network_id)) + + def delete_hostnetwork(self, host_network_id): + return self._delete('/host-networks/%s' % host_network_id) + + def get_host_state(self, host_id): + return self._get('/hosts/%s/state' % host_id) + + def update_host_state(self, host_id, state=None, + percentage=None, message=None, + raw_data=None): + data = {} + if raw_data: + data = raw_data + else: + if state: + data['state'] = state + + if percentage: + data['percentage'] = percentage + + if message: + data['message'] = message + + return self._put('/hosts/%s/state' % host_id, date=data) + + def poweron_host(self, host_id): + data = {} + data['poweron'] = True + + return self._post('/hosts/%s/action' % host_id, data=data) + + def poweroff_host(self, host_id): + data = {} + data['poweroff'] = True + + return self._post('/hosts/%s/action' % host_id, data=data) + + def reset_host(self, host_id): + data = {} + data['reset'] = True + + return self._post('/hosts/%s/action' % host_id, data=data) + + def clusterhost_ready(self, clusterhost_name): + data = {} + data['ready'] = True + + return self._post('/clusterhosts/%s/state_internal' % + clusterhost_name, data=data) diff --git a/compass-tasks-base/apiclient/v1/__init__.py b/compass-tasks-base/apiclient/v1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/compass-tasks-base/apiclient/v1/example.py b/compass-tasks-base/apiclient/v1/example.py new file mode 100755 index 0000000..6f7a7f7 --- /dev/null +++ b/compass-tasks-base/apiclient/v1/example.py @@ -0,0 +1,305 @@ +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example code to deploy a cluster by compass client api.""" +import os +import re +import requests +import sys +import time + +from compass.apiclient.restful import Client + + +COMPASS_SERVER_URL = 'http://127.0.0.1/api' +SWITCH_IP = '10.145.81.220' +SWITCH_SNMP_VERSION = 'v2c' +SWITCH_SNMP_COMMUNITY = 'public' +# MACHINES_TO_ADD = ['00:11:20:30:40:01'] +CLUSTER_NAME = 'cluster2' +HOST_NAME_PREFIX = 'host' +SERVER_USERNAME = 'root' +SERVER_PASSWORD = 'root' +SERVICE_USERNAME = 'service' +SERVICE_PASSWORD = 'service' +CONSOLE_USERNAME = 'console' +CONSOLE_PASSWORD = 'console' +HA_VIP = '' +# NAMESERVERS = '192.168.10.6' +SEARCH_PATH = 'ods.com' +# GATEWAY = '192.168.10.6' +# PROXY = 'http://192.168.10.6:3128' +# NTP_SERVER = '192.168.10.6' +MANAGEMENT_IP_START = '192.168.10.130' +MANAGEMENT_IP_END = '192.168.10.254' +MANAGEMENT_IP_GATEWAY = '192.168.10.1' +MANAGEMENT_NETMASK = '255.255.255.0' +MANAGEMENT_NIC = 'eth0' +MANAGEMENT_PROMISC = 0 +TENANT_IP_START = '192.168.10.100' +TENANT_IP_END = '192.168.10.255' +TENANT_IP_GATEWAY = '192.168.10.1' +TENANT_NETMASK = '255.255.255.0' +TENANT_NIC = 'eth0' +TENANT_PROMISC = 0 +PUBLIC_IP_START = '12.234.32.100' +PUBLIC_IP_END = '12.234.32.255' +PUBLIC_IP_GATEWAY = '12.234.32.1' +PUBLIC_NETMASK = '255.255.255.0' +PUBLIC_NIC = 'eth1' +PUBLIC_PROMISC = 1 +STORAGE_IP_START = '172.16.100.100' +STORAGE_IP_END = '172.16.100.255' +STORAGE_NETMASK = '255.255.255.0' +STORAGE_IP_GATEWAY = '172.16.100.1' +STORAGE_NIC = 'eth0' +STORAGE_PROMISC = 0 +HOME_PERCENTAGE = 5 +TMP_PERCENTAGE = 5 +VAR_PERCENTAGE = 10 +# ROLES_LIST = [['os-dashboard']] + +PRESET_VALUES = { + 'NAMESERVERS': '192.168.10.1', + 'NTP_SERVER': '192.168.10.1', + 'GATEWAY': '192.168.10.1', + 'PROXY': 'http://192.168.10.1:3128', + 'ROLES_LIST': 'os-dashboard', + 'MACHINES_TO_ADD': '00:11:20:30:40:01', + 'BUILD_TIMEOUT': 60 +} +for v in PRESET_VALUES: + if v in os.environ.keys(): + PRESET_VALUES[v] = os.environ.get(v) + print (v + PRESET_VALUES[v] + " is set by env variables") + else: + print (PRESET_VALUES[v]) + +# get apiclient object. +client = Client(COMPASS_SERVER_URL) + + +# get all switches. +status, resp = client.get_switches() +print 'get all switches status: %s resp: %s' % (status, resp) + +# add a switch. +status, resp = client.add_switch( + SWITCH_IP, version=SWITCH_SNMP_VERSION, + community=SWITCH_SNMP_COMMUNITY) + +print 'add a switch status: %s resp: %s' % (status, resp) + +if status < 400: + switch = resp['switch'] +else: + status, resp = client.get_switches() + print 'get all switches status: %s resp: %s' % (status, resp) + switch = None + for switch in resp['switches']: + if switch['ip'] == SWITCH_IP: + break + +switch_id = switch['id'] +switch_ip = switch['ip'] + + +# if the switch is not in under_monitoring, wait for the poll switch task +# update the swich information and change the switch state. +while switch['state'] != 'under_monitoring': + print 'waiting for the switch into under_monitoring' + status, resp = client.get_switch(switch_id) + print 'get switch %s status: %s, resp: %s' % (switch_id, status, resp) + switch = resp['switch'] + time.sleep(10) + + +# get machines connected to the switch. +status, resp = client.get_machines(switch_id=switch_id) +print 'get all machines under switch %s status: %s, resp: %s' % ( + switch_id, status, resp) +machines = {} +MACHINES_TO_ADD = PRESET_VALUES['MACHINES_TO_ADD'].split() +for machine in resp['machines']: + mac = machine['mac'] + if mac in MACHINES_TO_ADD: + machines[machine['id']] = mac + +print 'machine to add: %s' % machines + +if set(machines.values()) != set(MACHINES_TO_ADD): + print 'only found macs %s while expected are %s' % ( + machines.values(), MACHINES_TO_ADD) + sys.exit(1) + + +# get adapters. +status, resp = client.get_adapters() +print 'get all adapters status: %s, resp: %s' % (status, resp) +adapter_ids = [] +for adapter in resp['adapters']: + adapter_ids.append(adapter['id']) + +adapter_id = adapter_ids[0] +print 'adpater for deploying a cluster: %s' % adapter_id + + +# add a cluster. +status, resp = client.add_cluster( + cluster_name=CLUSTER_NAME, adapter_id=adapter_id) +print 'add cluster %s status: %s, resp: %s' % (CLUSTER_NAME, status, resp) +cluster = resp['cluster'] +cluster_id = cluster['id'] + +# add hosts to the cluster. +status, resp = client.add_hosts( + cluster_id=cluster_id, + machine_ids=machines.keys()) +print 'add hosts to cluster %s status: %s, resp: %s' % ( + cluster_id, status, resp) +host_ids = [] +for host in resp['cluster_hosts']: + host_ids.append(host['id']) + +print 'added hosts: %s' % host_ids + + +# set cluster security +status, resp = client.set_security( + cluster_id, server_username=SERVER_USERNAME, + server_password=SERVER_PASSWORD, + service_username=SERVICE_USERNAME, + service_password=SERVICE_PASSWORD, + console_username=CONSOLE_USERNAME, + console_password=CONSOLE_PASSWORD) +print 'set security config to cluster %s status: %s, resp: %s' % ( + cluster_id, status, resp) + + +# set cluster networking +status, resp = client.set_networking( + cluster_id, + nameservers=PRESET_VALUES["NAMESERVERS"], + search_path=SEARCH_PATH, + gateway=PRESET_VALUES["GATEWAY"], + proxy=PRESET_VALUES["PROXY"], + ntp_server=PRESET_VALUES["NTP_SERVER"], + ha_vip=HA_VIP, + management_ip_start=MANAGEMENT_IP_START, + management_ip_end=MANAGEMENT_IP_END, + management_netmask=MANAGEMENT_NETMASK, + management_nic=MANAGEMENT_NIC, + management_gateway=MANAGEMENT_IP_GATEWAY, + management_promisc=MANAGEMENT_PROMISC, + tenant_ip_start=TENANT_IP_START, + tenant_ip_end=TENANT_IP_END, + tenant_netmask=TENANT_NETMASK, + tenant_nic=TENANT_NIC, + tenant_gateway=TENANT_IP_GATEWAY, + tenant_promisc=TENANT_PROMISC, + public_ip_start=PUBLIC_IP_START, + public_ip_end=PUBLIC_IP_END, + public_netmask=PUBLIC_NETMASK, + public_nic=PUBLIC_NIC, + public_gateway=PUBLIC_IP_GATEWAY, + public_promisc=PUBLIC_PROMISC, + storage_ip_start=STORAGE_IP_START, + storage_ip_end=STORAGE_IP_END, + storage_netmask=STORAGE_NETMASK, + storage_nic=STORAGE_NIC, + storage_gateway=STORAGE_IP_GATEWAY, + storage_promisc=STORAGE_PROMISC) +print 'set networking config to cluster %s status: %s, resp: %s' % ( + cluster_id, status, resp) + + +# set partiton of each host in cluster +status, resp = client.set_partition( + cluster_id, + home_percentage=HOME_PERCENTAGE, + tmp_percentage=TMP_PERCENTAGE, + var_percentage=VAR_PERCENTAGE) +print 'set partition config to cluster %s status: %s, resp: %s' % ( + cluster_id, status, resp) + + +# set each host config in cluster. +ROLES_LIST = [PRESET_VALUES['ROLES_LIST'].split()] +for host_id in host_ids: + if ROLES_LIST: + roles = ROLES_LIST.pop(0) + else: + roles = [] + status, resp = client.update_host_config( + host_id, hostname='%s%s' % (HOST_NAME_PREFIX, host_id), + roles=roles) + print 'set roles to host %s status: %s, resp: %s' % ( + host_id, status, resp) + + +# deploy cluster. +status, resp = client.deploy_hosts(cluster_id) +print 'deploy cluster %s status: %s, resp: %s' % (cluster_id, status, resp) + + +# get intalling progress. +BUILD_TIMEOUT = float(PRESET_VALUES['BUILD_TIMEOUT']) +timeout = time.time() + BUILD_TIMEOUT * 60 +while True: + status, resp = client.get_cluster_installing_progress(cluster_id) + print 'get cluster %s installing progress status: %s, resp: %s' % ( + cluster_id, status, resp) + progress = resp['progress'] + if ( + progress['state'] not in ['UNINITIALIZED', 'INSTALLING'] or + progress['percentage'] >= 1.0 + ): + break + if ( + time.time() > timeout + ): + raise Exception("Timeout! The system is not ready in time.") + + for host_id in host_ids: + status, resp = client.get_host_installing_progress(host_id) + print 'get host %s installing progress status: %s, resp: %s' % ( + host_id, status, resp) + + time.sleep(60) + + +status, resp = client.get_dashboard_links(cluster_id) +print 'get cluster %s dashboardlinks status: %s, resp: %s' % ( + cluster_id, status, resp) +dashboardlinks = resp['dashboardlinks'] +if not dashboardlinks.keys(): + raise Exception("Dashboard link is not found!") +for x in dashboardlinks.keys(): + if x in ("os-dashboard", "os-controller"): + dashboardurl = dashboardlinks.get(x) + if dashboardurl is None: + raise Exception("No dashboard link is found") + r = requests.get(dashboardurl, verify=False) + r.raise_for_status() + match = re.search( + r'(?m)(http://\d+\.\d+\.\d+\.\d+:5000/v2\.0)', r.text) + if match: + print 'dashboard login page can be downloaded' + break + print ( + 'dashboard login page failed to be downloaded\n' + 'the context is:\n%s\n') % r.text + raise Exception("os-dashboard is not properly installed!") diff --git a/compass-tasks-base/apiclient/v1/restful.py b/compass-tasks-base/apiclient/v1/restful.py new file mode 100644 index 0000000..3fb235c --- /dev/null +++ b/compass-tasks-base/apiclient/v1/restful.py @@ -0,0 +1,655 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compass api client library. + + .. moduleauthor:: Xiaodong Wang +""" +import json +import logging +import requests + + +class Client(object): + """wrapper for compass restful api. + + .. note:: + Every api client method returns (status as int, resp as dict). + If the api succeeds, the status is 2xx, the resp includes + {'status': 'OK'} and other keys depend on method. + If the api fails, the status is 4xx, the resp includes { + 'status': '...', 'message': '...'} + """ + + def __init__(self, url, headers=None, proxies=None, stream=None): + """Restful api client initialization. + + :param url: url to the compass web service. + :type url: str. + :param headers: http header sent in each restful request. + :type headers: dict of header name (str) to heade value (str). + :param proxies: the proxy address for each protocol. + :type proxies: dict of protocol (str) to proxy url (str). + :param stream: wether the restful response should be streamed. + :type stream: bool. + """ + self.url_ = url + self.session_ = requests.Session() + if headers: + self.session_.headers = headers + + if proxies is not None: + self.session_.proxies = proxies + + if stream is not None: + self.session_.stream = stream + + def __del__(self): + self.session_.close() + + @classmethod + def _get_response(cls, resp): + """decapsulate the resp to status code and python formatted data.""" + resp_obj = {} + try: + resp_obj = resp.json() + except Exception as error: + logging.error('failed to load object from %s: %s', + resp.url, resp.content) + logging.exception(error) + resp_obj['status'] = 'Json Parsing Failure' + resp_obj['message'] = resp.content + + return resp.status_code, resp_obj + + def _get(self, relative_url, params=None): + """encapsulate get method.""" + url = '%s%s' % (self.url_, relative_url) + if params: + resp = self.session_.get(url, params=params) + else: + resp = self.session_.get(url) + + return self._get_response(resp) + + def _post(self, relative_url, data=None): + """encapsulate post method.""" + url = '%s%s' % (self.url_, relative_url) + if data: + resp = self.session_.post(url, json.dumps(data)) + else: + resp = self.session_.post(url) + + return self._get_response(resp) + + def _put(self, relative_url, data=None): + """encapsulate put method.""" + url = '%s%s' % (self.url_, relative_url) + if data: + resp = self.session_.put(url, json.dumps(data)) + else: + resp = self.session_.put(url) + + return self._get_response(resp) + + def _delete(self, relative_url): + """encapsulate delete method.""" + url = '%s%s' % (self.url_, relative_url) + return self._get_response(self.session_.delete(url)) + + def get_switches(self, switch_ips=None, switch_networks=None, limit=None): + """List details for switches. + + .. note:: + The switches can be filtered by switch_ips, siwtch_networks and + limit. These params can be None or missing. If the param is None + or missing, that filter will be ignored. + + :param switch_ips: Filter switch(es) with IP(s). + :type switch_ips: list of str. Each is as 'xxx.xxx.xxx.xxx'. + :param switch_networks: Filter switche(es) with network(s). + :type switch_networks: list of str. Each is as 'xxx.xxx.xxx.xxx/xx'. + :param limit: int, The maximum number of switches to return. + :type limit: int. 0 means unlimited. + """ + params = {} + if switch_ips: + params['switchIp'] = switch_ips + + if switch_networks: + params['switchIpNetwork'] = switch_networks + + if limit: + params['limit'] = limit + return self._get('/switches', params=params) + + def get_switch(self, switch_id): + """Lists details for a specified switch. + + :param switch_id: switch id. + :type switch_id: int. + """ + return self._get('/switches/%s' % switch_id) + + def add_switch(self, switch_ip, version=None, community=None, + username=None, password=None, raw_data=None): + """Create a switch with specified details. + + .. note:: + It will trigger switch polling if successful. During + the polling, MAC address of the devices connected to the + switch will be learned by SNMP or SSH. + + :param switch_ip: the switch IP address. + :type switch_ip: str, as xxx.xxx.xxx.xxx. + :param version: SNMP version when using SNMP to poll switch. + :type version: str, one in ['v1', 'v2c', 'v3'] + :param community: SNMP community when using SNMP to poll switch. + :type community: str, usually 'public'. + :param username: SSH username when using SSH to poll switch. + :type username: str. + :param password: SSH password when using SSH to poll switch. + :type password: str. + """ + data = {} + if raw_data: + data = raw_data + else: + data['switch'] = {} + data['switch']['ip'] = switch_ip + data['switch']['credential'] = {} + if version: + data['switch']['credential']['version'] = version + + if community: + data['switch']['credential']['community'] = community + + if username: + data['switch']['credential']['username'] = username + + if password: + data['switch']['credential']['password'] = password + + return self._post('/switches', data=data) + + def update_switch(self, switch_id, ip_addr=None, + version=None, community=None, + username=None, password=None, + raw_data=None): + """Updates a switch with specified details. + + .. note:: + It will trigger switch polling if successful. During + the polling, MAC address of the devices connected to the + switch will be learned by SNMP or SSH. + + :param switch_id: switch id + :type switch_id: int. + :param ip_addr: the switch ip address. + :type ip_addr: str, as 'xxx.xxx.xxx.xxx' format. + :param version: SNMP version when using SNMP to poll switch. + :type version: str, one in ['v1', 'v2c', 'v3']. + :param community: SNMP community when using SNMP to poll switch. + :type community: str, usually be 'public'. + :param username: username when using SSH to poll switch. + :type username: str. + :param password: password when using SSH to poll switch. + """ + data = {} + if raw_data: + data = raw_data + else: + data['switch'] = {} + if ip_addr: + data['switch']['ip'] = ip_addr + + data['switch']['credential'] = {} + if version: + data['switch']['credential']['version'] = version + + if community: + data['switch']['credential']['community'] = community + + if username: + data['switch']['credential']['username'] = username + + if password: + data['switch']['credential']['password'] = password + + return self._put('/switches/%s' % switch_id, data=data) + + def delete_switch(self, switch_id): + """Not implemented in api.""" + return self._delete('/switches/%s' % switch_id) + + def get_machines(self, switch_id=None, vlan_id=None, + port=None, limit=None): + """Get the details of machines. + + .. note:: + The machines can be filtered by switch_id, vlan_id, port + and limit. These params can be None or missing. If the param + is None or missing, the filter will be ignored. + + :param switch_id: Return machine(s) connected to the switch. + :type switch_id: int. + :param vlan_id: Return machine(s) belonging to the vlan. + :type vlan_id: int. + :param port: Return machine(s) connect to the port. + :type port: int. + :param limit: the maximum number of machines will be returned. + :type limit: int. 0 means no limit. + """ + params = {} + if switch_id: + params['switchId'] = switch_id + + if vlan_id: + params['vlanId'] = vlan_id + + if port: + params['port'] = port + + if limit: + params['limit'] = limit + + return self._get('/machines', params=params) + + def get_machine(self, machine_id): + """Lists the details for a specified machine. + + :param machine_id: Return machine with the id. + :type machine_id: int. + """ + return self._get('/machines/%s' % machine_id) + + def get_clusters(self): + """Lists the details for all clusters.""" + return self._get('/clusters') + + def get_cluster(self, cluster_id): + """Lists the details of the specified cluster. + + :param cluster_id: cluster id. + :type cluster_id: int. + """ + return self._get('/clusters/%d' % cluster_id) + + def add_cluster(self, cluster_name, adapter_id, raw_data=None): + """Creates a cluster by specified name and given adapter id. + + :param cluster_name: cluster name. + :type cluster_name: str. + :param adapter_id: adapter id. + :type adapter_id: int. + """ + data = {} + if raw_data: + data = raw_data + else: + data['cluster'] = {} + data['cluster']['name'] = cluster_name + data['cluster']['adapter_id'] = adapter_id + return self._post('/clusters', data=data) + + def add_hosts(self, cluster_id, machine_ids, raw_data=None): + """add the specified machine(s) as the host(s) to the cluster. + + :param cluster_id: cluster id. + :type cluster_id: int. + :param machine_ids: machine ids to add to cluster. + :type machine_ids: list of int, each is the id of one machine. + """ + data = {} + if raw_data: + data = raw_data + else: + data['addHosts'] = machine_ids + return self._post('/clusters/%d/action' % cluster_id, data=data) + + def remove_hosts(self, cluster_id, host_ids, raw_data=None): + """remove the specified host(s) from the cluster. + + :param cluster_id: cluster id. + :type cluster_id: int. + :param host_ids: host ids to remove from cluster. + :type host_ids: list of int, each is the id of one host. + """ + data = {} + if raw_data: + data = raw_data + else: + data['removeHosts'] = host_ids + return self._post('/clusters/%s/action' % cluster_id, data=data) + + def replace_hosts(self, cluster_id, machine_ids, raw_data=None): + """replace the cluster hosts with the specified machine(s). + + :param cluster_id: int, The unique identifier of the cluster. + :type cluster_id: int. + :param machine_ids: the machine ids to replace the hosts in cluster. + :type machine_ids: list of int, each is the id of one machine. + """ + data = {} + if raw_data: + data = raw_data + else: + data['replaceAllHosts'] = machine_ids + return self._post('/clusters/%s/action' % cluster_id, data=data) + + def deploy_hosts(self, cluster_id, raw_data=None): + """Deploy the cluster. + + :param cluster_id: The unique identifier of the cluster + :type cluster_id: int. + """ + data = {} + if raw_data: + data = raw_data + else: + data['deploy'] = [] + return self._post('/clusters/%d/action' % cluster_id, data=data) + + @classmethod + def parse_security(cls, kwargs): + """parse the arguments to security data.""" + data = {} + for key, value in kwargs.items(): + if '_' not in key: + continue + key_name, key_value = key.split('_', 1) + data.setdefault( + '%s_credentials' % key_name, {})[key_value] = value + + return data + + def set_security(self, cluster_id, **kwargs): + """Update the cluster security configuration. + + :param cluster_id: cluster id. + :type cluster_id: int. + :param _username: username of the security name. + :type _username: str. + :param _password: passowrd of the security name. + :type _password: str. + + .. note:: + security_name should be one of ['server', 'service', 'console']. + """ + data = {} + data['security'] = self.parse_security(kwargs) + return self._put('/clusters/%d/security' % cluster_id, data=data) + + @classmethod + def parse_networking(cls, kwargs): + """parse arguments to network data.""" + data = {} + global_keys = [ + 'nameservers', 'search_path', 'gateway', + 'proxy', 'ntp_server', 'ha_vip'] + for key, value in kwargs.items(): + if key in global_keys: + data.setdefault('global', {})[key] = value + else: + if '_' not in key: + continue + + key_name, key_value = key.split('_', 1) + data.setdefault( + 'interfaces', {} + ).setdefault( + key_name, {} + )[key_value] = value + + return data + + def set_networking(self, cluster_id, **kwargs): + """Update the cluster network configuration. + + :param cluster_id: cluster id. + :type cluster_id: int. + :param nameservers: comma seperated nameserver ip address. + :type nameservers: str. + :param search_path: comma seperated dns name search path. + :type search_path: str. + :param gateway: gateway ip address for routing to outside. + :type gateway: str. + :param proxy: proxy url for downloading packages. + :type proxy: str. + :param ntp_server: ntp server ip address to sync timestamp. + :type ntp_server: str. + :param ha_vip: ha vip address to run ha proxy. + :type ha_vip: str. + :param _ip_start: start ip address to host's interface. + :type _ip_start: str. + :param _ip_end: end ip address to host's interface. + :type _ip_end: str. + :param _netmask: netmask to host's interface. + :type _netmask: str. + :param _nic: host physical interface name. + :type _nic: str. + :param _promisc: if the interface in promiscous mode. + :type _promisc: int, 0 or 1. + + .. note:: + interface should be one of ['management', 'tenant', + 'public', 'storage']. + """ + data = {} + data['networking'] = self.parse_networking(kwargs) + return self._put('/clusters/%d/networking' % cluster_id, data=data) + + @classmethod + def parse_partition(cls, kwargs): + """parse arguments to partition data.""" + data = {} + for key, value in kwargs.items(): + if key.endswith('_percentage'): + key_name = key[:-len('_percentage')] + data[key_name] = '%s%%' % value + elif key.endswitch('_mbytes'): + key_name = key[:-len('_mbytes')] + data[key_name] = str(value) + + return ';'.join([ + '/%s %s' % (key, value) for key, value in data.items() + ]) + + def set_partition(self, cluster_id, **kwargs): + """Update the cluster partition configuration. + + :param cluster_id: cluster id. + :type cluster_id: int. + :param _percentage: the partiton percentage. + :type _percentage: float between 0 to 100. + :param _mbytes: the partition mbytes. + :type _mbytes: int. + + .. note:: + partition should be one of ['home', 'var', 'tmp']. + """ + data = {} + data['partition'] = self.parse_partition(kwargs) + return self._put('/clusters/%s/partition' % cluster_id, data=data) + + def get_hosts(self, hostname=None, clustername=None): + """Lists the details of hosts. + + .. note:: + The hosts can be filtered by hostname, clustername. + These params can be None or missing. If the param + is None or missing, the filter will be ignored. + + :param hostname: The name of a host. + :type hostname: str. + :param clustername: The name of a cluster. + :type clustername: str. + """ + params = {} + if hostname: + params['hostname'] = hostname + + if clustername: + params['clustername'] = clustername + + return self._get('/clusterhosts', params=params) + + def get_host(self, host_id): + """Lists the details for the specified host. + + :param host_id: host id. + :type host_id: int. + """ + return self._get('/clusterhosts/%s' % host_id) + + def get_host_config(self, host_id): + """Lists the details of the config for the specified host. + + :param host_id: host id. + :type host_id: int. + """ + return self._get('/clusterhosts/%s/config' % host_id) + + def update_host_config(self, host_id, hostname=None, + roles=None, raw_data=None, **kwargs): + """Updates config for the host. + + :param host_id: host id. + :type host_id: int. + :param hostname: host name. + :type hostname: str. + :param security__username: username of the security name. + :type security__username: str. + :param security__password: passowrd of the security name. + :type security__password: str. + :param networking_nameservers: comma seperated nameserver ip address. + :type networking_nameservers: str. + :param networking_search_path: comma seperated dns name search path. + :type networking_search_path: str. + :param networking_gateway: gateway ip address for routing to outside. + :type networking_gateway: str. + :param networking_proxy: proxy url for downloading packages. + :type networking_proxy: str. + :param networking_ntp_server: ntp server ip address to sync timestamp. + :type networking_ntp_server: str. + :param networking__ip: ip address to host interface. + :type networking__ip: str. + :param networking__netmask: netmask to host's interface. + :type networking__netmask: str. + :param networking__nic: host physical interface name. + :type networking__nic: str. + :param networking__promisc: if the interface is promiscous. + :type networking__promisc: int, 0 or 1. + :param partition__percentage: the partiton percentage. + :type partition__percentage: float between 0 to 100. + :param partition__mbytes: the partition mbytes. + :type partition__mbytes: int. + :param roles: host assigned roles in the cluster. + :type roles: list of str. + """ + data = {} + if raw_data: + data = raw_data + else: + if hostname: + data['hostname'] = hostname + + sub_kwargs = {} + for key, value in kwargs.items(): + key_name, key_value = key.split('_', 1) + sub_kwargs.setdefault(key_name, {})[key_value] = value + + if 'security' in sub_kwargs: + data['security'] = self.parse_security(sub_kwargs['security']) + + if 'networking' in sub_kwargs: + data['networking'] = self.parse_networking( + sub_kwargs['networking']) + if 'partition' in sub_kwargs: + data['partition'] = self.parse_partition( + sub_kwargs['partition']) + + if roles: + data['roles'] = roles + + return self._put('/clusterhosts/%s/config' % host_id, data) + + def delete_from_host_config(self, host_id, delete_key): + """Deletes one key in config for the host. + + :param host_id: host id. + :type host_id: int. + :param delete_key: the key in host config to be deleted. + :type delete_key: str. + """ + return self._delete('/clusterhosts/%s/config/%s' % ( + host_id, delete_key)) + + def get_adapters(self, name=None): + """Lists details of adapters. + + .. note:: + the adapter can be filtered by name of name is given and not None. + + :param name: adapter name. + :type name: str. + """ + params = {} + if name: + params['name'] = name + + return self._get('/adapters', params=params) + + def get_adapter(self, adapter_id): + """Lists details for the specified adapter. + + :param adapter_id: adapter id. + :type adapter_id: int. + """ + return self._get('/adapters/%s' % adapter_id) + + def get_adapter_roles(self, adapter_id): + """Lists roles to assign to hosts for the specified adapter. + + :param adapter_id: adapter id. + :type adapter_id: int. + """ + return self._get('/adapters/%s/roles' % adapter_id) + + def get_host_installing_progress(self, host_id): + """Lists progress details for the specified host. + + :param host_id: host id. + :type host_id: int. + """ + return self._get('/clusterhosts/%s/progress' % host_id) + + def get_cluster_installing_progress(self, cluster_id): + """Lists progress details for the specified cluster. + + :param cluster_id: cluster id. + :param cluster_id: int. + """ + + return self._get('/clusters/%s/progress' % cluster_id) + + def get_dashboard_links(self, cluster_id): + """Lists links for dashboards of deployed cluster. + + :param cluster_id: cluster id. + :type cluster_id: int. + """ + params = {} + params['cluster_id'] = cluster_id + return self._get('/dashboardlinks', params) diff --git a/compass-tasks-base/bin/switch_virtualenv.py b/compass-tasks-base/bin/switch_virtualenv.py new file mode 100644 index 0000000..cd50ea0 --- /dev/null +++ b/compass-tasks-base/bin/switch_virtualenv.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""utility switch to virtual env.""" +import os +import site +import sys + + +virtual_env = '/root/.virtualenvs/compass-core' +activate_this = '%s/bin/activate_this.py' % virtual_env +execfile(activate_this, dict(__file__=activate_this)) +site.addsitedir('%s/lib/python2.7/site-packages' % virtual_env) +if virtual_env not in sys.path: + sys.path.append(virtual_env) +os.environ['PYTHON_EGG_CACHE'] = '/tmp/.egg' diff --git a/compass-tasks-base/build.sh b/compass-tasks-base/build.sh new file mode 100755 index 0000000..9e76d25 --- /dev/null +++ b/compass-tasks-base/build.sh @@ -0,0 +1,58 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016-2017 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -x +COMPASS_DIR=${BASH_SOURCE[0]%/*} + +COMPASS_MODULE=(actions apiclient tasks utils deployment db hdsdiscovery log_analyzor) + +rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm +sed -i 's/^mirrorlist=https/mirrorlist=http/g' /etc/yum.repos.d/epel.repo +yum update -y + +yum --nogpgcheck install -y python python-devel git amqp python-pip libffi-devel openssl-devel gcc python-setuptools MySQL-python supervisor redis sshpass python-keyczar vim ansible-2.2.1.0 libyaml-devel make + +mkdir -p $COMPASS_DIR/compass +for((i=0; i<${#COMPASS_MODULE[@]}; i++)) +do + mv $COMPASS_DIR/${COMPASS_MODULE[i]} $COMPASS_DIR/compass/ +done +touch $COMPASS_DIR/compass/__init__.py + +mkdir -p /etc/compass/ +mkdir -p /etc/compass/machine_list +mkdir -p /etc/compass/switch_list +mkdir -p /var/log/compass +mkdir -p /opt/ansible_callbacks +mkdir -p /opt/compass +mkdir -p /root/.ssh; +echo "UserKnownHostsFile /dev/null" >> /root/.ssh/config; +echo "StrictHostKeyChecking no" >> /root/.ssh/config + +easy_install --upgrade pip +pip install --upgrade pip +pip install --upgrade setuptools +pip install --upgrade Flask +pip install --upgrade virtualenvwrapper + +source `which virtualenvwrapper.sh` +mkvirtualenv --system-site-packages compass-core +workon compass-core +cd $COMPASS_DIR +pip install -U -r requirements.txt +python setup.py install +cp -rf ./bin /opt/compass/bin +cp supervisord.conf /etc/supervisord.conf +cp start.sh /usr/local/bin/start.sh +ln -s `which celery` /usr/bin/celery +cd - + +yum clean all + +set +x diff --git a/compass-tasks-base/db/__init__.py b/compass-tasks-base/db/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/db/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/db/api/__init__.py b/compass-tasks-base/db/api/__init__.py new file mode 100644 index 0000000..5e42ae9 --- /dev/null +++ b/compass-tasks-base/db/api/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/db/api/adapter.py b/compass-tasks-base/db/api/adapter.py new file mode 100644 index 0000000..c3ad48d --- /dev/null +++ b/compass-tasks-base/db/api/adapter.py @@ -0,0 +1,313 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Adapter related database operations.""" +import logging +import re + +from compass.db.api import database +from compass.db.api import utils +from compass.db import exception +from compass.db import models + +from compass.utils import setting_wrapper as setting +from compass.utils import util + + +OSES = None +OS_INSTALLERS = None +PACKAGE_INSTALLERS = None +ADAPTERS = None +ADAPTERS_FLAVORS = None +ADAPTERS_ROLES = None + + +def _get_oses_from_configuration(): + """Get all os configs from os configuration dir. + + Example: { + : { + 'name': , + 'id': , + 'os_id': , + 'deployable': True + } + } + """ + configs = util.load_configs(setting.OS_DIR) + systems = {} + for config in configs: + logging.info('get config %s', config) + system_name = config['NAME'] + parent_name = config.get('PARENT', None) + system = { + 'name': system_name, + 'id': system_name, + 'os_id': system_name, + 'parent': parent_name, + 'parent_id': parent_name, + 'deployable': config.get('DEPLOYABLE', False) + } + systems[system_name] = system + parents = {} + for name, system in systems.items(): + parent = system.get('parent', None) + parents[name] = parent + for name, system in systems.items(): + util.recursive_merge_dict(name, systems, parents) + return systems + + +def _get_installers_from_configuration(configs): + """Get installers from configurations. + + Example: { + : { + 'alias': , + 'id': , + 'name': , + 'settings': + } + } + """ + installers = {} + for config in configs: + name = config['NAME'] + instance_name = config.get('INSTANCE_NAME', name) + installers[instance_name] = { + 'alias': instance_name, + 'id': instance_name, + 'name': name, + 'settings': config.get('SETTINGS', {}) + } + return installers + + +def _get_os_installers_from_configuration(): + """Get os installers from os installer config dir.""" + configs = util.load_configs(setting.OS_INSTALLER_DIR) + return _get_installers_from_configuration(configs) + + +def _get_package_installers_from_configuration(): + """Get package installers from package installer config dir.""" + configs = util.load_configs(setting.PACKAGE_INSTALLER_DIR) + return _get_installers_from_configuration(configs) + + +def _get_adapters_from_configuration(): + """Get adapters from adapter config dir.""" + configs = util.load_configs(setting.ADAPTER_DIR) + adapters = {} + for config in configs: + logging.info('add config %s to adapter', config) + if 'OS_INSTALLER' in config: + os_installer = OS_INSTALLERS[config['OS_INSTALLER']] + else: + os_installer = None + + if 'PACKAGE_INSTALLER' in config: + package_installer = PACKAGE_INSTALLERS[ + config['PACKAGE_INSTALLER'] + ] + else: + package_installer = None + + adapter_name = config['NAME'] + parent_name = config.get('PARENT', None) + adapter = { + 'name': adapter_name, + 'id': adapter_name, + 'parent': parent_name, + 'parent_id': parent_name, + 'display_name': config.get('DISPLAY_NAME', adapter_name), + 'os_installer': os_installer, + 'package_installer': package_installer, + 'deployable': config.get('DEPLOYABLE', False), + 'health_check_cmd': config.get('HEALTH_CHECK_COMMAND', None), + 'supported_oses': [], + 'roles': [], + 'flavors': [] + } + supported_os_patterns = [ + re.compile(supported_os_pattern) + for supported_os_pattern in config.get('SUPPORTED_OS_PATTERNS', []) + ] + for os_name, os in OSES.items(): + if not os.get('deployable', False): + continue + for supported_os_pattern in supported_os_patterns: + if supported_os_pattern.match(os_name): + adapter['supported_oses'].append(os) + break + adapters[adapter_name] = adapter + + parents = {} + for name, adapter in adapters.items(): + parent = adapter.get('parent', None) + parents[name] = parent + for name, adapter in adapters.items(): + util.recursive_merge_dict(name, adapters, parents) + return adapters + + +def _add_roles_from_configuration(): + """Get roles from roles config dir and update to adapters.""" + configs = util.load_configs(setting.ADAPTER_ROLE_DIR) + for config in configs: + logging.info( + 'add config %s to role', config + ) + adapter_name = config['ADAPTER_NAME'] + adapter = ADAPTERS[adapter_name] + adapter_roles = ADAPTERS_ROLES.setdefault(adapter_name, {}) + for role_dict in config['ROLES']: + role_name = role_dict['role'] + display_name = role_dict.get('display_name', role_name) + adapter_roles[role_name] = { + 'name': role_name, + 'id': '%s:%s' % (adapter_name, role_name), + 'adapter_id': adapter_name, + 'adapter_name': adapter_name, + 'display_name': display_name, + 'description': role_dict.get('description', display_name), + 'optional': role_dict.get('optional', False) + } + parents = {} + for name, adapter in ADAPTERS.items(): + parent = adapter.get('parent', None) + parents[name] = parent + for adapter_name, adapter_roles in ADAPTERS_ROLES.items(): + util.recursive_merge_dict(adapter_name, ADAPTERS_ROLES, parents) + for adapter_name, adapter_roles in ADAPTERS_ROLES.items(): + adapter = ADAPTERS[adapter_name] + adapter['roles'] = adapter_roles.values() + + +def _add_flavors_from_configuration(): + """Get flavors from flavor config dir and update to adapters.""" + configs = util.load_configs(setting.ADAPTER_FLAVOR_DIR) + for config in configs: + logging.info('add config %s to flavor', config) + adapter_name = config['ADAPTER_NAME'] + adapter = ADAPTERS[adapter_name] + adapter_flavors = ADAPTERS_FLAVORS.setdefault(adapter_name, {}) + adapter_roles = ADAPTERS_ROLES[adapter_name] + for flavor_dict in config['FLAVORS']: + flavor_name = flavor_dict['flavor'] + flavor_id = '%s:%s' % (adapter_name, flavor_name) + flavor = { + 'name': flavor_name, + 'id': flavor_id, + 'adapter_id': adapter_name, + 'adapter_name': adapter_name, + 'display_name': flavor_dict.get('display_name', flavor_name), + 'template': flavor_dict.get('template', None) + } + flavor_roles = flavor_dict.get('roles', []) + roles_in_flavor = [] + for flavor_role in flavor_roles: + if isinstance(flavor_role, basestring): + role_name = flavor_role + role_in_flavor = { + 'name': role_name, + 'flavor_id': flavor_id + } + else: + role_in_flavor = flavor_role + role_in_flavor['flavor_id'] = flavor_id + if 'role' in role_in_flavor: + role_in_flavor['name'] = role_in_flavor['role'] + del role_in_flavor['role'] + role_name = role_in_flavor['name'] + role = adapter_roles[role_name] + util.merge_dict(role_in_flavor, role, override=False) + roles_in_flavor.append(role_in_flavor) + flavor['roles'] = roles_in_flavor + adapter_flavors[flavor_name] = flavor + parents = {} + for name, adapter in ADAPTERS.items(): + parent = adapter.get('parent', None) + parents[name] = parent + for adapter_name, adapter_roles in ADAPTERS_FLAVORS.items(): + util.recursive_merge_dict(adapter_name, ADAPTERS_FLAVORS, parents) + for adapter_name, adapter_flavors in ADAPTERS_FLAVORS.items(): + adapter = ADAPTERS[adapter_name] + adapter['flavors'] = adapter_flavors.values() + + +def load_adapters_internal(force_reload=False): + """Load adapter related configurations into memory. + + If force_reload, reload all configurations even it is loaded already. + """ + global OSES + if force_reload or OSES is None: + OSES = _get_oses_from_configuration() + global OS_INSTALLERS + if force_reload or OS_INSTALLERS is None: + OS_INSTALLERS = _get_os_installers_from_configuration() + global PACKAGE_INSTALLERS + if force_reload or PACKAGE_INSTALLERS is None: + PACKAGE_INSTALLERS = _get_package_installers_from_configuration() + global ADAPTERS + if force_reload or ADAPTERS is None: + ADAPTERS = _get_adapters_from_configuration() + global ADAPTERS_ROLES + if force_reload or ADAPTERS_ROLES is None: + ADAPTERS_ROLES = {} + _add_roles_from_configuration() + global ADAPTERS_FLAVORS + if force_reload or ADAPTERS_FLAVORS is None: + ADAPTERS_FLAVORS = {} + _add_flavors_from_configuration() + + +def get_adapters_internal(force_reload=False): + """Get all deployable adapters.""" + load_adapters_internal(force_reload=force_reload) + adapter_mapping = {} + for adapter_name, adapter in ADAPTERS.items(): + if adapter.get('deployable'): + # TODO(xicheng): adapter should be filtered before + # return to caller. + adapter_mapping[adapter_name] = adapter + else: + logging.info( + 'ignore adapter %s since it is not deployable', + adapter_name + ) + return adapter_mapping + + +def get_flavors_internal(force_reload=False): + """Get all deployable flavors.""" + load_adapters_internal(force_reload=force_reload) + adapter_flavor_mapping = {} + for adapter_name, adapter_flavors in ADAPTERS_FLAVORS.items(): + adapter = ADAPTERS.get(adapter_name, {}) + for flavor_name, flavor in adapter_flavors.items(): + if adapter.get('deployable'): + # TODO(xicheng): flavor dict should be filtered before + # return to caller. + adapter_flavor_mapping.setdefault( + adapter_name, {} + )[flavor_name] = flavor + else: + logging.info( + 'ignore adapter %s since it is not deployable', + adapter_name + ) + + return adapter_flavor_mapping diff --git a/compass-tasks-base/db/api/adapter_holder.py b/compass-tasks-base/db/api/adapter_holder.py new file mode 100644 index 0000000..91c65c4 --- /dev/null +++ b/compass-tasks-base/db/api/adapter_holder.py @@ -0,0 +1,155 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Adapter related object holder.""" +import logging + +from compass.db.api import adapter as adapter_api +from compass.db.api import database +from compass.db.api import permission +from compass.db.api import user as user_api +from compass.db.api import utils +from compass.db import exception + + +SUPPORTED_FIELDS = [ + 'name', +] +RESP_FIELDS = [ + 'id', 'name', 'roles', 'flavors', + 'os_installer', 'package_installer', + 'supported_oses', 'display_name', 'health_check_cmd' +] +RESP_OS_FIELDS = [ + 'id', 'name', 'os_id' +] +RESP_ROLES_FIELDS = [ + 'id', 'name', 'display_name', 'description', 'optional' +] +RESP_FLAVORS_FIELDS = [ + 'id', 'adapter_id', 'adapter_name', 'name', 'display_name', + 'template', 'roles' +] + + +ADAPTER_MAPPING = None +FLAVOR_MAPPING = None + + +def load_adapters(force_reload=False): + global ADAPTER_MAPPING + if force_reload or ADAPTER_MAPPING is None: + logging.info('load adapters into memory') + ADAPTER_MAPPING = adapter_api.get_adapters_internal( + force_reload=force_reload + ) + + +def load_flavors(force_reload=False): + global FLAVOR_MAPPING + if force_reload or FLAVOR_MAPPING is None: + logging.info('load flavors into memory') + FLAVOR_MAPPING = {} + adapters_flavors = adapter_api.get_flavors_internal( + force_reload=force_reload + ) + for adapter_name, adapter_flavors in adapters_flavors.items(): + for flavor_name, flavor in adapter_flavors.items(): + FLAVOR_MAPPING['%s:%s' % (adapter_name, flavor_name)] = flavor + + +def _filter_adapters(adapter_config, filter_name, filter_value): + if filter_name not in adapter_config: + return False + if isinstance(filter_value, list): + return bool( + adapter_config[filter_name] in filter_value + ) + elif isinstance(filter_value, dict): + return all([ + _filter_adapters( + adapter_config[filter_name], + sub_filter_key, sub_filter_value + ) + for sub_filter_key, sub_filter_value in filter_value.items() + ]) + else: + return adapter_config[filter_name] == filter_value + + +@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_ADAPTERS +) +@utils.output_filters(name=utils.general_filter_callback) +@utils.wrap_to_dict( + RESP_FIELDS, + supported_oses=RESP_OS_FIELDS, + roles=RESP_ROLES_FIELDS, + flavors=RESP_FLAVORS_FIELDS +) +def list_adapters(user=None, session=None, **filters): + """list adapters.""" + load_adapters() + return ADAPTER_MAPPING.values() + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_ADAPTERS +) +@utils.wrap_to_dict( + RESP_FIELDS, + supported_oses=RESP_OS_FIELDS, + roles=RESP_ROLES_FIELDS, + flavors=RESP_FLAVORS_FIELDS +) +def get_adapter(adapter_id, user=None, session=None, **kwargs): + """get adapter.""" + load_adapters() + if adapter_id not in ADAPTER_MAPPING: + raise exception.RecordNotExists( + 'adpater %s does not exist' % adapter_id + ) + return ADAPTER_MAPPING[adapter_id] + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_FLAVORS_FIELDS) +def list_flavors(user=None, session=None, **filters): + """List flavors.""" + load_flavors() + return FLAVOR_MAPPING.values() + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_FLAVORS_FIELDS) +def get_flavor(flavor_id, user=None, session=None, **kwargs): + """Get flavor.""" + load_flavors() + if flavor_id not in FLAVOR_MAPPING: + raise exception.RecordNotExists( + 'flavor %s does not exist' % flavor_id + ) + return FLAVOR_MAPPING[flavor_id] diff --git a/compass-tasks-base/db/api/cluster.py b/compass-tasks-base/db/api/cluster.py new file mode 100644 index 0000000..7a7022c --- /dev/null +++ b/compass-tasks-base/db/api/cluster.py @@ -0,0 +1,2444 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, + +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Cluster database operations.""" +import copy +import functools +import logging +import re + +from compass.db.api import adapter_holder as adapter_api +from compass.db.api import database +from compass.db.api import metadata_holder as metadata_api +from compass.db.api import permission +from compass.db.api import user as user_api +from compass.db.api import utils +from compass.db import exception +from compass.db import models +from compass.utils import util + + +SUPPORTED_FIELDS = [ + 'name', 'os_name', 'owner', + 'adapter_name', 'flavor_name' +] +SUPPORTED_CLUSTERHOST_FIELDS = [] +RESP_FIELDS = [ + 'id', 'name', 'os_name', 'os_id', 'adapter_id', 'flavor_id', + 'reinstall_distributed_system', 'flavor', + 'distributed_system_installed', + 'owner', 'adapter_name', 'flavor_name', + 'created_at', 'updated_at' +] +RESP_CLUSTERHOST_FIELDS = [ + 'id', 'host_id', 'clusterhost_id', 'machine_id', + 'name', 'hostname', 'roles', 'os_installer', + 'cluster_id', 'clustername', 'location', 'tag', + 'networks', 'mac', 'switch_ip', 'port', 'switches', + 'os_installed', 'distributed_system_installed', + 'os_name', 'os_id', 'ip', + 'reinstall_os', 'reinstall_distributed_system', + 'owner', 'cluster_id', + 'created_at', 'updated_at', + 'patched_roles' +] +RESP_CONFIG_FIELDS = [ + 'os_config', + 'package_config', + 'config_step', + 'config_validated', + 'created_at', + 'updated_at' +] +RESP_DEPLOYED_CONFIG_FIELDS = [ + 'deployed_os_config', + 'deployed_package_config', + 'created_at', + 'updated_at' +] +RESP_METADATA_FIELDS = [ + 'os_config', 'package_config' +] +RESP_CLUSTERHOST_CONFIG_FIELDS = [ + 'package_config', + 'os_config', + 'config_step', + 'config_validated', + 'networks', + 'created_at', + 'updated_at' +] +RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS = [ + 'deployed_os_config', + 'deployed_package_config', + 'created_at', + 'updated_at' +] +RESP_STATE_FIELDS = [ + 'id', 'state', 'percentage', 'message', 'severity', + 'status', 'ready', + 'created_at', 'updated_at' +] +RESP_CLUSTERHOST_STATE_FIELDS = [ + 'id', 'state', 'percentage', 'message', 'severity', + 'ready', 'created_at', 'updated_at' +] +RESP_REVIEW_FIELDS = [ + 'cluster', 'hosts' +] +RESP_DEPLOY_FIELDS = [ + 'status', 'cluster', 'hosts' +] +IGNORE_FIELDS = ['id', 'created_at', 'updated_at'] +ADDED_FIELDS = ['name', 'adapter_id', 'os_id'] +OPTIONAL_ADDED_FIELDS = ['flavor_id'] +UPDATED_FIELDS = ['name', 'reinstall_distributed_system'] +ADDED_HOST_FIELDS = ['machine_id'] +UPDATED_HOST_FIELDS = ['name', 'reinstall_os'] +UPDATED_CLUSTERHOST_FIELDS = ['roles', 'patched_roles'] +PATCHED_CLUSTERHOST_FIELDS = ['patched_roles'] +UPDATED_CONFIG_FIELDS = [ + 'put_os_config', 'put_package_config', 'config_step' +] +UPDATED_DEPLOYED_CONFIG_FIELDS = [ + 'deployed_os_config', 'deployed_package_config' +] +PATCHED_CONFIG_FIELDS = [ + 'patched_os_config', 'patched_package_config', 'config_step' +] +UPDATED_CLUSTERHOST_CONFIG_FIELDS = [ + 'put_os_config', + 'put_package_config' +] +PATCHED_CLUSTERHOST_CONFIG_FIELDS = [ + 'patched_os_config', + 'patched_package_config' +] +UPDATED_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS = [ + 'deployed_os_config', + 'deployed_package_config' +] +UPDATED_CLUSTERHOST_STATE_FIELDS = [ + 'state', 'percentage', 'message', 'severity' +] +UPDATED_CLUSTERHOST_STATE_INTERNAL_FIELDS = [ + 'ready' +] +UPDATED_CLUSTER_STATE_FIELDS = ['state'] +IGNORE_UPDATED_CLUSTER_STATE_FIELDS = ['percentage', 'message', 'severity'] +UPDATED_CLUSTER_STATE_INTERNAL_FIELDS = ['ready'] +RESP_CLUSTERHOST_LOG_FIELDS = [ + 'clusterhost_id', 'id', 'host_id', 'cluster_id', + 'filename', 'position', 'partial_line', + 'percentage', + 'message', 'severity', 'line_matcher_name' +] +ADDED_CLUSTERHOST_LOG_FIELDS = [ + 'filename' +] +UPDATED_CLUSTERHOST_LOG_FIELDS = [ + 'position', 'partial_line', 'percentage', + 'message', 'severity', 'line_matcher_name' +] + + +@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_CLUSTERS +) +@utils.wrap_to_dict(RESP_FIELDS) +def list_clusters(user=None, session=None, **filters): + """List clusters.""" + clusters = utils.list_db_objects( + session, models.Cluster, **filters + ) + logging.info('user is %s', user.email) + if not user.is_admin and len(clusters): + clusters = [c for c in clusters if c.owner == user.email] + return clusters + + +def _get_cluster(cluster_id, session=None, **kwargs): + """Get cluster by id.""" + if isinstance(cluster_id, (int, long)): + return utils.get_db_object( + session, models.Cluster, id=cluster_id, **kwargs + ) + raise exception.InvalidParameter( + 'cluster id %s type is not int compatible' % cluster_id + ) + + +def get_cluster_internal(cluster_id, session=None, **kwargs): + """Helper function to get cluster. + + Should be only used by other files under db/api. + """ + return _get_cluster(cluster_id, session=session, **kwargs) + + +def _get_cluster_host( + cluster_id, host_id, session=None, **kwargs +): + """Get clusterhost by cluster id and host id.""" + cluster = _get_cluster(cluster_id, session=session, **kwargs) + from compass.db.api import host as host_api + host = host_api.get_host_internal(host_id, session=session, **kwargs) + return utils.get_db_object( + session, models.ClusterHost, + cluster_id=cluster.id, + host_id=host.id, + **kwargs + ) + + +def _get_clusterhost(clusterhost_id, session=None, **kwargs): + """Get clusterhost by clusterhost id.""" + if isinstance(clusterhost_id, (int, long)): + return utils.get_db_object( + session, models.ClusterHost, + clusterhost_id=clusterhost_id, + **kwargs + ) + raise exception.InvalidParameter( + 'clusterhost id %s type is not int compatible' % clusterhost_id + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_CLUSTERS +) +@utils.wrap_to_dict(RESP_FIELDS) +def get_cluster( + cluster_id, exception_when_missing=True, + user=None, session=None, **kwargs +): + """Get cluster info.""" + return _get_cluster( + cluster_id, + session=session, + exception_when_missing=exception_when_missing + ) + + +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_CLUSTERS) +def is_cluster_os_ready( + cluster_id, exception_when_missing=True, + user=None, session=None, **kwargs +): + cluster = utils.get_db_object( + session, models.Cluster, exception_when_missing, id=cluster_id) + + all_states = ([i.host.state.ready for i in cluster.clusterhosts]) + + logging.info("is_cluster_os_ready: all_states %s" % all_states) + + return all(all_states) + + +def check_cluster_validated(cluster): + """Check cluster is validated.""" + if not cluster.config_validated: + raise exception.Forbidden( + 'cluster %s is not validated' % cluster.name + ) + + +def check_clusterhost_validated(clusterhost): + """Check clusterhost is validated.""" + if not clusterhost.config_validated: + raise exception.Forbidden( + 'clusterhost %s is not validated' % clusterhost.name + ) + + +def check_cluster_editable( + cluster, user=None, + check_in_installing=False +): + """Check if cluster is editable. + + If we try to set cluster + reinstall_distributed_system attribute or any + checking to make sure the cluster is not in installing state, + we can set check_in_installing to True. + Otherwise we will make sure the cluster is not in deploying or + deployed. + If user is not admin or not the owner of the cluster, the check + will fail to make sure he can not update the cluster attributes. + """ + if check_in_installing: + if cluster.state.state == 'INSTALLING': + raise exception.Forbidden( + 'cluster %s is not editable ' + 'when state is installing' % cluster.name + ) +# elif ( +# cluster.flavor_name and +# not cluster.reinstall_distributed_system +# ): +# raise exception.Forbidden( +# 'cluster %s is not editable ' +# 'when not to be reinstalled' % cluster.name +# ) + if user and not user.is_admin and cluster.creator_id != user.id: + raise exception.Forbidden( + 'cluster %s is not editable ' + 'when user is not admin or cluster owner' % cluster.name + ) + + +def is_cluster_editable( + cluster, user=None, + check_in_installing=False +): + """Get if cluster is editble.""" + try: + check_cluster_editable( + cluster, user=user, + check_in_installing=check_in_installing + ) + return True + except exception.Forbidden: + return False + + +@utils.supported_filters( + ADDED_FIELDS, + optional_support_keys=OPTIONAL_ADDED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates(name=utils.check_name) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_CLUSTER +) +@utils.wrap_to_dict(RESP_FIELDS) +def add_cluster( + exception_when_existing=True, + name=None, adapter_id=None, flavor_id=None, + user=None, session=None, **kwargs +): + """Create a cluster.""" + adapter = adapter_api.get_adapter( + adapter_id, user=user, session=session + ) + # if flavor_id is not None, also set flavor field. + # In future maybe we can move the use of flavor from + # models.py to db/api and explictly get flavor when + # needed instead of setting flavor into cluster record. + flavor = {} + if flavor_id: + flavor = adapter_api.get_flavor( + flavor_id, + user=user, session=session + ) + if flavor['adapter_id'] != adapter['id']: + raise exception.InvalidParameter( + 'flavor %s is not of adapter %s' % ( + flavor_id, adapter_id + ) + ) + + cluster = utils.add_db_object( + session, models.Cluster, exception_when_existing, + name, user.id, adapter_id=adapter_id, + flavor_id=flavor_id, flavor=flavor, **kwargs + ) + return cluster + + +@utils.supported_filters( + optional_support_keys=UPDATED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates(name=utils.check_name) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_CLUSTER +) +@utils.wrap_to_dict(RESP_FIELDS) +def update_cluster(cluster_id, user=None, session=None, **kwargs): + """Update a cluster.""" + cluster = _get_cluster( + cluster_id, session=session + ) + check_cluster_editable( + cluster, user=user, + check_in_installing=( + kwargs.get('reinstall_distributed_system', False) + ) + ) + return utils.update_db_object(session, cluster, **kwargs) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEL_CLUSTER +) +@utils.wrap_to_dict( + RESP_FIELDS + ['status', 'cluster', 'hosts'], + cluster=RESP_FIELDS, + hosts=RESP_CLUSTERHOST_FIELDS +) +def del_cluster( + cluster_id, force=False, from_database_only=False, + delete_underlying_host=False, user=None, session=None, **kwargs +): + """Delete a cluster. + + If force, the cluster will be deleted anyway. It is used by cli to + force clean a cluster in any case. + If from_database_only, the cluster recored will only be removed from + database. Otherwise, a del task is sent to celery to do clean deletion. + If delete_underlying_host, all hosts under this cluster will also be + deleted. + The backend will call del_cluster again with from_database_only set + when it has done the deletion work on os installer/package installer. + """ + cluster = _get_cluster( + cluster_id, session=session + ) + logging.debug( + 'delete cluster %s with force=%s ' + 'from_database_only=%s delete_underlying_host=%s', + cluster.id, force, from_database_only, delete_underlying_host + ) + # force set cluster state to ERROR and the state of any clusterhost + # in the cluster to ERROR when we want to delete the cluster anyway + # even the cluster is in installing or already installed. + # It let the api know the deleting is in doing when backend is doing + # the real deleting. + # In future we may import a new state like INDELETE to indicate + # the deleting is processing. + # We need discuss about if we can delete a cluster when it is already + # installed by api. + for clusterhost in cluster.clusterhosts: + if clusterhost.state.state != 'UNINITIALIZED' and force: + clusterhost.state.state = 'ERROR' + if delete_underlying_host: + host = clusterhost.host + if host.state.state != 'UNINITIALIZED' and force: + host.state.state = 'ERROR' + if cluster.state.state != 'UNINITIALIZED' and force: + cluster.state.state = 'ERROR' + + check_cluster_editable( + cluster, user=user, + check_in_installing=True + ) + + # delete underlying host if delete_underlying_host is set. + if delete_underlying_host: + for clusterhost in cluster.clusterhosts: + # delete underlying host only user has permission. + from compass.db.api import host as host_api + host = clusterhost.host + if host_api.is_host_editable( + host, user=user, check_in_installing=True + ): + # Delete host record directly in database when there is no need + # to do the deletion in backend or from_database_only is set. + if host.state.state == 'UNINITIALIZED' or from_database_only: + utils.del_db_object( + session, host + ) + + # Delete cluster record directly in database when there + # is no need to do the deletion in backend or from_database_only is set. + if cluster.state.state == 'UNINITIALIZED' or from_database_only: + return utils.del_db_object( + session, cluster + ) + else: + from compass.tasks import client as celery_client + logging.info('send del cluster %s task to celery', cluster_id) + celery_client.celery.send_task( + 'compass.tasks.delete_cluster', + ( + user.email, cluster.id, + [ + clusterhost.host_id + for clusterhost in cluster.clusterhosts + ], + delete_underlying_host + ), + queue=user.email, + exchange=user.email, + routing_key=user.email + ) + return { + 'status': 'delete action is sent', + 'cluster': cluster, + 'hosts': cluster.clusterhosts + } + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_CLUSTER_CONFIG +) +@utils.wrap_to_dict(RESP_CONFIG_FIELDS) +def get_cluster_config(cluster_id, user=None, session=None, **kwargs): + """Get cluster config.""" + return _get_cluster(cluster_id, session=session) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_CLUSTER_CONFIG +) +@utils.wrap_to_dict(RESP_DEPLOYED_CONFIG_FIELDS) +def get_cluster_deployed_config(cluster_id, user=None, session=None, **kwargs): + """Get cluster deployed config.""" + return _get_cluster(cluster_id, session=session) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_METADATA_FIELDS) +def get_cluster_metadata(cluster_id, user=None, session=None, **kwargs): + """Get cluster metadata. + + If no flavor in the cluster, it means this is a os only cluster. + We ignore package metadata for os only cluster. + """ + cluster = _get_cluster(cluster_id, session=session) + metadatas = {} + os_name = cluster.os_name + if os_name: + metadatas.update( + metadata_api.get_os_metadata( + os_name, session=session + ) + ) + flavor_id = cluster.flavor_id + if flavor_id: + metadatas.update( + metadata_api.get_flavor_metadata( + flavor_id, + user=user, session=session + ) + ) + + return metadatas + + +def _cluster_os_config_validates( + config, cluster, session=None, user=None, **kwargs +): + """Check cluster os config validation.""" + metadata_api.validate_os_config( + config, cluster.os_id + ) + + +def _cluster_package_config_validates( + config, cluster, session=None, user=None, **kwargs +): + """Check cluster package config validation.""" + metadata_api.validate_flavor_config( + config, cluster.flavor_id + ) + + +@utils.input_validates_with_args( + put_os_config=_cluster_os_config_validates, + put_package_config=_cluster_package_config_validates +) +@utils.output_validates_with_args( + os_config=_cluster_os_config_validates, + package_config=_cluster_package_config_validates +) +@utils.wrap_to_dict(RESP_CONFIG_FIELDS) +def _update_cluster_config(cluster, session=None, user=None, **kwargs): + """Update a cluster config.""" + check_cluster_editable(cluster, user=user) + return utils.update_db_object( + session, cluster, **kwargs + ) + + +# replace os_config to deployed_os_config, +# package_config to deployed_package_config +@utils.replace_filters( + os_config='deployed_os_config', + package_config='deployed_package_config' +) +@utils.supported_filters( + optional_support_keys=UPDATED_DEPLOYED_CONFIG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_CLUSTER_CONFIG +) +@utils.wrap_to_dict(RESP_DEPLOYED_CONFIG_FIELDS) +def update_cluster_deployed_config( + cluster_id, user=None, session=None, **kwargs +): + """Update cluster deployed config.""" + cluster = _get_cluster(cluster_id, session=session) + check_cluster_editable(cluster, user=user) + check_cluster_validated(cluster) + return utils.update_db_object( + session, cluster, **kwargs + ) + + +# replace os_config to put_os_config, +# package_config to put_package_config in kwargs. +# It tells db these fields will be updated not patched. +@utils.replace_filters( + os_config='put_os_config', + package_config='put_package_config' +) +@utils.supported_filters( + optional_support_keys=UPDATED_CONFIG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_CLUSTER_CONFIG +) +def update_cluster_config(cluster_id, user=None, session=None, **kwargs): + """Update cluster config.""" + cluster = _get_cluster(cluster_id, session=session) + return _update_cluster_config( + cluster, session=session, user=user, **kwargs + ) + + +# replace os_config to patched_os_config and +# package_config to patched_package_config in kwargs. +# It tells db these fields will be patched not updated. +@utils.replace_filters( + os_config='patched_os_config', + package_config='patched_package_config' +) +@utils.supported_filters( + optional_support_keys=PATCHED_CONFIG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_CLUSTER_CONFIG +) +def patch_cluster_config(cluster_id, user=None, session=None, **kwargs): + """patch cluster config.""" + cluster = _get_cluster(cluster_id, session=session) + return _update_cluster_config( + cluster, session=session, user=user, **kwargs + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEL_CLUSTER_CONFIG +) +@utils.wrap_to_dict(RESP_CONFIG_FIELDS) +def del_cluster_config(cluster_id, user=None, session=None): + """Delete a cluster config.""" + cluster = _get_cluster( + cluster_id, session=session + ) + check_cluster_editable(cluster, user=user) + return utils.update_db_object( + session, cluster, os_config={}, + package_config={}, config_validated=False + ) + + +def _roles_validates(roles, cluster, session=None, user=None): + """Check roles is validated to a cluster's roles.""" + if roles: + if not cluster.flavor_name: + raise exception.InvalidParameter( + 'not flavor in cluster %s' % cluster.name + ) + cluster_roles = [role['name'] for role in cluster.flavor['roles']] + for role in roles: + if role not in cluster_roles: + raise exception.InvalidParameter( + 'role %s is not in cluster roles %s' % ( + role, cluster_roles + ) + ) + + +def _cluster_host_roles_validates( + value, cluster, host, session=None, user=None, **kwargs +): + """Check clusterhost roles is validated by cluster and host.""" + _roles_validates(value, cluster, session=session, user=user) + + +def _clusterhost_roles_validates( + value, clusterhost, session=None, user=None, **kwargs +): + """Check clusterhost roles is validated by clusterhost.""" + _roles_validates( + value, clusterhost.cluster, session=session, user=user + ) + + +@utils.supported_filters( + optional_support_keys=UPDATED_HOST_FIELDS, + ignore_support_keys=UPDATED_CLUSTERHOST_FIELDS +) +@utils.input_validates(name=utils.check_name) +def _add_host_if_not_exist( + machine_id, cluster, session=None, user=None, **kwargs +): + """Add underlying host if it does not exist.""" + from compass.db.api import host as host_api + host = host_api.get_host_internal( + machine_id, session=session, exception_when_missing=False + ) + if host: + if kwargs: + # ignore update underlying host if host is not editable. + from compass.db.api import host as host_api + if host_api.is_host_editable( + host, user=cluster.creator, + check_in_installing=kwargs.get('reinstall_os', False), + ): + utils.update_db_object( + session, host, + **kwargs + ) + else: + logging.debug( + 'ignore update host host %s ' + 'since it is not editable' % host.name + ) + else: + logging.debug('nothing to update for host %s', host.name) + else: + from compass.db.api import adapter_holder as adapter_api + adapter = adapter_api.get_adapter( + cluster.adapter_name, user=user, session=session + ) + host = utils.add_db_object( + session, models.Host, False, machine_id, + os_name=cluster.os_name, + os_installer=adapter['os_installer'], + creator=cluster.creator, + **kwargs + ) + return host + + +@utils.supported_filters( + optional_support_keys=UPDATED_CLUSTERHOST_FIELDS, + ignore_support_keys=UPDATED_HOST_FIELDS +) +@utils.input_validates_with_args( + roles=_cluster_host_roles_validates +) +def _add_clusterhost_only( + cluster, host, + exception_when_existing=False, + session=None, user=None, + **kwargs +): + """Get clusterhost only.""" + if not cluster.state.state == "UNINITIALIZED": + cluster.state.ready = False + cluster.state.state = "UNINITIALIZED" + cluster.state.percentage = 0.0 + utils.update_db_object(session, cluster.state, state="UNINITIALIZED") + + return utils.add_db_object( + session, models.ClusterHost, exception_when_existing, + cluster.id, host.id, **kwargs + ) + + +@utils.supported_filters( + ADDED_HOST_FIELDS, + optional_support_keys=UPDATED_HOST_FIELDS + UPDATED_CLUSTERHOST_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +def _add_clusterhost( + cluster, + exception_when_existing=False, + session=None, user=None, machine_id=None, **kwargs +): + """Add clusterhost and add underlying host if it does not exist.""" + host = _add_host_if_not_exist( + machine_id, cluster, session=session, + user=user, **kwargs + ) + + return _add_clusterhost_only( + cluster, host, exception_when_existing=exception_when_existing, + session=session, user=user, **kwargs + ) + + +def _add_clusterhosts(cluster, machines, session=None, user=None): + """Add machines to cluster. + + Args: + machines: list of dict which contains clusterost attr to update. + + Examples: + [{'machine_id': 1, 'name': 'host1'}] + """ + check_cluster_editable( + cluster, user=user, + check_in_installing=True + ) + if cluster.state.state == 'SUCCESSFUL': + cluster.state.state == 'UPDATE_PREPARING' + for machine_dict in machines: + _add_clusterhost( + cluster, session=session, user=user, **machine_dict + ) + + +def _remove_clusterhosts(cluster, hosts, session=None, user=None): + """Remove hosts from cluster. + + Args: + hosts: list of host id. + """ + check_cluster_editable( + cluster, user=user, + check_in_installing=True + ) + utils.del_db_objects( + session, models.ClusterHost, + cluster_id=cluster.id, host_id=hosts + ) + + +def _set_clusterhosts(cluster, machines, session=None, user=None): + """set machines to cluster. + + Args: + machines: list of dict which contains clusterost attr to update. + + Examples: + [{'machine_id': 1, 'name': 'host1'}] + """ + check_cluster_editable( + cluster, user=user, + check_in_installing=True + ) + utils.del_db_objects( + session, models.ClusterHost, + cluster_id=cluster.id + ) + if cluster.state.state == 'SUCCESSFUL': + cluster.state.state = 'UPDATE_PREPARING' + for machine_dict in machines: + _add_clusterhost( + cluster, True, session=session, user=user, **machine_dict + ) + + +@utils.supported_filters(optional_support_keys=SUPPORTED_CLUSTERHOST_FIELDS) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_CLUSTERHOSTS +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS) +def list_cluster_hosts(cluster_id, user=None, session=None, **filters): + """List clusterhosts of a cluster.""" + cluster = _get_cluster(cluster_id, session=session) + return utils.list_db_objects( + session, models.ClusterHost, cluster_id=cluster.id, + **filters + ) + + +@utils.supported_filters(optional_support_keys=SUPPORTED_CLUSTERHOST_FIELDS) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_CLUSTERHOSTS +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS) +def list_clusterhosts(user=None, session=None, **filters): + """List all clusterhosts.""" + return utils.list_db_objects( + session, models.ClusterHost, **filters + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_CLUSTERHOSTS +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS) +def get_cluster_host( + cluster_id, host_id, exception_when_missing=True, + user=None, session=None, **kwargs +): + """Get clusterhost info by cluster id and host id.""" + return _get_cluster_host( + cluster_id, host_id, session=session, + exception_when_missing=exception_when_missing, + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_CLUSTERHOSTS +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS) +def get_clusterhost( + clusterhost_id, exception_when_missing=True, + user=None, session=None, **kwargs +): + """Get clusterhost info by clusterhost id.""" + return _get_clusterhost( + clusterhost_id, session=session, + exception_when_missing=exception_when_missing, + user=user + ) + + +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_CLUSTER_HOSTS +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS) +def add_cluster_host( + cluster_id, exception_when_existing=True, + user=None, session=None, **kwargs +): + """Add a host to a cluster.""" + cluster = _get_cluster(cluster_id, session=session) + check_cluster_editable( + cluster, user=user, + check_in_installing=True + ) + if cluster.state.state == 'SUCCESSFUL': + cluster.state.state = 'UPDATE_PREPARING' + return _add_clusterhost( + cluster, exception_when_existing, + session=session, user=user, **kwargs + ) + + +@utils.supported_filters( + optional_support_keys=UPDATED_HOST_FIELDS, + ignore_support_keys=( + UPDATED_CLUSTERHOST_FIELDS + + PATCHED_CLUSTERHOST_FIELDS + ) +) +def _update_host_if_necessary( + clusterhost, session=None, user=None, **kwargs +): + """Update underlying host if there is something to update.""" + host = clusterhost.host + if kwargs: + # ignore update underlying host if the host is not editable. + from compass.db.api import host as host_api + if host_api.is_host_editable( + host, user=clusterhost.cluster.creator, + check_in_installing=kwargs.get('reinstall_os', False), + ): + utils.update_db_object( + session, host, + **kwargs + ) + else: + logging.debug( + 'ignore update host %s since it is not editable' % host.name + ) + else: + logging.debug( + 'nothing to update for host %s', host.name + ) + return host + + +@utils.supported_filters( + optional_support_keys=( + UPDATED_CLUSTERHOST_FIELDS + + PATCHED_CLUSTERHOST_FIELDS + ), + ignore_support_keys=UPDATED_HOST_FIELDS +) +@utils.input_validates_with_args( + roles=_clusterhost_roles_validates, + patched_roles=_clusterhost_roles_validates +) +def _update_clusterhost_only( + clusterhost, session=None, user=None, **kwargs +): + """Update clusterhost only.""" + check_cluster_editable(clusterhost.cluster, user=user) + return utils.update_db_object( + session, clusterhost, **kwargs + ) + + +@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS) +def _update_clusterhost(clusterhost, session=None, user=None, **kwargs): + """Update clusterhost and underlying host if necessary.""" + _update_host_if_necessary( + clusterhost, session=session, user=user, **kwargs + ) + return _update_clusterhost_only( + clusterhost, session=session, user=user, **kwargs + ) + + +@utils.supported_filters( + optional_support_keys=(UPDATED_HOST_FIELDS + UPDATED_CLUSTERHOST_FIELDS), + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_CLUSTER_HOSTS +) +def update_cluster_host( + cluster_id, host_id, user=None, + session=None, **kwargs +): + """Update clusterhost by cluster id and host id.""" + logging.info('updating kwargs: %s', kwargs) + clusterhost = _get_cluster_host( + cluster_id, host_id, session=session + ) + return _update_clusterhost( + clusterhost, session=session, user=user, **kwargs + ) + + +@utils.supported_filters( + optional_support_keys=(UPDATED_HOST_FIELDS + UPDATED_CLUSTERHOST_FIELDS), + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_CLUSTER_HOSTS +) +def update_clusterhost( + clusterhost_id, user=None, + session=None, **kwargs +): + """Update clusterhost by clusterhost id.""" + clusterhost = _get_clusterhost( + clusterhost_id, session=session + ) + return _update_clusterhost( + clusterhost, session=session, user=user, **kwargs + ) + + +# replace roles to patched_roles in kwargs. +# It tells db roles field will be patched. +@utils.replace_filters( + roles='patched_roles' +) +@utils.supported_filters( + optional_support_keys=PATCHED_CLUSTERHOST_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_CLUSTER_HOSTS +) +def patch_cluster_host( + cluster_id, host_id, user=None, + session=None, **kwargs +): + """Patch clusterhost by cluster id and host id.""" + logging.info("kwargs are %s", kwargs) + clusterhost = _get_cluster_host( + cluster_id, host_id, session=session + ) + updated_clusterhost = _update_clusterhost( + clusterhost, session=session, user=user, **kwargs + ) + return updated_clusterhost + + +# replace roles to patched_roles in kwargs. +# It tells db roles field will be patched. +@utils.replace_filters( + roles='patched_roles' +) +@utils.supported_filters( + optional_support_keys=PATCHED_CLUSTERHOST_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_CLUSTER_HOSTS +) +def patch_clusterhost( + clusterhost_id, user=None, session=None, + **kwargs +): + """Patch clusterhost by clusterhost id.""" + clusterhost = _get_clusterhost( + clusterhost_id, session=session + ) + return _update_clusterhost( + clusterhost, session=session, user=user, **kwargs + ) + + +@user_api.check_user_permission( + permission.PERMISSION_DEL_CLUSTER_HOST +) +@utils.wrap_to_dict( + RESP_CLUSTERHOST_FIELDS + ['status', 'host'], + host=RESP_CLUSTERHOST_FIELDS +) +def _del_cluster_host( + clusterhost, + force=False, from_database_only=False, + delete_underlying_host=False, user=None, + session=None, **kwargs +): + """delete clusterhost. + + If force, the cluster host will be deleted anyway. + If from_database_only, the cluster host recored will only be + deleted from database. Otherwise a celery task sent to do + clean deletion. + If delete_underlying_host, the underlying host will also be deleted. + The backend will call _del_cluster_host again when the clusterhost is + deleted from os installer/package installer with from_database_only + set. + """ + # force set clusterhost state to ERROR when we want to delete the + # clusterhost anyway even the clusterhost is in installing or already + # installed. It let the api know the deleting is in doing when backend + # is doing the real deleting. In future we may import a new state like + # INDELETE to indicate the deleting is processing. + # We need discuss about if we can delete a clusterhost when it is already + # installed by api. + if clusterhost.state.state != 'UNINITIALIZED' and force: + clusterhost.state.state = 'ERROR' + if not force: + check_cluster_editable( + clusterhost.cluster, user=user, + check_in_installing=True + ) + # delete underlying host if delete_underlying_host is set. + if delete_underlying_host: + host = clusterhost.host + if host.state.state != 'UNINITIALIZED' and force: + host.state.state = 'ERROR' + # only delete the host when user have the permission to delete it. + import compass.db.api.host as host_api + if host_api.is_host_editable( + host, user=user, + check_in_installing=True + ): + # if there is no need to do the deletion by backend or + # from_database_only is set, we only delete the record + # in database. + if host.state.state == 'UNINITIALIZED' or from_database_only: + utils.del_db_object( + session, host + ) + + # if there is no need to do the deletion by backend or + # from_database_only is set, we only delete the record in database. + if clusterhost.state.state == 'UNINITIALIZED' or from_database_only: + return utils.del_db_object( + session, clusterhost + ) + else: + logging.info( + 'send del cluster %s host %s task to celery', + clusterhost.cluster_id, clusterhost.host_id + ) + from compass.tasks import client as celery_client + celery_client.celery.send_task( + 'compass.tasks.delete_cluster_host', + ( + user.email, clusterhost.cluster_id, clusterhost.host_id, + delete_underlying_host + ), + queue=user.email, + exchange=user.email, + routing_key=user.email + ) + return { + 'status': 'delete action sent', + 'host': clusterhost, + } + + +@utils.supported_filters([]) +@database.run_in_session() +def del_cluster_host( + cluster_id, host_id, + force=False, from_database_only=False, + delete_underlying_host=False, user=None, + session=None, **kwargs +): + """Delete clusterhost by cluster id and host id.""" + clusterhost = _get_cluster_host( + cluster_id, host_id, session=session + ) + return _del_cluster_host( + clusterhost, force=force, from_database_only=from_database_only, + delete_underlying_host=delete_underlying_host, user=user, + session=session, **kwargs + ) + + +@utils.supported_filters([]) +@database.run_in_session() +def del_clusterhost( + clusterhost_id, + force=False, from_database_only=False, + delete_underlying_host=False, user=None, + session=None, **kwargs +): + """Delete clusterhost by clusterhost id.""" + clusterhost = _get_clusterhost( + clusterhost_id, session=session + ) + return _del_cluster_host( + clusterhost, force=force, from_database_only=from_database_only, + delete_underlying_host=delete_underlying_host, user=user, + session=session, **kwargs + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_CLUSTERHOST_CONFIG +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS) +def get_cluster_host_config( + cluster_id, host_id, user=None, + session=None, **kwargs +): + """Get clusterhost config by cluster id and host id.""" + return _get_cluster_host( + cluster_id, host_id, session=session + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_CLUSTERHOST_CONFIG +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS) +def get_cluster_host_deployed_config( + cluster_id, host_id, user=None, session=None, **kwargs +): + """Get clusterhost deployed config by cluster id and host id.""" + return _get_cluster_host( + cluster_id, host_id, session=session + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_CLUSTERHOST_CONFIG +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS) +def get_clusterhost_config(clusterhost_id, user=None, session=None, **kwargs): + """Get clusterhost config by clusterhost id.""" + return _get_clusterhost( + clusterhost_id, session=session + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_CLUSTERHOST_CONFIG +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS) +def get_clusterhost_deployed_config( + clusterhost_id, user=None, + session=None, **kwargs +): + """Get clusterhost deployed config by clusterhost id.""" + return _get_clusterhost( + clusterhost_id, session=session + ) + + +def _clusterhost_os_config_validates( + config, clusterhost, session=None, user=None, **kwargs +): + """Validate clusterhost's underlying host os config.""" + from compass.db.api import host as host_api + host = clusterhost.host + host_api.check_host_editable(host, user=user) + metadata_api.validate_os_config( + config, host.os_id + ) + + +def _clusterhost_package_config_validates( + config, clusterhost, session=None, user=None, **kwargs +): + """Validate clusterhost's cluster package config.""" + cluster = clusterhost.cluster + check_cluster_editable(cluster, user=user) + metadata_api.validate_flavor_config( + config, cluster.flavor_id + ) + + +def _filter_clusterhost_host_editable( + config, clusterhost, session=None, user=None, **kwargs +): + """Filter fields if the underlying host is not editable.""" + from compass.db.api import host as host_api + host = clusterhost.host + return host_api.is_host_editable(host, user=user) + + +@utils.input_filters( + put_os_config=_filter_clusterhost_host_editable, + patched_os_config=_filter_clusterhost_host_editable +) +@utils.input_validates_with_args( + put_os_config=_clusterhost_os_config_validates, + put_package_config=_clusterhost_package_config_validates +) +@utils.output_validates_with_args( + os_config=_clusterhost_os_config_validates, + package_config=_clusterhost_package_config_validates +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS) +def _update_clusterhost_config(clusterhost, session=None, user=None, **kwargs): + """Update clusterhost config.""" + return utils.update_db_object( + session, clusterhost, **kwargs + ) + + +def _clusterhost_host_validated( + config, clusterhost, session=None, user=None, **kwargs +): + """Check clusterhost's underlying host is validated.""" + from compass.db.api import host as host_api + host = clusterhost.host + host_api.check_host_editable(host, user=user) + host_api.check_host_validated(host) + + +def _clusterhost_cluster_validated( + config, clusterhost, session=None, user=None, **kwargs +): + """Check clusterhost's cluster is validated.""" + cluster = clusterhost.cluster + check_cluster_editable(cluster, user=user) + check_clusterhost_validated(clusterhost) + + +@utils.input_filters( + deployed_os_config=_filter_clusterhost_host_editable, +) +@utils.input_validates_with_args( + deployed_os_config=_clusterhost_host_validated, + deployed_package_config=_clusterhost_cluster_validated +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS) +def _update_clusterhost_deployed_config( + clusterhost, session=None, user=None, **kwargs +): + """Update clusterhost deployed config.""" + return utils.update_db_object( + session, clusterhost, **kwargs + ) + + +# replace os_config to put_os_config and +# package_config to put_package_config in kwargs. +# It tells db these fields will be updated not patched. +@utils.replace_filters( + os_config='put_os_config', + package_config='put_package_config' +) +@utils.supported_filters( + optional_support_keys=UPDATED_CLUSTERHOST_CONFIG_FIELDS, +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_CLUSTERHOST_CONFIG +) +def update_cluster_host_config( + cluster_id, host_id, user=None, session=None, **kwargs +): + """Update clusterhost config by cluster id and host id.""" + clusterhost = _get_cluster_host( + cluster_id, host_id, session=session + ) + return _update_clusterhost_config( + clusterhost, user=user, session=session, **kwargs + ) + + +# replace os_config to deployed_os_config and +# package_config to deployed_package_config in kwargs. +@utils.replace_filters( + os_config='deployed_os_config', + package_config='deployed_package_config' +) +@utils.supported_filters( + optional_support_keys=UPDATED_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_CLUSTERHOST_CONFIG +) +def update_cluster_host_deployed_config( + cluster_id, host_id, user=None, session=None, **kwargs +): + """Update clusterhost deployed config by cluster id and host id.""" + clusterhost = _get_cluster_host( + cluster_id, host_id, session=session + ) + return _update_clusterhost_deployed_config( + clusterhost, session=session, user=user, **kwargs + ) + + +# replace os_config to put_os_config and +# package_config to put_package_config in kwargs. +# It tells db these fields will be updated not patched. +@utils.replace_filters( + os_config='put_os_config', + package_config='put_package_config' +) +@utils.supported_filters( + optional_support_keys=UPDATED_CLUSTERHOST_CONFIG_FIELDS, +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_CLUSTERHOST_CONFIG +) +def update_clusterhost_config( + clusterhost_id, user=None, session=None, **kwargs +): + """Update clusterhost config by clusterhost id.""" + clusterhost = _get_clusterhost( + clusterhost_id, session=session + ) + return _update_clusterhost_config( + clusterhost, session=session, user=user, **kwargs + ) + + +# replace os_config to deployed_os_config and +# package_config to deployed_package_config in kwargs. +@utils.replace_filters( + os_config='deployed_os_config', + package_config='deployed_package_config' +) +@utils.supported_filters( + optional_support_keys=UPDATED_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_CLUSTERHOST_CONFIG +) +def update_clusterhost_deployed_config( + clusterhost_id, user=None, session=None, **kwargs +): + """Update clusterhost deployed config by clusterhost id.""" + clusterhost = _get_clusterhost( + clusterhost_id, session=session + ) + return _update_clusterhost_deployed_config( + clusterhost, session=session, user=user, **kwargs + ) + + +# replace os_config to patched_os_config and +# package_config to patched_package_config in kwargs +# It tells db these fields will be patched not updated. +@utils.replace_filters( + os_config='patched_os_config', + package_config='patched_package_config' +) +@utils.supported_filters( + optional_support_keys=PATCHED_CLUSTERHOST_CONFIG_FIELDS, +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_CLUSTERHOST_CONFIG +) +def patch_cluster_host_config( + cluster_id, host_id, user=None, session=None, **kwargs +): + """patch clusterhost config by cluster id and host id.""" + clusterhost = _get_cluster_host( + cluster_id, host_id, session=session + ) + return _update_clusterhost_config( + clusterhost, session=session, user=user, **kwargs + ) + + +# replace os_config to patched_os_config and +# package_config to patched_package_config in kwargs +# It tells db these fields will be patched not updated. +@utils.replace_filters( + os_config='patched_os_config', + package_config='patched_package_config' +) +@utils.supported_filters( + optional_support_keys=PATCHED_CLUSTERHOST_CONFIG_FIELDS, +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_CLUSTERHOST_CONFIG +) +def patch_clusterhost_config( + clusterhost_id, user=None, session=None, **kwargs +): + """patch clusterhost config by clusterhost id.""" + clusterhost = _get_clusterhost( + clusterhost_id, session=session + ) + return _update_clusterhost_config( + clusterhost, session=session, user=user, **kwargs + ) + + +def _clusterhost_host_editable( + config, clusterhost, session=None, user=None, **kwargs +): + """Check clusterhost underlying host is editable.""" + from compass.db.api import host as host_api + host_api.check_host_editable(clusterhost.host, user=user) + + +def _clusterhost_cluster_editable( + config, clusterhost, session=None, user=None, **kwargs +): + """Check clusterhost's cluster is editable.""" + check_cluster_editable(clusterhost.cluster, user=user) + + +@utils.supported_filters( + optional_support_keys=['os_config', 'package_config'] +) +@utils.input_filters( + os_config=_filter_clusterhost_host_editable, +) +@utils.output_validates_with_args( + package_config=_clusterhost_cluster_editable +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS) +def _delete_clusterhost_config( + clusterhost, session=None, user=None, **kwargs +): + """delete clusterhost config.""" + return utils.update_db_object( + session, clusterhost, config_validated=False, + **kwargs + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEL_CLUSTERHOST_CONFIG +) +def delete_cluster_host_config( + cluster_id, host_id, user=None, session=None +): + """Delete a clusterhost config by cluster id and host id.""" + clusterhost = _get_cluster_host( + cluster_id, host_id, session=session + ) + return _delete_clusterhost_config( + clusterhost, session=session, user=user, + os_config={}, package_config={} + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEL_CLUSTERHOST_CONFIG +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS) +def delete_clusterhost_config(clusterhost_id, user=None, session=None): + """Delet a clusterhost config by clusterhost id.""" + clusterhost = _get_clusterhost( + clusterhost_id, session=session + ) + return _delete_clusterhost_config( + clusterhost, session=session, user=user, + os_config={}, package_config={} + ) + + +@utils.supported_filters( + optional_support_keys=['add_hosts', 'remove_hosts', 'set_hosts'] +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_CLUSTER_HOSTS +) +@utils.wrap_to_dict( + ['hosts'], + hosts=RESP_CLUSTERHOST_FIELDS +) +def update_cluster_hosts( + cluster_id, add_hosts={}, set_hosts=None, + remove_hosts={}, user=None, session=None +): + """Update cluster hosts.""" + cluster = _get_cluster(cluster_id, session=session) + if remove_hosts: + _remove_clusterhosts( + cluster, session=session, user=user, **remove_hosts + ) + if add_hosts: + _add_clusterhosts( + cluster, session=session, user=user, **add_hosts + ) + if set_hosts is not None: + _set_clusterhosts( + cluster, session=session, user=user, **set_hosts + ) + + return { + 'hosts': list_cluster_hosts(cluster_id, session=session) + } + + +def validate_clusterhost(clusterhost, session=None): + """validate clusterhost.""" + roles = clusterhost.roles + if not roles: + if clusterhost.cluster.flavor_name: + raise exception.InvalidParameter( + 'empty roles for clusterhost %s' % clusterhost.name + ) + + +def validate_cluster(cluster, session=None): + """Validate cluster.""" + if not cluster.clusterhosts: + raise exception.InvalidParameter( + 'cluster %s does not have any hosts' % cluster.name + ) + if cluster.flavor_name: + cluster_roles = cluster.flavor['roles'] + else: + cluster_roles = [] + necessary_roles = set([ + role['name'] for role in cluster_roles if not role.get('optional') + ]) + clusterhost_roles = set([]) + interface_subnets = {} + for clusterhost in cluster.clusterhosts: + roles = clusterhost.roles + for role in roles: + clusterhost_roles.add(role['name']) + host = clusterhost.host + for host_network in host.host_networks: + interface_subnets.setdefault( + host_network.interface, set([]) + ).add(host_network.subnet.subnet) + missing_roles = necessary_roles - clusterhost_roles + if missing_roles: + raise exception.InvalidParameter( + 'cluster %s have some roles %s not assigned to any host' % ( + cluster.name, list(missing_roles) + ) + ) + for interface, subnets in interface_subnets.items(): + if len(subnets) > 1: + raise exception.InvalidParameter( + 'cluster %s multi subnets %s in interface %s' % ( + cluster.name, list(subnets), interface + ) + ) + + +@utils.supported_filters(optional_support_keys=['review']) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_REVIEW_CLUSTER +) +@utils.wrap_to_dict( + RESP_REVIEW_FIELDS, + cluster=RESP_CONFIG_FIELDS, + hosts=RESP_CLUSTERHOST_CONFIG_FIELDS +) +def review_cluster(cluster_id, review={}, user=None, session=None, **kwargs): + """review cluster. + + Args: + cluster_id: the cluster id. + review: dict contains hosts to be reviewed. either contains key + hosts or clusterhosts. where hosts is a list of host id, + clusterhosts is a list of clusterhost id. + """ + from compass.db.api import host as host_api + cluster = _get_cluster(cluster_id, session=session) + check_cluster_editable(cluster, user=user) + host_ids = review.get('hosts', []) + clusterhost_ids = review.get('clusterhosts', []) + clusterhosts = [] + # Get clusterhosts need to be reviewed. + for clusterhost in cluster.clusterhosts: + if ( + clusterhost.clusterhost_id in clusterhost_ids or + clusterhost.host_id in host_ids + ): + clusterhosts.append(clusterhost) + + os_config = copy.deepcopy(cluster.os_config) + os_config = metadata_api.autofill_os_config( + os_config, cluster.os_id, cluster=cluster + ) + metadata_api.validate_os_config( + os_config, cluster.os_id, True + ) + for clusterhost in clusterhosts: + host = clusterhost.host + # ignore underlying host os config validation + # since the host is not editable + if not host_api.is_host_editable( + host, user=user, check_in_installing=False + ): + logging.info( + 'ignore update host %s config ' + 'since it is not editable' % host.name + ) + continue + host_os_config = copy.deepcopy(host.os_config) + host_os_config = metadata_api.autofill_os_config( + host_os_config, host.os_id, + host=host + ) + deployed_os_config = util.merge_dict( + os_config, host_os_config + ) + metadata_api.validate_os_config( + deployed_os_config, host.os_id, True + ) + host_api.validate_host(host) + utils.update_db_object( + session, host, os_config=host_os_config, config_validated=True + ) + + package_config = copy.deepcopy(cluster.package_config) + if cluster.flavor_name: + package_config = metadata_api.autofill_flavor_config( + package_config, cluster.flavor_id, + cluster=cluster + ) + metadata_api.validate_flavor_config( + package_config, cluster.flavor_id, True + ) + for clusterhost in clusterhosts: + clusterhost_package_config = copy.deepcopy( + clusterhost.package_config + ) + clusterhost_package_config = ( + metadata_api.autofill_flavor_config( + clusterhost_package_config, + cluster.flavor_id, + clusterhost=clusterhost + ) + ) + deployed_package_config = util.merge_dict( + package_config, clusterhost_package_config + ) + metadata_api.validate_flavor_config( + deployed_package_config, + cluster.flavor_id, True + ) + validate_clusterhost(clusterhost, session=session) + utils.update_db_object( + session, clusterhost, + package_config=clusterhost_package_config, + config_validated=True + ) + + validate_cluster(cluster, session=session) + utils.update_db_object( + session, cluster, os_config=os_config, package_config=package_config, + config_validated=True + ) + return { + 'cluster': cluster, + 'hosts': clusterhosts + } + + +@utils.supported_filters(optional_support_keys=['deploy']) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEPLOY_CLUSTER +) +@utils.wrap_to_dict( + RESP_DEPLOY_FIELDS, + cluster=RESP_CONFIG_FIELDS, + hosts=RESP_CLUSTERHOST_FIELDS +) +def deploy_cluster( + cluster_id, deploy={}, user=None, session=None, **kwargs +): + """deploy cluster. + + Args: + cluster_id: cluster id. + deploy: dict contains key either hosts or clusterhosts. + deploy['hosts'] is a list of host id, + deploy['clusterhosts'] is a list of clusterhost id. + """ + from compass.db.api import host as host_api + from compass.tasks import client as celery_client + cluster = _get_cluster(cluster_id, session=session) + host_ids = deploy.get('hosts', []) + clusterhost_ids = deploy.get('clusterhosts', []) + clusterhosts = [] + # get clusterhost to deploy. + for clusterhost in cluster.clusterhosts: + if ( + clusterhost.clusterhost_id in clusterhost_ids or + clusterhost.host_id in host_ids + ): + clusterhosts.append(clusterhost) + check_cluster_editable(cluster, user=user) + check_cluster_validated(cluster) + utils.update_db_object(session, cluster.state, state='INITIALIZED') + for clusterhost in clusterhosts: + host = clusterhost.host + # ignore checking if underlying host is validated if + # the host is not editable. + if host_api.is_host_editable(host, user=user): + host_api.check_host_validated(host) + utils.update_db_object(session, host.state, state='INITIALIZED') + if cluster.flavor_name: + check_clusterhost_validated(clusterhost) + utils.update_db_object( + session, clusterhost.state, state='INITIALIZED' + ) + + celery_client.celery.send_task( + 'compass.tasks.deploy_cluster', + ( + user.email, cluster_id, + [clusterhost.host_id for clusterhost in clusterhosts] + ), + queue=user.email, + exchange=user.email, + routing_key=user.email + ) + return { + 'status': 'deploy action sent', + 'cluster': cluster, + 'hosts': clusterhosts + } + + +@utils.supported_filters(optional_support_keys=['redeploy']) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEPLOY_CLUSTER +) +@utils.wrap_to_dict( + RESP_DEPLOY_FIELDS, + cluster=RESP_CONFIG_FIELDS, + hosts=RESP_CLUSTERHOST_FIELDS +) +def redeploy_cluster( + cluster_id, deploy={}, user=None, session=None, **kwargs +): + """redeploy cluster. + + Args: + cluster_id: cluster id. + """ + from compass.db.api import host as host_api + from compass.tasks import client as celery_client + cluster = _get_cluster(cluster_id, session=session) + + check_cluster_editable(cluster, user=user) + check_cluster_validated(cluster) + utils.update_db_object( + session, cluster.state, + state='INITIALIZED', + percentage=0, + ready=False + ) + for clusterhost in cluster.clusterhosts: + host = clusterhost.host + # ignore checking if underlying host is validated if + # the host is not editable. + host_api.check_host_validated(host) + utils.update_db_object( + session, host.state, + state='INITIALIZED', + percentage=0, + ready=False + ) + if cluster.flavor_name: + check_clusterhost_validated(clusterhost) + utils.update_db_object( + session, + clusterhost.state, + state='INITIALIZED', + percentage=0, + ready=False + ) + + celery_client.celery.send_task( + 'compass.tasks.redeploy_cluster', + ( + user.email, cluster_id + ), + queue=user.email, + exchange=user.email, + routing_key=user.email + ) + return { + 'status': 'redeploy action sent', + 'cluster': cluster + } + + +@utils.supported_filters(optional_support_keys=['apply_patch']) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEPLOY_CLUSTER +) +@utils.wrap_to_dict( + RESP_DEPLOY_FIELDS, + cluster=RESP_CONFIG_FIELDS, + hosts=RESP_CLUSTERHOST_FIELDS +) +def patch_cluster(cluster_id, user=None, session=None, **kwargs): + + from compass.tasks import client as celery_client + + cluster = _get_cluster(cluster_id, session=session) + celery_client.celery.send_task( + 'compass.tasks.patch_cluster', + ( + user.email, cluster_id, + ), + queue=user.email, + exchange=user.email, + routing_key=user.email + ) + return { + 'status': 'patch action sent', + 'cluster': cluster + } + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_GET_CLUSTER_STATE +) +@utils.wrap_to_dict(RESP_STATE_FIELDS) +def get_cluster_state(cluster_id, user=None, session=None, **kwargs): + """Get cluster state info.""" + return _get_cluster(cluster_id, session=session).state_dict() + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_GET_CLUSTERHOST_STATE +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS) +def get_cluster_host_state( + cluster_id, host_id, user=None, session=None, **kwargs +): + """Get clusterhost state merged with underlying host state.""" + return _get_cluster_host( + cluster_id, host_id, session=session + ).state_dict() + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_GET_CLUSTERHOST_STATE +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS) +def get_cluster_host_self_state( + cluster_id, host_id, user=None, session=None, **kwargs +): + """Get clusterhost itself state.""" + return _get_cluster_host( + cluster_id, host_id, session=session + ).state + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_GET_CLUSTERHOST_STATE +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS) +def get_clusterhost_state( + clusterhost_id, user=None, session=None, **kwargs +): + """Get clusterhost state merged with underlying host state.""" + return _get_clusterhost( + clusterhost_id, session=session + ).state_dict() + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_GET_CLUSTERHOST_STATE +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS) +def get_clusterhost_self_state( + clusterhost_id, user=None, session=None, **kwargs +): + """Get clusterhost itself state.""" + return _get_clusterhost( + clusterhost_id, session=session + ).state + + +@utils.supported_filters( + optional_support_keys=UPDATED_CLUSTERHOST_STATE_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_CLUSTERHOST_STATE +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS) +def update_cluster_host_state( + cluster_id, host_id, user=None, session=None, **kwargs +): + """Update a clusterhost itself state.""" + clusterhost = _get_cluster_host( + cluster_id, host_id, session=session + ) + # Modify(harry): without progress_update.py to update cluster state + # update cluster state here + cluster = _get_cluster(clusterhost.cluster_id, session=session) + utils.update_db_object(session, clusterhost.state, **kwargs) + utils.update_db_object(session, cluster.state, **kwargs) + return clusterhost.state_dict() + + +def _update_clusterhost_state( + clusterhost, from_database_only=False, + session=None, user=None, **kwargs +): + """Update clusterhost state. + + If from_database_only, the state will only be updated in database. + Otherwise a task sent to celery and os installer/package installer + will also update its state if needed. + """ + if 'ready' in kwargs and kwargs['ready'] and not clusterhost.state.ready: + ready_triggered = True + else: + ready_triggered = False + cluster_ready = False + host = clusterhost.host + cluster = clusterhost.cluster + host_ready = not host.state.ready + if ready_triggered: + cluster_ready = True + for clusterhost_in_cluster in cluster.clusterhosts: + if ( + clusterhost_in_cluster.clusterhost_id + == clusterhost.clusterhost_id + ): + continue + if not clusterhost_in_cluster.state.ready: + cluster_ready = False + + logging.info( + 'clusterhost %s ready: %s', + clusterhost.name, ready_triggered + ) + logging.info('cluster ready: %s', cluster_ready) + logging.info('host ready: %s', host_ready) + if not ready_triggered or from_database_only: + logging.info('%s state is set to %s', clusterhost.name, kwargs) + utils.update_db_object(session, clusterhost.state, **kwargs) + if not clusterhost.state.ready: + logging.info('%s state ready is set to False', cluster.name) + utils.update_db_object(session, cluster.state, ready=False) + status = '%s state is updated' % clusterhost.name + else: + if not user: + user_id = cluster.creator_id + user_dict = user_api.get_user(user_id, session=session) + user_email = user_dict['email'] + else: + user_email = user.email + from compass.tasks import client as celery_client + celery_client.celery.send_task( + 'compass.tasks.package_installed', + ( + clusterhost.cluster_id, clusterhost.host_id, + cluster_ready, host_ready + ), + queue=user_email, + exchange=user_email, + routing_key=user_email + ) + status = '%s: cluster ready %s host ready %s' % ( + clusterhost.name, cluster_ready, host_ready + ) + logging.info('action status: %s', status) + return { + 'status': status, + 'clusterhost': clusterhost.state_dict() + } + + +@util.deprecated +@utils.supported_filters( + optional_support_keys=UPDATED_CLUSTERHOST_STATE_INTERNAL_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_CLUSTERHOST_STATE +) +@utils.wrap_to_dict(['status', 'clusterhost']) +def update_cluster_host_state_internal( + cluster_id, host_id, from_database_only=False, + user=None, session=None, **kwargs +): + """Update a clusterhost state by installation process.""" + # TODO(xicheng): it should be merged into update_cluster_host_state + clusterhost = _get_cluster_host( + cluster_id, host_id, session=session + ) + return _update_clusterhost_state( + clusterhost, from_database_only=from_database_only, + session=session, users=user, **kwargs + ) + + +@utils.supported_filters( + optional_support_keys=UPDATED_CLUSTERHOST_STATE_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_CLUSTERHOST_STATE +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS) +def update_clusterhost_state( + clusterhost_id, user=None, session=None, **kwargs +): + """Update a clusterhost itself state.""" + clusterhost = _get_clusterhost( + clusterhost_id, session=session + ) + # Modify(harry): without progress_update.py to update cluster state + # update cluster state here + cluster = _get_cluster(clusterhost.cluster_id, session=session) + utils.update_db_object(session, clusterhost.state, **kwargs) + utils.update_db_object(session, cluster.state, **kwargs) + return clusterhost.state_dict() + + +@util.deprecated +@utils.supported_filters( + optional_support_keys=UPDATED_CLUSTERHOST_STATE_INTERNAL_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_CLUSTERHOST_STATE +) +@utils.wrap_to_dict(['status', 'clusterhost']) +def update_clusterhost_state_internal( + clusterhost_id, from_database_only=False, + user=None, session=None, **kwargs +): + """Update a clusterhost state by installation process.""" + # TODO(xicheng): it should be merged into update_clusterhost_state + clusterhost = _get_clusterhost(clusterhost_id, session=session) + return _update_clusterhost_state( + clusterhost, from_database_only=from_database_only, + session=session, user=user, **kwargs + ) + + +@utils.supported_filters( + optional_support_keys=UPDATED_CLUSTER_STATE_FIELDS, + ignore_support_keys=(IGNORE_FIELDS + IGNORE_UPDATED_CLUSTER_STATE_FIELDS) +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_CLUSTER_STATE +) +@utils.wrap_to_dict(RESP_STATE_FIELDS) +def update_cluster_state( + cluster_id, user=None, session=None, **kwargs +): + """Update a cluster state.""" + cluster = _get_cluster( + cluster_id, session=session + ) + utils.update_db_object(session, cluster.state, **kwargs) + return cluster.state_dict() + + +@util.deprecated +@utils.supported_filters( + optional_support_keys=UPDATED_CLUSTER_STATE_INTERNAL_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_CLUSTER_STATE +) +@utils.wrap_to_dict(['status', 'cluster']) +def update_cluster_state_internal( + cluster_id, from_database_only=False, + user=None, session=None, **kwargs +): + """Update a cluster state by installation process. + + If from_database_only, the state will only be updated in database. + Otherwise a task sent to do state update in os installer and + package installer. + """ + # TODO(xicheng): it should be merged into update_cluster_state + cluster = _get_cluster(cluster_id, session=session) + if 'ready' in kwargs and kwargs['ready'] and not cluster.state.ready: + ready_triggered = True + else: + ready_triggered = False + clusterhost_ready = {} + if ready_triggered: + for clusterhost in cluster.clusterhosts: + clusterhost_ready[clusterhost.host_id] = ( + not clusterhost.state.ready + ) + + logging.info('cluster %s ready: %s', cluster_id, ready_triggered) + logging.info('clusterhost ready: %s', clusterhost_ready) + + if not ready_triggered or from_database_only: + logging.info('%s state is set to %s', cluster.name, kwargs) + utils.update_db_object(session, cluster.state, **kwargs) + if not cluster.state.ready: + for clusterhost in cluster.clusterhosts: + logging.info('%s state ready is to False', clusterhost.name) + utils.update_db_object( + session, clusterhost.state, ready=False + ) + status = '%s state is updated' % cluster.name + else: + if not user: + user_id = cluster.creator_id + user_dict = user_api.get_user(user_id, session=session) + user_email = user_dict['email'] + else: + user_email = user.email + from compass.tasks import client as celery_client + celery_client.celery.send_task( + 'compass.tasks.cluster_installed', + (clusterhost.cluster_id, clusterhost_ready), + queue=user_email, + exchange=user_email, + routing_key=user_email + ) + status = '%s installed action set clusterhost ready %s' % ( + cluster.name, clusterhost_ready + ) + logging.info('action status: %s', status) + return { + 'status': status, + 'cluster': cluster.state_dict() + } + + +@utils.supported_filters([]) +@database.run_in_session() +@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) +def get_cluster_host_log_histories( + cluster_id, host_id, user=None, session=None, **kwargs +): + """Get clusterhost log history by cluster id and host id.""" + return _get_cluster_host( + cluster_id, host_id, session=session + ).log_histories + + +@utils.supported_filters([]) +@database.run_in_session() +@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) +def get_clusterhost_log_histories( + clusterhost_id, user=None, + session=None, **kwargs +): + """Get clusterhost log history by clusterhost id.""" + return _get_clusterhost( + clusterhost_id, session=session + ).log_histories + + +def _get_cluster_host_log_history( + cluster_id, host_id, filename, session=None, **kwargs +): + """Get clusterhost log history by cluster id, host id and filename.""" + clusterhost = _get_cluster_host(cluster_id, host_id, session=session) + return utils.get_db_object( + session, models.ClusterHostLogHistory, + clusterhost_id=clusterhost.clusterhost_id, filename=filename, + **kwargs + ) + + +def _get_clusterhost_log_history( + clusterhost_id, filename, session=None, **kwargs +): + """Get clusterhost log history by clusterhost id and filename.""" + clusterhost = _get_clusterhost(clusterhost_id, session=session) + return utils.get_db_object( + session, models.ClusterHostLogHistory, + clusterhost_id=clusterhost.clusterhost_id, filename=filename, + **kwargs + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) +def get_cluster_host_log_history( + cluster_id, host_id, filename, user=None, session=None, **kwargs +): + """Get clusterhost log history by cluster id, host id and filename.""" + return _get_cluster_host_log_history( + cluster_id, host_id, filename, session=session + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) +def get_clusterhost_log_history( + clusterhost_id, filename, user=None, session=None, **kwargs +): + """Get host log history by clusterhost id and filename.""" + return _get_clusterhost_log_history( + clusterhost_id, filename, session=session + ) + + +@utils.supported_filters( + optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) +def update_cluster_host_log_history( + cluster_id, host_id, filename, user=None, session=None, **kwargs +): + """Update a host log history by cluster id, host id and filename.""" + cluster_host_log_history = _get_cluster_host_log_history( + cluster_id, host_id, filename, session=session + ) + return utils.update_db_object( + session, cluster_host_log_history, **kwargs + ) + + +@utils.supported_filters( + optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) +def update_clusterhost_log_history( + clusterhost_id, filename, user=None, session=None, **kwargs +): + """Update a host log history by clusterhost id and filename.""" + clusterhost_log_history = _get_clusterhost_log_history( + clusterhost_id, filename, session=session + ) + return utils.update_db_object(session, clusterhost_log_history, **kwargs) + + +@utils.supported_filters( + ADDED_CLUSTERHOST_LOG_FIELDS, + optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) +def add_clusterhost_log_history( + clusterhost_id, exception_when_existing=False, + filename=None, user=None, session=None, **kwargs +): + """add a host log history by clusterhost id and filename.""" + clusterhost = _get_clusterhost(clusterhost_id, session=session) + return utils.add_db_object( + session, models.ClusterHostLogHistory, + exception_when_existing, + clusterhost.clusterhost_id, filename, **kwargs + ) + + +@utils.supported_filters( + ADDED_CLUSTERHOST_LOG_FIELDS, + optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) +def add_cluster_host_log_history( + cluster_id, host_id, exception_when_existing=False, + filename=None, user=None, session=None, **kwargs +): + """add a host log history by cluster id, host id and filename.""" + clusterhost = _get_cluster_host( + cluster_id, host_id, session=session + ) + return utils.add_db_object( + session, models.ClusterHostLogHistory, exception_when_existing, + clusterhost.clusterhost_id, filename, **kwargs + ) diff --git a/compass-tasks-base/db/api/database.py b/compass-tasks-base/db/api/database.py new file mode 100644 index 0000000..49769d7 --- /dev/null +++ b/compass-tasks-base/db/api/database.py @@ -0,0 +1,264 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provider interface to manipulate database.""" +import functools +import logging +import netaddr + +from contextlib import contextmanager +from sqlalchemy import create_engine +from sqlalchemy.exc import IntegrityError +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import scoped_session +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import NullPool +from sqlalchemy.pool import QueuePool +from sqlalchemy.pool import SingletonThreadPool +from sqlalchemy.pool import StaticPool +from threading import local + +from compass.db import exception +from compass.db import models +from compass.utils import logsetting +from compass.utils import setting_wrapper as setting + + +ENGINE = None +SESSION = sessionmaker(autocommit=False, autoflush=False) +SCOPED_SESSION = None +SESSION_HOLDER = local() + +POOL_MAPPING = { + 'instant': NullPool, + 'static': StaticPool, + 'queued': QueuePool, + 'thread_single': SingletonThreadPool +} + + +def init(database_url=None): + """Initialize database. + + Adjust sqlalchemy logging if necessary. + + :param database_url: string, database url. + """ + global ENGINE + global SCOPED_SESSION + if not database_url: + database_url = setting.SQLALCHEMY_DATABASE_URI + logging.info('init database %s', database_url) + root_logger = logging.getLogger() + fine_debug = root_logger.isEnabledFor(logsetting.LOGLEVEL_MAPPING['fine']) + if fine_debug: + logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + finest_debug = root_logger.isEnabledFor( + logsetting.LOGLEVEL_MAPPING['finest'] + ) + if finest_debug: + logging.getLogger('sqlalchemy.dialects').setLevel(logging.INFO) + logging.getLogger('sqlalchemy.pool').setLevel(logging.INFO) + logging.getLogger('sqlalchemy.orm').setLevel(logging.INFO) + poolclass = POOL_MAPPING[setting.SQLALCHEMY_DATABASE_POOL_TYPE] + ENGINE = create_engine( + database_url, convert_unicode=True, + poolclass=poolclass + ) + SESSION.configure(bind=ENGINE) + SCOPED_SESSION = scoped_session(SESSION) + models.BASE.query = SCOPED_SESSION.query_property() + + +def in_session(): + """check if in database session scope.""" + bool(hasattr(SESSION_HOLDER, 'session')) + + +@contextmanager +def session(exception_when_in_session=True): + """database session scope. + + To operate database, it should be called in database session. + If not exception_when_in_session, the with session statement support + nested session and only the out most session commit/rollback the + transaction. + """ + if not ENGINE: + init() + + nested_session = False + if hasattr(SESSION_HOLDER, 'session'): + if exception_when_in_session: + logging.error('we are already in session') + raise exception.DatabaseException('session already exist') + else: + new_session = SESSION_HOLDER.session + nested_session = True + logging.log( + logsetting.getLevelByName('fine'), + 'reuse session %s', nested_session + ) + else: + new_session = SCOPED_SESSION() + setattr(SESSION_HOLDER, 'session', new_session) + logging.log( + logsetting.getLevelByName('fine'), + 'enter session %s', new_session + ) + try: + yield new_session + if not nested_session: + new_session.commit() + except Exception as error: + if not nested_session: + new_session.rollback() + logging.error('failed to commit session') + logging.exception(error) + if isinstance(error, IntegrityError): + for item in error.statement.split(): + if item.islower(): + object = item + break + raise exception.DuplicatedRecord( + '%s in %s' % (error.orig, object) + ) + elif isinstance(error, OperationalError): + raise exception.DatabaseException( + 'operation error in database' + ) + elif isinstance(error, exception.DatabaseException): + raise error + else: + raise exception.DatabaseException(str(error)) + finally: + if not nested_session: + new_session.close() + SCOPED_SESSION.remove() + delattr(SESSION_HOLDER, 'session') + logging.log( + logsetting.getLevelByName('fine'), + 'exit session %s', new_session + ) + + +def current_session(): + """Get the current session scope when it is called. + + :return: database session. + :raises: DatabaseException when it is not in session. + """ + try: + return SESSION_HOLDER.session + except Exception as error: + logging.error('It is not in the session scope') + logging.exception(error) + if isinstance(error, exception.DatabaseException): + raise error + else: + raise exception.DatabaseException(str(error)) + + +def run_in_session(exception_when_in_session=True): + """Decorator to make sure the decorated function run in session. + + When not exception_when_in_session, the run_in_session can be + decorated several times. + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + my_session = kwargs.get('session') + if my_session is not None: + return func(*args, **kwargs) + else: + with session( + exception_when_in_session=exception_when_in_session + ) as my_session: + kwargs['session'] = my_session + return func(*args, **kwargs) + except Exception as error: + logging.error( + 'got exception with func %s args %s kwargs %s', + func, args, kwargs + ) + logging.exception(error) + raise error + return wrapper + return decorator + + +def _setup_user_table(user_session): + """Initialize user table with default user.""" + logging.info('setup user table') + from compass.db.api import user + user.add_user( + session=user_session, + email=setting.COMPASS_ADMIN_EMAIL, + password=setting.COMPASS_ADMIN_PASSWORD, + is_admin=True + ) + + +def _setup_permission_table(permission_session): + """Initialize permission table.""" + logging.info('setup permission table.') + from compass.db.api import permission + permission.add_permissions_internal( + session=permission_session + ) + + +def _setup_switch_table(switch_session): + """Initialize switch table.""" + # TODO(xicheng): deprecate setup default switch. + logging.info('setup switch table') + from compass.db.api import switch + switch.add_switch( + True, setting.DEFAULT_SWITCH_IP, + session=switch_session, + machine_filters=['allow ports all'] + ) + + +def _update_others(other_session): + """Update other tables.""" + logging.info('update other tables') + from compass.db.api import utils + from compass.db import models + utils.update_db_objects( + other_session, models.Cluster + ) + utils.update_db_objects( + other_session, models.Host + ) + utils.update_db_objects( + other_session, models.ClusterHost + ) + + +@run_in_session() +def create_db(session=None): + """Create database.""" + models.BASE.metadata.create_all(bind=ENGINE) + _setup_permission_table(session) + _setup_user_table(session) + _setup_switch_table(session) + _update_others(session) + + +def drop_db(): + """Drop database.""" + models.BASE.metadata.drop_all(bind=ENGINE) diff --git a/compass-tasks-base/db/api/health_check_report.py b/compass-tasks-base/db/api/health_check_report.py new file mode 100644 index 0000000..aaea7a7 --- /dev/null +++ b/compass-tasks-base/db/api/health_check_report.py @@ -0,0 +1,190 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, + +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Cluster health check report.""" +import logging + +from compass.db.api import cluster as cluster_api +from compass.db.api import database +from compass.db.api import host as host_api +from compass.db.api import permission +from compass.db.api import user as user_api +from compass.db.api import utils +from compass.db import exception +from compass.db import models + + +REQUIRED_INSERT_FIELDS = ['name'] +OPTIONAL_INSERT_FIELDS = [ + 'display_name', 'report', 'category', 'state', 'error_message' +] +UPDATE_FIELDS = ['report', 'state', 'error_message'] +RESP_FIELDS = [ + 'cluster_id', 'name', 'display_name', 'report', + 'category', 'state', 'error_message' +] +RESP_ACTION_FIELDS = ['cluster_id', 'status'] + + +@utils.supported_filters(REQUIRED_INSERT_FIELDS, OPTIONAL_INSERT_FIELDS) +@database.run_in_session() +@utils.wrap_to_dict(RESP_FIELDS) +def add_report_record(cluster_id, name=None, report={}, + state='verifying', session=None, **kwargs): + """Create a health check report record.""" + # Replace any white space into '-' + words = name.split() + name = '-'.join(words) + cluster = cluster_api.get_cluster_internal(cluster_id, session=session) + return utils.add_db_object( + session, models.HealthCheckReport, True, cluster.id, name, + report=report, state=state, **kwargs + ) + + +def _get_report(cluster_id, name, session=None): + cluster = cluster_api.get_cluster_internal(cluster_id, session=session) + return utils.get_db_object( + session, models.HealthCheckReport, cluster_id=cluster.id, name=name + ) + + +@utils.supported_filters(UPDATE_FIELDS) +@database.run_in_session() +@utils.wrap_to_dict(RESP_FIELDS) +def update_report(cluster_id, name, session=None, **kwargs): + """Update health check report.""" + report = _get_report(cluster_id, name, session=session) + if report.state == 'finished': + err_msg = 'Report cannot be updated if state is in "finished"' + raise exception.Forbidden(err_msg) + + return utils.update_db_object(session, report, **kwargs) + + +@utils.supported_filters(UPDATE_FIELDS) +@database.run_in_session() +@utils.wrap_to_dict(RESP_FIELDS) +def update_multi_reports(cluster_id, session=None, **kwargs): + """Bulk update reports.""" + # TODO(grace): rename the fuction if needed to reflect the fact. + return set_error(cluster_id, session=session, **kwargs) + + +def set_error(cluster_id, report={}, session=None, + state='error', error_message=None): + cluster = cluster_api.get_cluster_internal(cluster_id, session=session) + logging.debug( + "updates all reports as %s in cluster %s", + state, cluster_id + ) + return utils.update_db_objects( + session, models.HealthCheckReport, + updates={ + 'report': {}, + 'state': 'error', + 'error_message': error_message + }, cluster_id=cluster.id + ) + + +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_HEALTH_REPORT +) +@utils.wrap_to_dict(RESP_FIELDS) +def list_health_reports(cluster_id, user=None, session=None): + """List all reports in the specified cluster.""" + cluster = cluster_api.get_cluster_internal(cluster_id, session=session) + return utils.list_db_objects( + session, models.HealthCheckReport, cluster_id=cluster.id + ) + + +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_GET_HEALTH_REPORT +) +@utils.wrap_to_dict(RESP_FIELDS) +def get_health_report(cluster_id, name, user=None, session=None): + return _get_report( + cluster_id, name, session=session + ) + + +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DELETE_REPORT +) +@utils.wrap_to_dict(RESP_FIELDS) +def delete_reports(cluster_id, name=None, user=None, session=None): + # TODO(grace): better to separate this function into two. + # One is to delete a report of a cluster, the other to delete all + # reports under a cluster. + if name: + report = _get_report(cluster_id, name, session=session) + return utils.del_db_object(session, report) + else: + cluster = cluster_api.get_cluster_internal( + cluster_id, session=session + ) + return utils.del_db_objects( + session, models.HealthCheckReport, cluster_id=cluster.id + ) + + +@utils.supported_filters(optional_support_keys=['check_health']) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_CHECK_CLUSTER_HEALTH +) +@utils.wrap_to_dict(RESP_ACTION_FIELDS) +def start_check_cluster_health(cluster_id, send_report_url, + user=None, session=None, check_health={}): + """Start to check cluster health.""" + cluster = cluster_api.get_cluster_internal(cluster_id, session=session) + + if cluster.state.state != 'SUCCESSFUL': + logging.debug("state is %s" % cluster.state.state) + err_msg = "Healthcheck starts only after cluster finished deployment!" + raise exception.Forbidden(err_msg) + + reports = utils.list_db_objects( + session, models.HealthCheckReport, + cluster_id=cluster.id, state='verifying' + ) + if reports: + err_msg = 'Healthcheck in progress, please wait for it to complete!' + raise exception.Forbidden(err_msg) + + # Clear all preivous report + # TODO(grace): the delete should be moved into celery task. + # We should consider the case that celery task is down. + utils.del_db_objects( + session, models.HealthCheckReport, cluster_id=cluster.id + ) + + from compass.tasks import client as celery_client + celery_client.celery.send_task( + 'compass.tasks.cluster_health', + (cluster.id, send_report_url, user.email), + queue=user.email, + exchange=user.email, + routing_key=user.email + ) + return { + "cluster_id": cluster.id, + "status": "start to check cluster health." + } diff --git a/compass-tasks-base/db/api/host.py b/compass-tasks-base/db/api/host.py new file mode 100644 index 0000000..15e0bb6 --- /dev/null +++ b/compass-tasks-base/db/api/host.py @@ -0,0 +1,1120 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Host database operations.""" +import functools +import logging +import netaddr +import re + +from compass.db.api import database +from compass.db.api import metadata_holder as metadata_api +from compass.db.api import permission +from compass.db.api import user as user_api +from compass.db.api import utils +from compass.db import exception +from compass.db import models +from compass.utils import util + + +SUPPORTED_FIELDS = ['name', 'os_name', 'owner', 'mac', 'id'] +SUPPORTED_MACHINE_HOST_FIELDS = [ + 'mac', 'tag', 'location', 'os_name', 'os_id' +] +SUPPORTED_NETOWORK_FIELDS = [ + 'interface', 'ip', 'is_mgmt', 'is_promiscuous' +] +RESP_FIELDS = [ + 'id', 'name', 'hostname', 'os_name', 'owner', 'mac', + 'switch_ip', 'port', 'switches', 'os_installer', 'os_id', 'ip', + 'reinstall_os', 'os_installed', 'tag', 'location', 'networks', + 'created_at', 'updated_at' +] +RESP_CLUSTER_FIELDS = [ + 'id', 'name', 'os_name', 'reinstall_distributed_system', + 'owner', 'adapter_name', 'flavor_name', + 'distributed_system_installed', 'created_at', 'updated_at' +] +RESP_NETWORK_FIELDS = [ + 'id', 'ip', 'interface', 'netmask', 'is_mgmt', 'is_promiscuous', + 'created_at', 'updated_at' +] +RESP_CONFIG_FIELDS = [ + 'os_config', + 'config_setp', + 'config_validated', + 'networks', + 'created_at', + 'updated_at' +] +RESP_DEPLOYED_CONFIG_FIELDS = [ + 'deployed_os_config' +] +RESP_DEPLOY_FIELDS = [ + 'status', 'host' +] +UPDATED_FIELDS = ['name', 'reinstall_os'] +UPDATED_CONFIG_FIELDS = [ + 'put_os_config' +] +PATCHED_CONFIG_FIELDS = [ + 'patched_os_config' +] +UPDATED_DEPLOYED_CONFIG_FIELDS = [ + 'deployed_os_config' +] +ADDED_NETWORK_FIELDS = [ + 'interface', 'ip', 'subnet_id' +] +OPTIONAL_ADDED_NETWORK_FIELDS = ['is_mgmt', 'is_promiscuous'] +UPDATED_NETWORK_FIELDS = [ + 'interface', 'ip', 'subnet_id', 'subnet', 'is_mgmt', + 'is_promiscuous' +] +IGNORE_FIELDS = [ + 'id', 'created_at', 'updated_at' +] +RESP_STATE_FIELDS = [ + 'id', 'state', 'percentage', 'message', 'severity', 'ready' +] +UPDATED_STATE_FIELDS = [ + 'state', 'percentage', 'message', 'severity' +] +UPDATED_STATE_INTERNAL_FIELDS = [ + 'ready' +] +RESP_LOG_FIELDS = [ + 'id', 'filename', 'position', 'partial_line', 'percentage', + 'message', 'severity', 'line_matcher_name' +] +ADDED_LOG_FIELDS = [ + 'filename' +] +UPDATED_LOG_FIELDS = [ + 'position', 'partial_line', 'percentage', + 'message', 'severity', 'line_matcher_name' +] + + +@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_HOSTS +) +@utils.wrap_to_dict(RESP_FIELDS) +def list_hosts(user=None, session=None, **filters): + """List hosts.""" + return utils.list_db_objects( + session, models.Host, **filters + ) + + +@utils.supported_filters( + optional_support_keys=SUPPORTED_MACHINE_HOST_FIELDS) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_HOSTS +) +@utils.output_filters( + missing_ok=True, + tag=utils.general_filter_callback, + location=utils.general_filter_callback, + os_name=utils.general_filter_callback, + os_id=utils.general_filter_callback +) +@utils.wrap_to_dict(RESP_FIELDS) +def list_machines_or_hosts(user=None, session=None, **filters): + """List machines or hosts if possible.""" + machines = utils.list_db_objects( + session, models.Machine, **filters + ) + machines_or_hosts = [] + for machine in machines: + host = machine.host + if host: + machines_or_hosts.append(host) + else: + machines_or_hosts.append(machine) + return machines_or_hosts + + +def _get_host(host_id, session=None, **kwargs): + """Get host by id.""" + if isinstance(host_id, (int, long)): + return utils.get_db_object( + session, models.Host, + id=host_id, **kwargs + ) + else: + raise exception.InvalidParameter( + 'host id %s type is not int compatible' % host_id + ) + + +def get_host_internal(host_id, session=None, **kwargs): + """Helper function to get host. + + Used by other files under db/api. + """ + return _get_host(host_id, session=session, **kwargs) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_HOSTS +) +@utils.wrap_to_dict(RESP_FIELDS) +def get_host( + host_id, exception_when_missing=True, + user=None, session=None, **kwargs +): + """get host info.""" + return _get_host( + host_id, + exception_when_missing=exception_when_missing, + session=session + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_HOSTS +) +@utils.wrap_to_dict(RESP_FIELDS) +def get_machine_or_host( + host_id, exception_when_missing=True, + user=None, session=None, **kwargs +): + """get machine or host if possible.""" + from compass.db.api import machine as machine_api + machine = machine_api.get_machine_internal( + host_id, + exception_when_missing=exception_when_missing, + session=session + ) + if machine.host: + return machine.host + else: + return machine + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_HOST_CLUSTERS +) +@utils.wrap_to_dict(RESP_CLUSTER_FIELDS) +def get_host_clusters(host_id, user=None, session=None, **kwargs): + """get host clusters.""" + host = _get_host(host_id, session=session) + return [clusterhost.cluster for clusterhost in host.clusterhosts] + + +def check_host_validated(host): + """Check host is validated.""" + if not host.config_validated: + raise exception.Forbidden( + 'host %s is not validated' % host.name + ) + + +def check_host_editable( + host, user=None, + check_in_installing=False +): + """Check host is editable. + + If we try to set reinstall_os or check the host is not in installing + state, we should set check_in_installing to True. + Otherwise we will check the host is not in installing or installed. + We also make sure the user is admin or the owner of the host to avoid + unauthorized user to update host attributes. + """ + if check_in_installing: + if host.state.state == 'INSTALLING': + raise exception.Forbidden( + 'host %s is not editable ' + 'when state is in installing' % host.name + ) + elif not host.reinstall_os: + raise exception.Forbidden( + 'host %s is not editable ' + 'when not to be reinstalled' % host.name + ) + if user and not user.is_admin and host.creator_id != user.id: + raise exception.Forbidden( + 'host %s is not editable ' + 'when user is not admin or the owner of the host' % host.name + ) + + +def is_host_editable( + host, user=None, + check_in_installing=False +): + """Get if host is editable.""" + try: + check_host_editable( + host, user=user, + check_in_installing=check_in_installing + ) + return True + except exception.Forbidden: + return False + + +def validate_host(host): + """Validate host. + + Makesure hostname is not empty, there is only one mgmt network, + The mgmt network is not in promiscuous mode. + """ + if not host.hostname: + raise exception.Invalidparameter( + 'host %s does not set hostname' % host.name + ) + if not host.host_networks: + raise exception.InvalidParameter( + 'host %s does not have any network' % host.name + ) + mgmt_interface_set = False + for host_network in host.host_networks: + if host_network.is_mgmt: + if mgmt_interface_set: + raise exception.InvalidParameter( + 'host %s multi interfaces set mgmt ' % host.name + ) + if host_network.is_promiscuous: + raise exception.InvalidParameter( + 'host %s interface %s is mgmt but promiscuous' % ( + host.name, host_network.interface + ) + ) + mgmt_interface_set = True + if not mgmt_interface_set: + raise exception.InvalidParameter( + 'host %s has no mgmt interface' % host.name + ) + + +@utils.supported_filters( + optional_support_keys=UPDATED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates(name=utils.check_name) +@utils.wrap_to_dict(RESP_FIELDS) +def _update_host(host_id, session=None, user=None, **kwargs): + """Update a host internal.""" + host = _get_host(host_id, session=session) + if host.state.state == "SUCCESSFUL" and not host.reinstall_os: + logging.info("ignoring successful host: %s", host_id) + return {} + check_host_editable( + host, user=user, + check_in_installing=kwargs.get('reinstall_os', False) + ) + return utils.update_db_object(session, host, **kwargs) + + +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_HOST +) +def update_host(host_id, user=None, session=None, **kwargs): + """Update a host.""" + return _update_host(host_id, session=session, user=user, **kwargs) + + +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_HOST +) +def update_hosts(data=[], user=None, session=None): + """Update hosts.""" + # TODO(xicheng): this batch function is not similar as others. + # try to make it similar output as others and batch update should + # tolerate partial failure. + hosts = [] + for host_data in data: + hosts.append(_update_host(session=session, user=user, **host_data)) + return hosts + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEL_HOST +) +@utils.wrap_to_dict( + RESP_FIELDS + ['status', 'host'], + host=RESP_FIELDS +) +def del_host( + host_id, force=False, from_database_only=False, + user=None, session=None, **kwargs +): + """Delete a host. + + If force, we delete the host anyway. + If from_database_only, we only delete the host record in databaes. + Otherwise we send to del host task to celery to delete the host + record in os installer and package installer, clean installation logs + and at last clean database record. + The backend will call this function again after it deletes the record + in os installer and package installer with from_database_only set. + """ + from compass.db.api import cluster as cluster_api + host = _get_host(host_id, session=session) + # force set host state to ERROR when we want to delete the + # host anyway even the host is in installing or already + # installed. It let the api know the deleting is in doing when backend + # is doing the real deleting. In future we may import a new state like + # INDELETE to indicate the deleting is processing. + # We need discuss about if we can delete a host when it is already + # installed by api. + if host.state.state != 'UNINITIALIZED' and force: + host.state.state = 'ERROR' + check_host_editable( + host, user=user, + check_in_installing=True + ) + cluster_ids = [] + for clusterhost in host.clusterhosts: + if clusterhost.state.state != 'UNINITIALIZED' and force: + clusterhost.state.state = 'ERROR' + # TODO(grace): here we check all clusters which use this host editable. + # Because in backend we do not have functions to delete host without + # reference its cluster. After deleting pure host supported in backend, + # we should change code here to is_cluster_editable. + # Here delete a host may fail even we set force flag. + cluster_api.check_cluster_editable( + clusterhost.cluster, user=user, + check_in_installing=True + ) + cluster_ids.append(clusterhost.cluster_id) + + # Delete host record directly if there is no need to delete it + # in backend or from_database_only is set. + if host.state.state == 'UNINITIALIZED' or from_database_only: + return utils.del_db_object(session, host) + else: + logging.info( + 'send del host %s task to celery', host_id + ) + if not user: + user_id = host.creator_id + user_dict = user_api.get_user(user_id, session=session) + user_email = user_dict['email'] + else: + user_email = user.email + from compass.tasks import client as celery_client + celery_client.celery.send_task( + 'compass.tasks.delete_host', + ( + user.email, host.id, cluster_ids + ), + queue=user_email, + exchange=user_email, + routing_key=user_email + ) + return { + 'status': 'delete action sent', + 'host': host, + } + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_HOST_CONFIG +) +@utils.wrap_to_dict(RESP_CONFIG_FIELDS) +def get_host_config(host_id, user=None, session=None, **kwargs): + """Get host config.""" + return _get_host(host_id, session=session) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_HOST_CONFIG +) +@utils.wrap_to_dict(RESP_DEPLOYED_CONFIG_FIELDS) +def get_host_deployed_config(host_id, user=None, session=None, **kwargs): + """Get host deployed config.""" + return _get_host(host_id, session=session) + + +# replace os_config to deployed_os_config in kwargs. +@utils.replace_filters( + os_config='deployed_os_config' +) +@utils.supported_filters( + UPDATED_DEPLOYED_CONFIG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_HOST_CONFIG +) +@utils.wrap_to_dict(RESP_CONFIG_FIELDS) +def update_host_deployed_config(host_id, user=None, session=None, **kwargs): + """Update host deployed config.""" + host = _get_host(host_id, session=session) + check_host_editable(host, user=user) + check_host_validated(host) + return utils.update_db_object(session, host, **kwargs) + + +def _host_os_config_validates( + config, host, session=None, user=None, **kwargs +): + """Check host os config's validation.""" + metadata_api.validate_os_config( + config, host.os_id + ) + + +@utils.input_validates_with_args( + put_os_config=_host_os_config_validates +) +@utils.output_validates_with_args( + os_config=_host_os_config_validates +) +@utils.wrap_to_dict(RESP_CONFIG_FIELDS) +def _update_host_config(host, session=None, user=None, **kwargs): + """Update host config.""" + check_host_editable(host, user=user) + return utils.update_db_object(session, host, **kwargs) + + +# replace os_config to put_os_config in kwargs. +# It tells db the os_config will be updated not patched. +@utils.replace_filters( + os_config='put_os_config' +) +@utils.supported_filters( + UPDATED_CONFIG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_HOST_CONFIG +) +def update_host_config(host_id, user=None, session=None, **kwargs): + """Update host config.""" + host = _get_host(host_id, session=session) + return _update_host_config( + host, session=session, user=user, **kwargs + ) + + +# replace os_config to patched_os_config in kwargs. +# It tells db os_config will be patched not be updated. +@utils.replace_filters( + os_config='patched_os_config' +) +@utils.supported_filters( + PATCHED_CONFIG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_HOST_CONFIG +) +def patch_host_config(host_id, user=None, session=None, **kwargs): + """Patch host config.""" + host = _get_host(host_id, session=session) + return _update_host_config( + host, session=session, user=user, **kwargs + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEL_HOST_CONFIG +) +@utils.wrap_to_dict(RESP_CONFIG_FIELDS) +def del_host_config(host_id, user=None, session=None): + """delete a host config.""" + host = _get_host(host_id, session=session) + check_host_editable(host, user=user) + return utils.update_db_object( + session, host, os_config={}, config_validated=False + ) + + +@utils.supported_filters( + optional_support_keys=SUPPORTED_NETOWORK_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_HOST_NETWORKS +) +@utils.wrap_to_dict(RESP_NETWORK_FIELDS) +def list_host_networks(host_id, user=None, session=None, **filters): + """Get host networks for a host.""" + host = _get_host(host_id, session=session) + return utils.list_db_objects( + session, models.HostNetwork, + host_id=host.id, **filters + ) + + +@utils.supported_filters( + optional_support_keys=SUPPORTED_NETOWORK_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_HOST_NETWORKS +) +@utils.wrap_to_dict(RESP_NETWORK_FIELDS) +def list_hostnetworks(user=None, session=None, **filters): + """Get host networks.""" + return utils.list_db_objects( + session, models.HostNetwork, **filters + ) + + +def _get_hostnetwork(host_network_id, session=None, **kwargs): + """Get hostnetwork by hostnetwork id.""" + if isinstance(host_network_id, (int, long)): + return utils.get_db_object( + session, models.HostNetwork, + id=host_network_id, **kwargs + ) + raise exception.InvalidParameter( + 'host network id %s type is not int compatible' % host_network_id + ) + + +def _get_host_network(host_id, host_network_id, session=None, **kwargs): + """Get hostnetwork by host id and hostnetwork id.""" + host = _get_host(host_id, session=session) + host_network = _get_hostnetwork(host_network_id, session=session, **kwargs) + if host_network.host_id != host.id: + raise exception.RecordNotExists( + 'host %s does not own host network %s' % ( + host.id, host_network.id + ) + ) + return host_network + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_HOST_NETWORKS +) +@utils.wrap_to_dict(RESP_NETWORK_FIELDS) +def get_host_network( + host_id, host_network_id, + user=None, session=None, **kwargs +): + """Get host network.""" + return _get_host_network( + host_id, host_network_id, session=session + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_HOST_NETWORKS +) +@utils.wrap_to_dict(RESP_NETWORK_FIELDS) +def get_hostnetwork(host_network_id, user=None, session=None, **kwargs): + """Get host network.""" + return _get_hostnetwork(host_network_id, session=session) + + +@utils.supported_filters( + ADDED_NETWORK_FIELDS, + optional_support_keys=OPTIONAL_ADDED_NETWORK_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates( + ip=utils.check_ip +) +@utils.wrap_to_dict(RESP_NETWORK_FIELDS) +def _add_host_network( + host_id, exception_when_existing=True, + session=None, user=None, interface=None, ip=None, **kwargs +): + """Add hostnetwork to a host.""" + host = _get_host(host_id, session=session) + check_host_editable(host, user=user) + user_id = user.id + return utils.add_db_object( + session, models.HostNetwork, + exception_when_existing, + host.id, interface, user_id, ip=ip, **kwargs + ) + + +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_HOST_NETWORK +) +def add_host_network( + host_id, exception_when_existing=True, + interface=None, user=None, session=None, **kwargs +): + """Create a hostnetwork to a host.""" + return _add_host_network( + host_id, + exception_when_existing, + interface=interface, session=session, user=user, **kwargs + ) + + +def _get_hostnetwork_by_ip( + ip, session=None, **kwargs +): + ip_int = long(netaddr.IPAddress(ip)) + return utils.get_db_object( + session, models.HostNetwork, + ip_int=ip_int, **kwargs + ) + + +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_HOST_NETWORK +) +def add_host_networks( + exception_when_existing=False, + data=[], user=None, session=None +): + """Create host networks.""" + hosts = [] + failed_hosts = [] + for host_data in data: + host_id = host_data['host_id'] + host = _get_host(host_id, session=session) + networks = host_data['networks'] + host_networks = [] + failed_host_networks = [] + for network in networks: + host_network = _get_hostnetwork_by_ip( + network['ip'], session=session, + exception_when_missing=False + ) + if ( + host_network and not ( + host_network.host_id == host.id and + host_network.interface == network['interface'] + ) + ): + logging.error('ip %s exists in host network %s' % ( + network['ip'], host_network.id + )) + failed_host_networks.append(network) + else: + host_networks.append(_add_host_network( + host.id, exception_when_existing, + session=session, user=user, **network + )) + if host_networks: + hosts.append({'host_id': host.id, 'networks': host_networks}) + if failed_host_networks: + failed_hosts.append({ + 'host_id': host.id, 'networks': failed_host_networks + }) + return { + 'hosts': hosts, + 'failed_hosts': failed_hosts + } + + +@utils.wrap_to_dict(RESP_NETWORK_FIELDS) +def _update_host_network( + host_network, session=None, user=None, **kwargs +): + """Update host network.""" + check_host_editable(host_network.host, user=user) + return utils.update_db_object(session, host_network, **kwargs) + + +@utils.supported_filters( + optional_support_keys=UPDATED_NETWORK_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates( + ip=utils.check_ip +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_HOST_NETWORK +) +def update_host_network( + host_id, host_network_id, user=None, session=None, **kwargs +): + """Update a host network by host id and host network id.""" + host = _get_host( + host_id, session=session + ) + if host.state.state == "SUCCESSFUL" and not host.reinstall_os: + logging.info("ignoring updating request for successful hosts") + return {} + + host_network = _get_host_network( + host_id, host_network_id, session=session + ) + return _update_host_network( + host_network, session=session, user=user, **kwargs + ) + + +@utils.supported_filters( + optional_support_keys=UPDATED_NETWORK_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates( + ip=utils.check_ip +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_HOST_NETWORK +) +def update_hostnetwork(host_network_id, user=None, session=None, **kwargs): + """Update a host network by host network id.""" + host_network = _get_hostnetwork( + host_network_id, session=session + ) + return _update_host_network( + host_network, session=session, user=user, **kwargs + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEL_HOST_NETWORK +) +@utils.wrap_to_dict(RESP_NETWORK_FIELDS) +def del_host_network( + host_id, host_network_id, user=None, + session=None, **kwargs +): + """Delete a host network by host id and host network id.""" + host_network = _get_host_network( + host_id, host_network_id, session=session + ) + check_host_editable(host_network.host, user=user) + return utils.del_db_object(session, host_network) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEL_HOST_NETWORK +) +@utils.wrap_to_dict(RESP_NETWORK_FIELDS) +def del_hostnetwork(host_network_id, user=None, session=None, **kwargs): + """Delete a host network by host network id.""" + host_network = _get_hostnetwork( + host_network_id, session=session + ) + check_host_editable(host_network.host, user=user) + return utils.del_db_object(session, host_network) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_GET_HOST_STATE +) +@utils.wrap_to_dict(RESP_STATE_FIELDS) +def get_host_state(host_id, user=None, session=None, **kwargs): + """Get host state info.""" + return _get_host(host_id, session=session).state + + +@utils.supported_filters( + optional_support_keys=UPDATED_STATE_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_HOST_STATE +) +@utils.wrap_to_dict(RESP_STATE_FIELDS) +def update_host_state(host_id, user=None, session=None, **kwargs): + """Update a host state.""" + host = _get_host(host_id, session=session) + utils.update_db_object(session, host.state, **kwargs) + return host.state + + +@util.deprecated +@utils.supported_filters( + optional_support_keys=UPDATED_STATE_INTERNAL_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_HOST_STATE +) +@utils.wrap_to_dict(['status', 'host']) +def update_host_state_internal( + host_id, from_database_only=False, + user=None, session=None, **kwargs +): + """Update a host state. + + This function is called when host os is installed. + If from_database_only, the state is updated in database. + Otherwise a celery task sent to os installer and package installer + to do some future actions. + """ + # TODO(xicheng): should be merged into update_host_state + host = _get_host(host_id, session=session) + logging.info("======host state: %s", host.state) + if 'ready' in kwargs and kwargs['ready'] and not host.state.ready: + ready_triggered = True + else: + ready_triggered = False + clusterhosts_ready = {} + clusters_os_ready = {} + if ready_triggered: + for clusterhost in host.clusterhosts: + cluster = clusterhost.cluster + if cluster.flavor_name: + clusterhosts_ready[cluster.id] = False + else: + clusterhosts_ready[cluster.id] = True + all_os_ready = True + for clusterhost_in_cluster in cluster.clusterhosts: + host_in_cluster = clusterhost_in_cluster.host + if host_in_cluster.id == host.id: + continue + if not host_in_cluster.state.ready: + all_os_ready = False + clusters_os_ready[cluster.id] = all_os_ready + logging.debug('host %s ready: %s', host_id, ready_triggered) + logging.debug("clusterhosts_ready is: %s", clusterhosts_ready) + logging.debug("clusters_os_ready is %s", clusters_os_ready) + + if not ready_triggered or from_database_only: + logging.debug('%s state is set to %s', host.name, kwargs) + utils.update_db_object(session, host.state, **kwargs) + if not host.state.ready: + for clusterhost in host.clusterhosts: + utils.update_db_object( + session, clusterhost.state, ready=False + ) + utils.update_db_object( + session, clusterhost.cluster.state, ready=False + ) + status = '%s state is updated' % host.name + else: + if not user: + user_id = host.creator_id + user_dict = user_api.get_user(user_id, session=session) + user_email = user_dict['email'] + else: + user_email = user.email + from compass.tasks import client as celery_client + celery_client.celery.send_task( + 'compass.tasks.os_installed', + ( + host.id, clusterhosts_ready, + clusters_os_ready + ), + queue=user_email, + exchange=user_email, + routing_key=user_email + ) + status = '%s: clusterhosts ready %s clusters os ready %s' % ( + host.name, clusterhosts_ready, clusters_os_ready + ) + logging.info('action status: %s', status) + return { + 'status': status, + 'host': host.state + } + + +@utils.supported_filters([]) +@database.run_in_session() +@utils.wrap_to_dict(RESP_LOG_FIELDS) +def get_host_log_histories(host_id, user=None, session=None, **kwargs): + """Get host log history.""" + host = _get_host(host_id, session=session) + return utils.list_db_objects( + session, models.HostLogHistory, id=host.id, **kwargs + ) + + +def _get_host_log_history(host_id, filename, session=None, **kwargs): + host = _get_host(host_id, session=session) + return utils.get_db_object( + session, models.HostLogHistory, id=host.id, + filename=filename, **kwargs + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@utils.wrap_to_dict(RESP_LOG_FIELDS) +def get_host_log_history(host_id, filename, user=None, session=None, **kwargs): + """Get host log history.""" + return _get_host_log_history( + host_id, filename, session=session + ) + + +@utils.supported_filters( + optional_support_keys=UPDATED_LOG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@utils.wrap_to_dict(RESP_LOG_FIELDS) +def update_host_log_history( + host_id, filename, user=None, + session=None, **kwargs +): + """Update a host log history.""" + host_log_history = _get_host_log_history( + host_id, filename, session=session + ) + return utils.update_db_object(session, host_log_history, **kwargs) + + +@utils.supported_filters( + ADDED_LOG_FIELDS, + optional_support_keys=UPDATED_LOG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@utils.wrap_to_dict(RESP_LOG_FIELDS) +def add_host_log_history( + host_id, exception_when_existing=False, + filename=None, user=None, session=None, **kwargs +): + """add a host log history.""" + host = _get_host(host_id, session=session) + return utils.add_db_object( + session, models.HostLogHistory, exception_when_existing, + host.id, filename, **kwargs + ) + + +@utils.supported_filters(optional_support_keys=['poweron']) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEPLOY_HOST +) +@utils.wrap_to_dict( + RESP_DEPLOY_FIELDS, + host=RESP_CONFIG_FIELDS +) +def poweron_host( + host_id, poweron={}, user=None, session=None, **kwargs +): + """power on host.""" + from compass.tasks import client as celery_client + host = _get_host(host_id, session=session) + check_host_validated(host) + if not user: + user_id = host.creator_id + user_dict = user_api.get_user(user_id, session=session) + user_email = user_dict['email'] + else: + user_email = user.email + celery_client.celery.send_task( + 'compass.tasks.poweron_host', + (host.id,), + queue=user_email, + exchange=user_email, + routing_key=user_email + ) + return { + 'status': 'poweron %s action sent' % host.name, + 'host': host + } + + +@utils.supported_filters(optional_support_keys=['poweroff']) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEPLOY_HOST +) +@utils.wrap_to_dict( + RESP_DEPLOY_FIELDS, + host=RESP_CONFIG_FIELDS +) +def poweroff_host( + host_id, poweroff={}, user=None, session=None, **kwargs +): + """power off host.""" + from compass.tasks import client as celery_client + host = _get_host(host_id, session=session) + check_host_validated(host) + if not user: + user_id = host.creator_id + user_dict = user_api.get_user(user_id, session=session) + user_email = user_dict['email'] + else: + user_email = user.email + celery_client.celery.send_task( + 'compass.tasks.poweroff_host', + (host.id,), + queue=user_email, + exchange=user_email, + routing_key=user_email + ) + return { + 'status': 'poweroff %s action sent' % host.name, + 'host': host + } + + +@utils.supported_filters(optional_support_keys=['reset']) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEPLOY_HOST +) +@utils.wrap_to_dict( + RESP_DEPLOY_FIELDS, + host=RESP_CONFIG_FIELDS +) +def reset_host( + host_id, reset={}, user=None, session=None, **kwargs +): + """reset host.""" + from compass.tasks import client as celery_client + host = _get_host(host_id, session=session) + check_host_validated(host) + if not user: + user_id = host.creator_id + user_dict = user_api.get_user(user_id, session=session) + user_email = user_dict['email'] + else: + user_email = user.email + celery_client.celery.send_task( + 'compass.tasks.reset_host', + (host.id,), + queue=user_email, + exchange=user_email, + routing_key=user_email + ) + return { + 'status': 'reset %s action sent' % host.name, + 'host': host + } diff --git a/compass-tasks-base/db/api/machine.py b/compass-tasks-base/db/api/machine.py new file mode 100644 index 0000000..b7b16b2 --- /dev/null +++ b/compass-tasks-base/db/api/machine.py @@ -0,0 +1,317 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Switch database operations.""" +import logging +import re + +from compass.db.api import database +from compass.db.api import permission +from compass.db.api import user as user_api +from compass.db.api import utils +from compass.db import exception +from compass.db import models + +from compass.utils import setting_wrapper as setting +from compass.utils import util + + +MACHINE_PRIMARY_FILEDS = ['mac', 'owner_id'] +SUPPORTED_FIELDS = [ + 'mac', 'tag', 'location', + 'machine_attributes', 'owner_id'] +IGNORE_FIELDS = ['id', 'created_at', 'updated_at'] +UPDATED_FIELDS = [ + 'ipmi_credentials', 'machine_attributes', + 'tag', 'location'] +PATCHED_FIELDS = [ + 'patched_ipmi_credentials', 'patched_tag', + 'patched_location' +] +RESP_FIELDS = [ + 'id', 'mac', 'ipmi_credentials', 'switches', 'switch_ip', + 'port', 'vlans', 'machine_attributes', 'owner_id', + 'tag', 'location', 'created_at', 'updated_at' +] +RESP_DEPLOY_FIELDS = [ + 'status', 'machine' +] + + +def _get_machine(machine_id, session=None, **kwargs): + """Get machine by id.""" + if isinstance(machine_id, (int, long)): + return utils.get_db_object( + session, models.Machine, + id=machine_id, **kwargs + ) + raise exception.InvalidParameter( + 'machine id %s type is not int compatible' % machine_id + ) + + +@utils.supported_filters( + MACHINE_PRIMARY_FILEDS, + optional_support_keys=SUPPORTED_FIELDS +) +@utils.input_validates(mac=utils.check_mac) +def _add_machine(mac, owner_id=None, session=None, **kwargs): + """Add a machine.""" + if isinstance(owner_id, (int, long)): + return utils.add_db_object( + session, models.Machine, + True, + mac, + owner_id=owner_id, + **kwargs + ) + raise exception.InvalidParameter( + 'owner id %s type is not int compatible' % owner_id + ) + + +@database.run_in_session() +@utils.wrap_to_dict(RESP_FIELDS) +def add_machine( + mac, owner_id=None, session=None, user=None, **kwargs +): + """Add a machine.""" + return _add_machine( + mac, + owner_id=owner_id, + session=session, **kwargs + ) + + +def get_machine_internal(machine_id, session=None, **kwargs): + """Helper function to other files under db/api.""" + return _get_machine(machine_id, session=session, **kwargs) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_MACHINES +) +@utils.wrap_to_dict(RESP_FIELDS) +def get_machine( + machine_id, exception_when_missing=True, + user=None, session=None, **kwargs +): + """get a machine.""" + return _get_machine( + machine_id, session=session, + exception_when_missing=exception_when_missing + ) + + +@utils.supported_filters( + optional_support_keys=SUPPORTED_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_MACHINES +) +@utils.output_filters( + tag=utils.general_filter_callback, + location=utils.general_filter_callback +) +@utils.wrap_to_dict(RESP_FIELDS) +def list_machines(user=None, session=None, **filters): + """List machines.""" + machines = utils.list_db_objects( + session, models.Machine, **filters + ) + if not user.is_admin and len(machines): + machines = [m for m in machines if m.owner_id == user.id] + return machines + + +@utils.wrap_to_dict(RESP_FIELDS) +def _update_machine(machine_id, session=None, **kwargs): + """Update a machine.""" + machine = _get_machine(machine_id, session=session) + return utils.update_db_object(session, machine, **kwargs) + + +@utils.supported_filters( + optional_support_keys=UPDATED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates(ipmi_credentials=utils.check_ipmi_credentials) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_MACHINE +) +def update_machine(machine_id, user=None, session=None, **kwargs): + """Update a machine.""" + return _update_machine( + machine_id, session=session, **kwargs + ) + + +# replace [ipmi_credentials, tag, location] to +# [patched_ipmi_credentials, patched_tag, patched_location] +# in kwargs. It tells db these fields will be patched. +@utils.replace_filters( + ipmi_credentials='patched_ipmi_credentials', + tag='patched_tag', + location='patched_location' +) +@utils.supported_filters( + optional_support_keys=PATCHED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@utils.output_validates(ipmi_credentials=utils.check_ipmi_credentials) +@user_api.check_user_permission( + permission.PERMISSION_ADD_MACHINE +) +def patch_machine(machine_id, user=None, session=None, **kwargs): + """Patch a machine.""" + return _update_machine( + machine_id, session=session, **kwargs + ) + + +def _check_machine_deletable(machine): + """Check a machine deletable.""" + if machine.host: + host = machine.host + raise exception.NotAcceptable( + 'machine %s has host %s on it' % ( + machine.mac, host.name + ) + ) + + +@utils.supported_filters() +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEL_MACHINE +) +@utils.wrap_to_dict(RESP_FIELDS) +def del_machine(machine_id, user=None, session=None, **kwargs): + """Delete a machine.""" + machine = _get_machine(machine_id, session=session) + _check_machine_deletable(machine) + return utils.del_db_object(session, machine) + + +@utils.supported_filters(optional_support_keys=['poweron']) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEPLOY_HOST +) +@utils.wrap_to_dict( + RESP_DEPLOY_FIELDS, + machine=RESP_FIELDS +) +def poweron_machine( + machine_id, poweron={}, user=None, session=None, **kwargs +): + """power on machine.""" + from compass.tasks import client as celery_client + machine = _get_machine( + machine_id, session=session + ) + if not user: + user_id = machine.owner_id + user_dict = user_api.get_user(user_id, session=session) + user_email = user_dict['email'] + else: + user_email = user.email + celery_client.celery.send_task( + 'compass.tasks.poweron_machine', + (machine_id,), + queue=user_email, + exchange=user_email, + routing_key=user_email + ) + return { + 'status': 'poweron %s action sent' % machine.mac, + 'machine': machine + } + + +@utils.supported_filters(optional_support_keys=['poweroff']) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEPLOY_HOST +) +@utils.wrap_to_dict( + RESP_DEPLOY_FIELDS, + machine=RESP_FIELDS +) +def poweroff_machine( + machine_id, poweroff={}, user=None, session=None, **kwargs +): + """power off machine.""" + from compass.tasks import client as celery_client + machine = _get_machine( + machine_id, session=session + ) + if not user: + user_id = machine.owner_id + user_dict = user_api.get_user(user_id, session=session) + user_email = user_dict['email'] + else: + user_email = user.email + celery_client.celery.send_task( + 'compass.tasks.poweroff_machine', + (machine_id,), + queue=user_email, + exchange=user_email, + routing_key=user_email + ) + return { + 'status': 'poweroff %s action sent' % machine.mac, + 'machine': machine + } + + +@utils.supported_filters(optional_support_keys=['reset']) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEPLOY_HOST +) +@utils.wrap_to_dict( + RESP_DEPLOY_FIELDS, + machine=RESP_FIELDS +) +def reset_machine( + machine_id, reset={}, user=None, session=None, **kwargs +): + """reset machine.""" + from compass.tasks import client as celery_client + machine = _get_machine( + machine_id, session=session + ) + if not user: + user_id = machine.owner_id + user_dict = user_api.get_user(user_id, session=session) + user_email = user_dict['email'] + else: + user_email = user.email + celery_client.celery.send_task( + 'compass.tasks.reset_machine', + (machine_id,), + queue=user_email, + exchange=user_email, + routing_key=user_email + ) + return { + 'status': 'reset %s action sent' % machine.mac, + 'machine': machine + } diff --git a/compass-tasks-base/db/api/metadata.py b/compass-tasks-base/db/api/metadata.py new file mode 100644 index 0000000..16310c8 --- /dev/null +++ b/compass-tasks-base/db/api/metadata.py @@ -0,0 +1,517 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Metadata related database operations.""" +import copy +import logging +import string + +from compass.db.api import adapter as adapter_api +from compass.db.api import database +from compass.db.api import utils +from compass.db import callback as metadata_callback +from compass.db import exception +from compass.db import models +from compass.db import validator as metadata_validator + + +from compass.utils import setting_wrapper as setting +from compass.utils import util + + +OS_FIELDS = None +PACKAGE_FIELDS = None +FLAVOR_FIELDS = None +OSES_METADATA = None +PACKAGES_METADATA = None +FLAVORS_METADATA = None +OSES_METADATA_UI_CONVERTERS = None +FLAVORS_METADATA_UI_CONVERTERS = None + + +def _get_field_from_configuration(configs): + """Get fields from configurations.""" + fields = {} + for config in configs: + if not isinstance(config, dict): + raise exception.InvalidParameter( + 'config %s is not dict' % config + ) + field_name = config['NAME'] + fields[field_name] = { + 'name': field_name, + 'id': field_name, + 'field_type': config.get('FIELD_TYPE', basestring), + 'display_type': config.get('DISPLAY_TYPE', 'text'), + 'validator': config.get('VALIDATOR', None), + 'js_validator': config.get('JS_VALIDATOR', None), + 'description': config.get('DESCRIPTION', field_name) + } + return fields + + +def _get_os_fields_from_configuration(): + """Get os fields from os field config dir.""" + env_locals = {} + env_locals.update(metadata_validator.VALIDATOR_LOCALS) + env_locals.update(metadata_callback.CALLBACK_LOCALS) + configs = util.load_configs( + setting.OS_FIELD_DIR, + env_locals=env_locals + ) + return _get_field_from_configuration( + configs + ) + + +def _get_package_fields_from_configuration(): + """Get package fields from package field config dir.""" + env_locals = {} + env_locals.update(metadata_validator.VALIDATOR_LOCALS) + env_locals.update(metadata_callback.CALLBACK_LOCALS) + configs = util.load_configs( + setting.PACKAGE_FIELD_DIR, + env_locals=env_locals + ) + return _get_field_from_configuration( + configs + ) + + +def _get_flavor_fields_from_configuration(): + """Get flavor fields from flavor field config dir.""" + env_locals = {} + env_locals.update(metadata_validator.VALIDATOR_LOCALS) + env_locals.update(metadata_callback.CALLBACK_LOCALS) + configs = util.load_configs( + setting.FLAVOR_FIELD_DIR, + env_locals=env_locals + ) + return _get_field_from_configuration( + configs + ) + + +def _get_metadata_from_configuration( + path, name, config, + fields, **kwargs +): + """Recursively get metadata from configuration. + + Args: + path: used to indicate the path to the root element. + mainly for trouble shooting. + name: the key of the metadata section. + config: the value of the metadata section. + fields: all fields defined in os fields or package fields dir. + """ + if not isinstance(config, dict): + raise exception.InvalidParameter( + '%s config %s is not dict' % (path, config) + ) + metadata_self = config.get('_self', {}) + if 'field' in metadata_self: + field_name = metadata_self['field'] + field = fields[field_name] + else: + field = {} + # mapping to may contain $ like $partition. Here we replace the + # $partition to the key of the correspendent config. The backend then + # can use this kind of feature to support multi partitions when we + # only declare the partition metadata in one place. + mapping_to_template = metadata_self.get('mapping_to', None) + if mapping_to_template: + mapping_to = string.Template( + mapping_to_template + ).safe_substitute( + **kwargs + ) + else: + mapping_to = None + self_metadata = { + 'name': name, + 'display_name': metadata_self.get('display_name', name), + 'field_type': field.get('field_type', dict), + 'display_type': field.get('display_type', None), + 'description': metadata_self.get( + 'description', field.get('description', None) + ), + 'is_required': metadata_self.get('is_required', False), + 'required_in_whole_config': metadata_self.get( + 'required_in_whole_config', False), + 'mapping_to': mapping_to, + 'validator': metadata_self.get( + 'validator', field.get('validator', None) + ), + 'js_validator': metadata_self.get( + 'js_validator', field.get('js_validator', None) + ), + 'default_value': metadata_self.get('default_value', None), + 'default_callback': metadata_self.get('default_callback', None), + 'default_callback_params': metadata_self.get( + 'default_callback_params', {}), + 'options': metadata_self.get('options', None), + 'options_callback': metadata_self.get('options_callback', None), + 'options_callback_params': metadata_self.get( + 'options_callback_params', {}), + 'autofill_callback': metadata_self.get( + 'autofill_callback', None), + 'autofill_callback_params': metadata_self.get( + 'autofill_callback_params', {}), + 'required_in_options': metadata_self.get( + 'required_in_options', False) + } + self_metadata.update(kwargs) + metadata = {'_self': self_metadata} + # Key extension used to do two things: + # one is to return the extended metadata that $ + # will be replace to possible extensions. + # The other is to record the $ to extended value + # and used in future mapping_to subsititution. + # TODO(grace): select proper name instead of key_extensions if + # you think it is better. + # Suppose key_extension is {'$partition': ['/var', '/']} for $partition + # the metadata for $partition will be mapped to { + # '/var': ..., '/': ...} and kwargs={'partition': '/var'} and + # kwargs={'partition': '/'} will be parsed to recursive metadata parsing + # for sub metadata under '/var' and '/'. Then in the metadata parsing + # for the sub metadata, this kwargs will be used to substitute mapping_to. + key_extensions = metadata_self.get('key_extensions', {}) + general_keys = [] + for key, value in config.items(): + if key.startswith('_'): + continue + if key in key_extensions: + if not key.startswith('$'): + raise exception.InvalidParameter( + '%s subkey %s should start with $' % ( + path, key + ) + ) + extended_keys = key_extensions[key] + for extended_key in extended_keys: + if extended_key.startswith('$'): + raise exception.InvalidParameter( + '%s extended key %s should not start with $' % ( + path, extended_key + ) + ) + sub_kwargs = dict(kwargs) + sub_kwargs[key[1:]] = extended_key + metadata[extended_key] = _get_metadata_from_configuration( + '%s/%s' % (path, extended_key), extended_key, value, + fields, **sub_kwargs + ) + else: + if key.startswith('$'): + general_keys.append(key) + metadata[key] = _get_metadata_from_configuration( + '%s/%s' % (path, key), key, value, + fields, **kwargs + ) + if len(general_keys) > 1: + raise exception.InvalidParameter( + 'foud multi general keys in %s: %s' % ( + path, general_keys + ) + ) + return metadata + + +def _get_oses_metadata_from_configuration(): + """Get os metadata from os metadata config dir.""" + oses_metadata = {} + env_locals = {} + env_locals.update(metadata_validator.VALIDATOR_LOCALS) + env_locals.update(metadata_callback.CALLBACK_LOCALS) + configs = util.load_configs( + setting.OS_METADATA_DIR, + env_locals=env_locals + ) + for config in configs: + os_name = config['OS'] + os_metadata = oses_metadata.setdefault(os_name, {}) + for key, value in config['METADATA'].items(): + os_metadata[key] = _get_metadata_from_configuration( + key, key, value, OS_FIELDS + ) + + oses = adapter_api.OSES + parents = {} + for os_name, os in oses.items(): + parent = os.get('parent', None) + parents[os_name] = parent + for os_name, os in oses.items(): + oses_metadata[os_name] = util.recursive_merge_dict( + os_name, oses_metadata, parents + ) + return oses_metadata + + +def _get_packages_metadata_from_configuration(): + """Get package metadata from package metadata config dir.""" + packages_metadata = {} + env_locals = {} + env_locals.update(metadata_validator.VALIDATOR_LOCALS) + env_locals.update(metadata_callback.CALLBACK_LOCALS) + configs = util.load_configs( + setting.PACKAGE_METADATA_DIR, + env_locals=env_locals + ) + for config in configs: + adapter_name = config['ADAPTER'] + package_metadata = packages_metadata.setdefault(adapter_name, {}) + for key, value in config['METADATA'].items(): + package_metadata[key] = _get_metadata_from_configuration( + key, key, value, PACKAGE_FIELDS + ) + adapters = adapter_api.ADAPTERS + parents = {} + for adapter_name, adapter in adapters.items(): + parent = adapter.get('parent', None) + parents[adapter_name] = parent + for adapter_name, adapter in adapters.items(): + packages_metadata[adapter_name] = util.recursive_merge_dict( + adapter_name, packages_metadata, parents + ) + return packages_metadata + + +def _get_flavors_metadata_from_configuration(): + """Get flavor metadata from flavor metadata config dir.""" + flavors_metadata = {} + env_locals = {} + env_locals.update(metadata_validator.VALIDATOR_LOCALS) + env_locals.update(metadata_callback.CALLBACK_LOCALS) + configs = util.load_configs( + setting.FLAVOR_METADATA_DIR, + env_locals=env_locals + ) + for config in configs: + adapter_name = config['ADAPTER'] + flavor_name = config['FLAVOR'] + flavor_metadata = flavors_metadata.setdefault( + adapter_name, {} + ).setdefault(flavor_name, {}) + for key, value in config['METADATA'].items(): + flavor_metadata[key] = _get_metadata_from_configuration( + key, key, value, FLAVOR_FIELDS + ) + + packages_metadata = PACKAGES_METADATA + adapters_flavors = adapter_api.ADAPTERS_FLAVORS + for adapter_name, adapter_flavors in adapters_flavors.items(): + package_metadata = packages_metadata.get(adapter_name, {}) + for flavor_name, flavor in adapter_flavors.items(): + flavor_metadata = flavors_metadata.setdefault( + adapter_name, {} + ).setdefault(flavor_name, {}) + util.merge_dict(flavor_metadata, package_metadata, override=False) + return flavors_metadata + + +def _filter_metadata(metadata, **kwargs): + if not isinstance(metadata, dict): + return metadata + filtered_metadata = {} + for key, value in metadata.items(): + if key == '_self': + default_value = value.get('default_value', None) + if default_value is None: + default_callback_params = value.get( + 'default_callback_params', {} + ) + callback_params = dict(kwargs) + if default_callback_params: + callback_params.update(default_callback_params) + default_callback = value.get('default_callback', None) + if default_callback: + default_value = default_callback(key, **callback_params) + options = value.get('options', None) + if options is None: + options_callback_params = value.get( + 'options_callback_params', {} + ) + callback_params = dict(kwargs) + if options_callback_params: + callback_params.update(options_callback_params) + + options_callback = value.get('options_callback', None) + if options_callback: + options = options_callback(key, **callback_params) + filtered_metadata[key] = value + if default_value is not None: + filtered_metadata[key]['default_value'] = default_value + if options is not None: + filtered_metadata[key]['options'] = options + else: + filtered_metadata[key] = _filter_metadata(value, **kwargs) + return filtered_metadata + + +def _load_metadata(force_reload=False): + """Load metadata information into memory. + + If force_reload, the metadata information will be reloaded + even if the metadata is already loaded. + """ + adapter_api.load_adapters_internal(force_reload=force_reload) + global OS_FIELDS + if force_reload or OS_FIELDS is None: + OS_FIELDS = _get_os_fields_from_configuration() + global PACKAGE_FIELDS + if force_reload or PACKAGE_FIELDS is None: + PACKAGE_FIELDS = _get_package_fields_from_configuration() + global FLAVOR_FIELDS + if force_reload or FLAVOR_FIELDS is None: + FLAVOR_FIELDS = _get_flavor_fields_from_configuration() + global OSES_METADATA + if force_reload or OSES_METADATA is None: + OSES_METADATA = _get_oses_metadata_from_configuration() + global PACKAGES_METADATA + if force_reload or PACKAGES_METADATA is None: + PACKAGES_METADATA = _get_packages_metadata_from_configuration() + global FLAVORS_METADATA + if force_reload or FLAVORS_METADATA is None: + FLAVORS_METADATA = _get_flavors_metadata_from_configuration() + global OSES_METADATA_UI_CONVERTERS + if force_reload or OSES_METADATA_UI_CONVERTERS is None: + OSES_METADATA_UI_CONVERTERS = ( + _get_oses_metadata_ui_converters_from_configuration() + ) + global FLAVORS_METADATA_UI_CONVERTERS + if force_reload or FLAVORS_METADATA_UI_CONVERTERS is None: + FLAVORS_METADATA_UI_CONVERTERS = ( + _get_flavors_metadata_ui_converters_from_configuration() + ) + + +def _get_oses_metadata_ui_converters_from_configuration(): + """Get os metadata ui converters from os metadata mapping config dir. + + os metadata ui converter is used to convert os metadata to + the format UI can understand and show. + """ + oses_metadata_ui_converters = {} + configs = util.load_configs(setting.OS_MAPPING_DIR) + for config in configs: + os_name = config['OS'] + oses_metadata_ui_converters[os_name] = config.get('CONFIG_MAPPING', {}) + + oses = adapter_api.OSES + parents = {} + for os_name, os in oses.items(): + parent = os.get('parent', None) + parents[os_name] = parent + for os_name, os in oses.items(): + oses_metadata_ui_converters[os_name] = util.recursive_merge_dict( + os_name, oses_metadata_ui_converters, parents + ) + return oses_metadata_ui_converters + + +def _get_flavors_metadata_ui_converters_from_configuration(): + """Get flavor metadata ui converters from flavor mapping config dir.""" + flavors_metadata_ui_converters = {} + configs = util.load_configs(setting.FLAVOR_MAPPING_DIR) + for config in configs: + adapter_name = config['ADAPTER'] + flavor_name = config['FLAVOR'] + flavors_metadata_ui_converters.setdefault( + adapter_name, {} + )[flavor_name] = config.get('CONFIG_MAPPING', {}) + adapters = adapter_api.ADAPTERS + parents = {} + for adapter_name, adapter in adapters.items(): + parent = adapter.get('parent', None) + parents[adapter_name] = parent + for adapter_name, adapter in adapters.items(): + flavors_metadata_ui_converters[adapter_name] = ( + util.recursive_merge_dict( + adapter_name, flavors_metadata_ui_converters, parents + ) + ) + return flavors_metadata_ui_converters + + +def get_packages_metadata_internal(force_reload=False): + """Get deployable package metadata.""" + _load_metadata(force_reload=force_reload) + metadata_mapping = {} + adapters = adapter_api.ADAPTERS + for adapter_name, adapter in adapters.items(): + if adapter.get('deployable'): + metadata_mapping[adapter_name] = _filter_metadata( + PACKAGES_METADATA.get(adapter_name, {}) + ) + else: + logging.info( + 'ignore metadata since its adapter %s is not deployable', + adapter_name + ) + return metadata_mapping + + +def get_flavors_metadata_internal(force_reload=False): + """Get deployable flavor metadata.""" + _load_metadata(force_reload=force_reload) + metadata_mapping = {} + adapters_flavors = adapter_api.ADAPTERS_FLAVORS + for adapter_name, adapter_flavors in adapters_flavors.items(): + adapter = adapter_api.ADAPTERS[adapter_name] + if not adapter.get('deployable'): + logging.info( + 'ignore metadata since its adapter %s is not deployable', + adapter_name + ) + continue + for flavor_name, flavor in adapter_flavors.items(): + flavor_metadata = FLAVORS_METADATA.get( + adapter_name, {} + ).get(flavor_name, {}) + metadata = _filter_metadata(flavor_metadata) + metadata_mapping.setdefault( + adapter_name, {} + )[flavor_name] = metadata + return metadata_mapping + + +def get_flavors_metadata_ui_converters_internal(force_reload=False): + """Get usable flavor metadata ui converters.""" + _load_metadata(force_reload=force_reload) + return FLAVORS_METADATA_UI_CONVERTERS + + +def get_oses_metadata_internal(force_reload=False): + """Get deployable os metadata.""" + _load_metadata(force_reload=force_reload) + metadata_mapping = {} + oses = adapter_api.OSES + for os_name, os in oses.items(): + if os.get('deployable'): + metadata_mapping[os_name] = _filter_metadata( + OSES_METADATA.get(os_name, {}) + ) + else: + logging.info( + 'ignore metadata since its os %s is not deployable', + os_name + ) + return metadata_mapping + + +def get_oses_metadata_ui_converters_internal(force_reload=False): + """Get usable os metadata ui converters.""" + _load_metadata(force_reload=force_reload) + return OSES_METADATA_UI_CONVERTERS diff --git a/compass-tasks-base/db/api/metadata_holder.py b/compass-tasks-base/db/api/metadata_holder.py new file mode 100644 index 0000000..24afc67 --- /dev/null +++ b/compass-tasks-base/db/api/metadata_holder.py @@ -0,0 +1,731 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Metadata related object holder.""" +import logging + +from compass.db.api import adapter as adapter_api +from compass.db.api import adapter_holder as adapter_holder_api +from compass.db.api import database +from compass.db.api import metadata as metadata_api +from compass.db.api import permission +from compass.db.api import user as user_api +from compass.db.api import utils +from compass.db import exception +from compass.db import models +from compass.utils import setting_wrapper as setting +from compass.utils import util + + +RESP_METADATA_FIELDS = [ + 'os_config', 'package_config' +] +RESP_UI_METADATA_FIELDS = [ + 'os_global_config', 'flavor_config' +] + + +def load_metadatas(force_reload=False): + """Load metadatas.""" + # TODO(xicheng): today we load metadata in memory as it original + # format in files in metadata.py. We get these inmemory metadata + # and do some translation, store the translated metadata into memory + # too in metadata_holder.py. api can only access the global inmemory + # data in metadata_holder.py. + _load_os_metadatas(force_reload=force_reload) + _load_package_metadatas(force_reload=force_reload) + _load_flavor_metadatas(force_reload=force_reload) + _load_os_metadata_ui_converters(force_reload=force_reload) + _load_flavor_metadata_ui_converters(force_reload=force_reload) + + +def _load_os_metadata_ui_converters(force_reload=False): + global OS_METADATA_UI_CONVERTERS + if force_reload or OS_METADATA_UI_CONVERTERS is None: + logging.info('load os metadatas ui converters into memory') + OS_METADATA_UI_CONVERTERS = ( + metadata_api.get_oses_metadata_ui_converters_internal( + force_reload=force_reload + ) + ) + + +def _load_os_metadatas(force_reload=False): + """Load os metadata from inmemory db and map it by os_id.""" + global OS_METADATA_MAPPING + if force_reload or OS_METADATA_MAPPING is None: + logging.info('load os metadatas into memory') + OS_METADATA_MAPPING = metadata_api.get_oses_metadata_internal( + force_reload=force_reload + ) + + +def _load_flavor_metadata_ui_converters(force_reload=False): + """Load flavor metadata ui converters from inmemory db. + + The loaded metadata is mapped by flavor id. + """ + global FLAVOR_METADATA_UI_CONVERTERS + if force_reload or FLAVOR_METADATA_UI_CONVERTERS is None: + logging.info('load flavor metadata ui converters into memory') + FLAVOR_METADATA_UI_CONVERTERS = {} + adapters_flavors_metadata_ui_converters = ( + metadata_api.get_flavors_metadata_ui_converters_internal( + force_reload=force_reload + ) + ) + for adapter_name, adapter_flavors_metadata_ui_converters in ( + adapters_flavors_metadata_ui_converters.items() + ): + for flavor_name, flavor_metadata_ui_converter in ( + adapter_flavors_metadata_ui_converters.items() + ): + FLAVOR_METADATA_UI_CONVERTERS[ + '%s:%s' % (adapter_name, flavor_name) + ] = flavor_metadata_ui_converter + + +@util.deprecated +def _load_package_metadatas(force_reload=False): + """Load deployable package metadata from inmemory db.""" + global PACKAGE_METADATA_MAPPING + if force_reload or PACKAGE_METADATA_MAPPING is None: + logging.info('load package metadatas into memory') + PACKAGE_METADATA_MAPPING = ( + metadata_api.get_packages_metadata_internal( + force_reload=force_reload + ) + ) + + +def _load_flavor_metadatas(force_reload=False): + """Load flavor metadata from inmemory db. + + The loaded metadata are mapped by flavor id. + """ + global FLAVOR_METADATA_MAPPING + if force_reload or FLAVOR_METADATA_MAPPING is None: + logging.info('load flavor metadatas into memory') + FLAVOR_METADATA_MAPPING = {} + adapters_flavors_metadata = ( + metadata_api.get_flavors_metadata_internal( + force_reload=force_reload + ) + ) + for adapter_name, adapter_flavors_metadata in ( + adapters_flavors_metadata.items() + ): + for flavor_name, flavor_metadata in ( + adapter_flavors_metadata.items() + ): + FLAVOR_METADATA_MAPPING[ + '%s:%s' % (adapter_name, flavor_name) + ] = flavor_metadata + + +OS_METADATA_MAPPING = None +PACKAGE_METADATA_MAPPING = None +FLAVOR_METADATA_MAPPING = None +OS_METADATA_UI_CONVERTERS = None +FLAVOR_METADATA_UI_CONVERTERS = None + + +def validate_os_config( + config, os_id, whole_check=False, **kwargs +): + """Validate os config.""" + load_metadatas() + if os_id not in OS_METADATA_MAPPING: + raise exception.InvalidParameter( + 'os %s is not found in os metadata mapping' % os_id + ) + _validate_config( + '', config, OS_METADATA_MAPPING[os_id], + whole_check, **kwargs + ) + + +@util.deprecated +def validate_package_config( + config, adapter_id, whole_check=False, **kwargs +): + """Validate package config.""" + load_metadatas() + if adapter_id not in PACKAGE_METADATA_MAPPING: + raise exception.InvalidParameter( + 'adapter %s is not found in package metedata mapping' % adapter_id + ) + _validate_config( + '', config, PACKAGE_METADATA_MAPPING[adapter_id], + whole_check, **kwargs + ) + + +def validate_flavor_config( + config, flavor_id, whole_check=False, **kwargs +): + """Validate flavor config.""" + load_metadatas() + if not flavor_id: + logging.info('There is no flavor, skipping flavor validation...') + elif flavor_id not in FLAVOR_METADATA_MAPPING: + raise exception.InvalidParameter( + 'flavor %s is not found in flavor metedata mapping' % flavor_id + ) + else: + _validate_config( + '', config, FLAVOR_METADATA_MAPPING[flavor_id], + whole_check, **kwargs + ) + + +def _filter_metadata(metadata, **kwargs): + """Filter metadata before return it to api. + + Some metadata fields are not json compatible or + only used in db/api internally. + We should strip these fields out before return to api. + """ + if not isinstance(metadata, dict): + return metadata + filtered_metadata = {} + for key, value in metadata.items(): + if key == '_self': + filtered_metadata[key] = { + 'name': value['name'], + 'description': value.get('description', None), + 'default_value': value.get('default_value', None), + 'is_required': value.get('is_required', False), + 'required_in_whole_config': value.get( + 'required_in_whole_config', False), + 'js_validator': value.get('js_validator', None), + 'options': value.get('options', None), + 'required_in_options': value.get( + 'required_in_options', False), + 'field_type': value.get( + 'field_type_data', 'str'), + 'display_type': value.get('display_type', None), + 'mapping_to': value.get('mapping_to', None) + } + else: + filtered_metadata[key] = _filter_metadata(value, **kwargs) + return filtered_metadata + + +@util.deprecated +def _get_package_metadata(adapter_id): + """get package metadata.""" + load_metadatas() + if adapter_id not in PACKAGE_METADATA_MAPPING: + raise exception.RecordNotExists( + 'adpater %s does not exist' % adapter_id + ) + return _filter_metadata( + PACKAGE_METADATA_MAPPING[adapter_id] + ) + + +@util.deprecated +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_METADATA_FIELDS) +def get_package_metadata(adapter_id, user=None, session=None, **kwargs): + """Get package metadata from adapter.""" + return { + 'package_config': _get_package_metadata(adapter_id) + } + + +def _get_flavor_metadata(flavor_id): + """get flavor metadata.""" + load_metadatas() + if not flavor_id: + logging.info('There is no flavor id, skipping...') + elif flavor_id not in FLAVOR_METADATA_MAPPING: + raise exception.RecordNotExists( + 'flavor %s does not exist' % flavor_id + ) + else: + return _filter_metadata(FLAVOR_METADATA_MAPPING[flavor_id]) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_METADATA_FIELDS) +def get_flavor_metadata(flavor_id, user=None, session=None, **kwargs): + """Get flavor metadata by flavor.""" + return { + 'package_config': _get_flavor_metadata(flavor_id) + } + + +def _get_os_metadata(os_id): + """get os metadata.""" + load_metadatas() + if os_id not in OS_METADATA_MAPPING: + raise exception.RecordNotExists( + 'os %s does not exist' % os_id + ) + return _filter_metadata(OS_METADATA_MAPPING[os_id]) + + +def _get_os_metadata_ui_converter(os_id): + """get os metadata ui converter.""" + load_metadatas() + if os_id not in OS_METADATA_UI_CONVERTERS: + raise exception.RecordNotExists( + 'os %s does not exist' % os_id + ) + return OS_METADATA_UI_CONVERTERS[os_id] + + +def _get_flavor_metadata_ui_converter(flavor_id): + """get flavor metadata ui converter.""" + load_metadatas() + if flavor_id not in FLAVOR_METADATA_UI_CONVERTERS: + raise exception.RecordNotExists( + 'flavor %s does not exist' % flavor_id + ) + return FLAVOR_METADATA_UI_CONVERTERS[flavor_id] + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_METADATA_FIELDS) +def get_os_metadata(os_id, user=None, session=None, **kwargs): + """get os metadatas.""" + return {'os_config': _get_os_metadata(os_id)} + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_UI_METADATA_FIELDS) +def get_os_ui_metadata(os_id, user=None, session=None, **kwargs): + """Get os metadata ui converter by os.""" + metadata = _get_os_metadata(os_id) + metadata_ui_converter = _get_os_metadata_ui_converter(os_id) + return _get_ui_metadata(metadata, metadata_ui_converter) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_UI_METADATA_FIELDS) +def get_flavor_ui_metadata(flavor_id, user=None, session=None, **kwargs): + """Get flavor ui metadata by flavor.""" + metadata = _get_flavor_metadata(flavor_id) + metadata_ui_converter = _get_flavor_metadata_ui_converter(flavor_id) + return _get_ui_metadata(metadata, metadata_ui_converter) + + +def _get_ui_metadata(metadata, metadata_ui_converter): + """convert metadata to ui metadata. + + Args: + metadata: metadata we defined in metadata files. + metadata_ui_converter: metadata ui converter defined in metadata + mapping files. Used to convert orignal + metadata to ui understandable metadata. + + Returns: + ui understandable metadata. + """ + ui_metadata = {} + ui_metadata[metadata_ui_converter['mapped_name']] = [] + for mapped_child in metadata_ui_converter['mapped_children']: + data_dict = {} + for ui_key, ui_value in mapped_child.items(): + for key, value in ui_value.items(): + if 'data' == key: + result_data = [] + _get_ui_metadata_data( + metadata[ui_key], value, result_data + ) + data_dict['data'] = result_data + else: + data_dict[key] = value + ui_metadata[metadata_ui_converter['mapped_name']].append(data_dict) + return ui_metadata + + +def _get_ui_metadata_data(metadata, config, result_data): + """Get ui metadata data and fill to result.""" + data_dict = {} + for key, config_value in config.items(): + if isinstance(config_value, dict) and key != 'content_data': + if key in metadata.keys(): + _get_ui_metadata_data(metadata[key], config_value, result_data) + else: + _get_ui_metadata_data(metadata, config_value, result_data) + elif isinstance(config_value, list): + option_list = [] + for item in config_value: + if isinstance(item, dict): + option_list.append(item) + data_dict[key] = option_list + else: + if isinstance(metadata['_self'][item], bool): + data_dict[item] = str(metadata['_self'][item]).lower() + else: + data_dict[item] = metadata['_self'][item] + else: + data_dict[key] = config_value + if data_dict: + result_data.append(data_dict) + return result_data + + +@util.deprecated +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_METADATA_FIELDS) +def get_package_os_metadata( + adapter_id, os_id, + user=None, session=None, **kwargs +): + """Get metadata by adapter and os.""" + adapter = adapter_holder_api.get_adapter( + adapter_id, user=user, session=session + ) + os_ids = [os['id'] for os in adapter['supported_oses']] + if os_id not in os_ids: + raise exception.InvalidParameter( + 'os %s is not in the supported os list of adapter %s' % ( + os_id, adapter_id + ) + ) + metadatas = {} + metadatas['os_config'] = _get_os_metadata( + os_id + ) + metadatas['package_config'] = _get_package_metadata( + adapter_id + ) + return metadatas + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_METADATA_FIELDS) +def get_flavor_os_metadata( + flavor_id, os_id, + user=None, session=None, **kwargs +): + """Get metadata by flavor and os.""" + flavor = adapter_holder_api.get_flavor( + flavor_id, user=user, session=session + ) + adapter_id = flavor['adapter_id'] + adapter = adapter_holder_api.get_adapter( + adapter_id, user=user, session=session + ) + os_ids = [os['id'] for os in adapter['supported_oses']] + if os_id not in os_ids: + raise exception.InvalidParameter( + 'os %s is not in the supported os list of adapter %s' % ( + os_id, adapter_id + ) + ) + metadatas = {} + metadatas['os_config'] = _get_os_metadata( + session, os_id + ) + metadatas['package_config'] = _get_flavor_metadata( + session, flavor_id + ) + return metadatas + + +def _validate_self( + config_path, config_key, config, + metadata, whole_check, + **kwargs +): + """validate config by metadata self section.""" + logging.debug('validate config self %s', config_path) + if '_self' not in metadata: + if isinstance(config, dict): + _validate_config( + config_path, config, metadata, whole_check, **kwargs + ) + return + field_type = metadata['_self'].get('field_type', basestring) + if not isinstance(config, field_type): + raise exception.InvalidParameter( + '%s config type is not %s: %s' % (config_path, field_type, config) + ) + is_required = metadata['_self'].get( + 'is_required', False + ) + required_in_whole_config = metadata['_self'].get( + 'required_in_whole_config', False + ) + if isinstance(config, basestring): + if config == '' and not is_required and not required_in_whole_config: + # ignore empty config when it is optional + return + required_in_options = metadata['_self'].get( + 'required_in_options', False + ) + options = metadata['_self'].get('options', None) + if required_in_options: + if field_type in [int, basestring, float, bool]: + if options and config not in options: + raise exception.InvalidParameter( + '%s config is not in %s: %s' % ( + config_path, options, config + ) + ) + elif field_type in [list, tuple]: + if options and not set(config).issubset(set(options)): + raise exception.InvalidParameter( + '%s config is not in %s: %s' % ( + config_path, options, config + ) + ) + elif field_type == dict: + if options and not set(config.keys()).issubset(set(options)): + raise exception.InvalidParameter( + '%s config is not in %s: %s' % ( + config_path, options, config + ) + ) + validator = metadata['_self'].get('validator', None) + logging.debug('validate by validator %s', validator) + if validator: + if not validator(config_key, config, **kwargs): + raise exception.InvalidParameter( + '%s config is invalid' % config_path + ) + if isinstance(config, dict): + _validate_config( + config_path, config, metadata, whole_check, **kwargs + ) + + +def _validate_config( + config_path, config, metadata, whole_check, + **kwargs +): + """validate config by metadata.""" + logging.debug('validate config %s', config_path) + generals = {} + specified = {} + for key, value in metadata.items(): + if key.startswith('$'): + generals[key] = value + elif key.startswith('_'): + pass + else: + specified[key] = value + config_keys = set(config.keys()) + specified_keys = set(specified.keys()) + intersect_keys = config_keys & specified_keys + not_found_keys = config_keys - specified_keys + redundant_keys = specified_keys - config_keys + for key in redundant_keys: + if '_self' not in specified[key]: + continue + if specified[key]['_self'].get('is_required', False): + raise exception.InvalidParameter( + '%s/%s does not find but it is required' % ( + config_path, key + ) + ) + if ( + whole_check and + specified[key]['_self'].get( + 'required_in_whole_config', False + ) + ): + raise exception.InvalidParameter( + '%s/%s does not find but it is required in whole config' % ( + config_path, key + ) + ) + for key in intersect_keys: + _validate_self( + '%s/%s' % (config_path, key), + key, config[key], specified[key], whole_check, + **kwargs + ) + for key in not_found_keys: + if not generals: + raise exception.InvalidParameter( + 'key %s missing in metadata %s' % ( + key, config_path + ) + ) + for general_key, general_value in generals.items(): + _validate_self( + '%s/%s' % (config_path, key), + key, config[key], general_value, whole_check, + **kwargs + ) + + +def _autofill_self_config( + config_path, config_key, config, + metadata, + **kwargs +): + """Autofill config by metadata self section.""" + if '_self' not in metadata: + if isinstance(config, dict): + _autofill_config( + config_path, config, metadata, **kwargs + ) + return config + logging.debug( + 'autofill %s by metadata %s', config_path, metadata['_self'] + ) + autofill_callback = metadata['_self'].get( + 'autofill_callback', None + ) + autofill_callback_params = metadata['_self'].get( + 'autofill_callback_params', {} + ) + callback_params = dict(kwargs) + if autofill_callback_params: + callback_params.update(autofill_callback_params) + default_value = metadata['_self'].get( + 'default_value', None + ) + if default_value is not None: + callback_params['default_value'] = default_value + options = metadata['_self'].get( + 'options', None + ) + if options is not None: + callback_params['options'] = options + if autofill_callback: + config = autofill_callback( + config_key, config, **callback_params + ) + if config is None: + new_config = {} + else: + new_config = config + if isinstance(new_config, dict): + _autofill_config( + config_path, new_config, metadata, **kwargs + ) + if new_config: + config = new_config + return config + + +def _autofill_config( + config_path, config, metadata, **kwargs +): + """autofill config by metadata.""" + generals = {} + specified = {} + for key, value in metadata.items(): + if key.startswith('$'): + generals[key] = value + elif key.startswith('_'): + pass + else: + specified[key] = value + config_keys = set(config.keys()) + specified_keys = set(specified.keys()) + intersect_keys = config_keys & specified_keys + not_found_keys = config_keys - specified_keys + redundant_keys = specified_keys - config_keys + for key in redundant_keys: + self_config = _autofill_self_config( + '%s/%s' % (config_path, key), + key, None, specified[key], **kwargs + ) + if self_config is not None: + config[key] = self_config + for key in intersect_keys: + config[key] = _autofill_self_config( + '%s/%s' % (config_path, key), + key, config[key], specified[key], + **kwargs + ) + for key in not_found_keys: + for general_key, general_value in generals.items(): + config[key] = _autofill_self_config( + '%s/%s' % (config_path, key), + key, config[key], general_value, + **kwargs + ) + return config + + +def autofill_os_config( + config, os_id, **kwargs +): + load_metadatas() + if os_id not in OS_METADATA_MAPPING: + raise exception.InvalidParameter( + 'os %s is not found in os metadata mapping' % os_id + ) + + return _autofill_config( + '', config, OS_METADATA_MAPPING[os_id], **kwargs + ) + + +def autofill_package_config( + config, adapter_id, **kwargs +): + load_metadatas() + if adapter_id not in PACKAGE_METADATA_MAPPING: + raise exception.InvalidParameter( + 'adapter %s is not found in package metadata mapping' % adapter_id + ) + + return _autofill_config( + '', config, PACKAGE_METADATA_MAPPING[adapter_id], **kwargs + ) + + +def autofill_flavor_config( + config, flavor_id, **kwargs +): + load_metadatas() + if not flavor_id: + logging.info('There is no flavor, skipping...') + elif flavor_id not in FLAVOR_METADATA_MAPPING: + raise exception.InvalidParameter( + 'flavor %s is not found in flavor metadata mapping' % flavor_id + ) + else: + return _autofill_config( + '', config, FLAVOR_METADATA_MAPPING[flavor_id], **kwargs + ) diff --git a/compass-tasks-base/db/api/network.py b/compass-tasks-base/db/api/network.py new file mode 100644 index 0000000..e2bf7d3 --- /dev/null +++ b/compass-tasks-base/db/api/network.py @@ -0,0 +1,160 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Network related database operations.""" +import logging +import netaddr +import re + +from compass.db.api import database +from compass.db.api import permission +from compass.db.api import user as user_api +from compass.db.api import utils +from compass.db import exception +from compass.db import models + + +SUPPORTED_FIELDS = ['subnet', 'name'] +RESP_FIELDS = [ + 'id', 'name', 'subnet', 'created_at', 'updated_at' +] +ADDED_FIELDS = ['subnet'] +OPTIONAL_ADDED_FIELDS = ['name'] +IGNORE_FIELDS = [ + 'id', 'created_at', 'updated_at' +] +UPDATED_FIELDS = ['subnet', 'name'] + + +def _check_subnet(subnet): + """Check subnet format is correct.""" + try: + netaddr.IPNetwork(subnet) + except Exception as error: + logging.exception(error) + raise exception.InvalidParameter( + 'subnet %s format unrecognized' % subnet) + + +@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_SUBNETS +) +@utils.wrap_to_dict(RESP_FIELDS) +def list_subnets(user=None, session=None, **filters): + """List subnets.""" + return utils.list_db_objects( + session, models.Subnet, **filters + ) + + +def _get_subnet(subnet_id, session=None, **kwargs): + """Get subnet by subnet id.""" + if isinstance(subnet_id, (int, long)): + return utils.get_db_object( + session, models.Subnet, + id=subnet_id, **kwargs + ) + raise exception.InvalidParameter( + 'subnet id %s type is not int compatible' % subnet_id + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_SUBNETS +) +@utils.wrap_to_dict(RESP_FIELDS) +def get_subnet( + subnet_id, exception_when_missing=True, + user=None, session=None, **kwargs +): + """Get subnet info.""" + return _get_subnet( + subnet_id, session=session, + exception_when_missing=exception_when_missing + ) + + +@utils.supported_filters( + ADDED_FIELDS, optional_support_keys=OPTIONAL_ADDED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates(subnet=_check_subnet) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_SUBNET +) +@utils.wrap_to_dict(RESP_FIELDS) +def add_subnet( + exception_when_existing=True, subnet=None, + user=None, session=None, **kwargs +): + """Create a subnet.""" + return utils.add_db_object( + session, models.Subnet, + exception_when_existing, subnet, **kwargs + ) + + +@utils.supported_filters( + optional_support_keys=UPDATED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates(subnet=_check_subnet) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_SUBNET +) +@utils.wrap_to_dict(RESP_FIELDS) +def update_subnet(subnet_id, user=None, session=None, **kwargs): + """Update a subnet.""" + subnet = _get_subnet( + subnet_id, session=session + ) + return utils.update_db_object(session, subnet, **kwargs) + + +def _check_subnet_deletable(subnet): + """Check a subnet deletable.""" + if subnet.host_networks: + host_networks = [ + '%s:%s=%s' % ( + host_network.host.name, host_network.interface, + host_network.ip + ) + for host_network in subnet.host_networks + ] + raise exception.NotAcceptable( + 'subnet %s contains host networks %s' % ( + subnet.subnet, host_networks + ) + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEL_SUBNET +) +@utils.wrap_to_dict(RESP_FIELDS) +def del_subnet(subnet_id, user=None, session=None, **kwargs): + """Delete a subnet.""" + subnet = _get_subnet( + subnet_id, session=session + ) + _check_subnet_deletable(subnet) + return utils.del_db_object(session, subnet) diff --git a/compass-tasks-base/db/api/permission.py b/compass-tasks-base/db/api/permission.py new file mode 100644 index 0000000..f4d777a --- /dev/null +++ b/compass-tasks-base/db/api/permission.py @@ -0,0 +1,357 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Permission database operations.""" +import re + +from compass.db.api import database +from compass.db.api import user as user_api +from compass.db.api import utils +from compass.db import exception +from compass.db import models +from compass.utils import util + + +SUPPORTED_FIELDS = ['id', 'name', 'alias', 'description'] +RESP_FIELDS = ['id', 'name', 'alias', 'description'] + + +class PermissionWrapper(object): + def __init__(self, name, alias, description): + self.name = name + self.alias = alias + self.description = description + + def to_dict(self): + return { + 'name': self.name, + 'alias': self.alias, + 'description': self.description + } + + +PERMISSION_LIST_PERMISSIONS = PermissionWrapper( + 'list_permissions', 'list permissions', 'list all permissions' +) +PERMISSION_LIST_SWITCHES = PermissionWrapper( + 'list_switches', 'list switches', 'list all switches' +) +PERMISSION_LIST_SWITCH_FILTERS = PermissionWrapper( + 'list_switch_filters', + 'list switch filters', + 'list switch filters' +) +PERMISSION_ADD_SWITCH = PermissionWrapper( + 'add_switch', 'add switch', 'add switch' +) +PERMISSION_UPDATE_SWITCH_FILTERS = PermissionWrapper( + 'update_switch_filters', + 'update switch filters', + 'update switch filters' +) +PERMISSION_DEL_SWITCH = PermissionWrapper( + 'delete_switch', 'delete switch', 'delete switch' +) +PERMISSION_LIST_SWITCH_MACHINES = PermissionWrapper( + 'list_switch_machines', 'list switch machines', 'list switch machines' +) +PERMISSION_ADD_SWITCH_MACHINE = PermissionWrapper( + 'add_switch_machine', 'add switch machine', 'add switch machine' +) +PERMISSION_DEL_SWITCH_MACHINE = PermissionWrapper( + 'del_switch_machine', 'delete switch machine', 'del switch machine' +) +PERMISSION_UPDATE_SWITCH_MACHINES = PermissionWrapper( + 'update_switch_machines', + 'update switch machines', + 'update switch machines' +) +PERMISSION_LIST_MACHINES = PermissionWrapper( + 'list_machines', 'list machines', 'list machines' +) +PERMISSION_ADD_MACHINE = PermissionWrapper( + 'add_machine', 'add machine', 'add machine' +) +PERMISSION_DEL_MACHINE = PermissionWrapper( + 'delete_machine', 'delete machine', 'delete machine' +) +PERMISSION_LIST_ADAPTERS = PermissionWrapper( + 'list_adapters', 'list adapters', 'list adapters' +) +PERMISSION_LIST_METADATAS = PermissionWrapper( + 'list_metadatas', 'list metadatas', 'list metadatas' +) +PERMISSION_LIST_SUBNETS = PermissionWrapper( + 'list_subnets', 'list subnets', 'list subnets' +) +PERMISSION_ADD_SUBNET = PermissionWrapper( + 'add_subnet', 'add subnet', 'add subnet' +) +PERMISSION_DEL_SUBNET = PermissionWrapper( + 'del_subnet', 'del subnet', 'del subnet' +) +PERMISSION_LIST_CLUSTERS = PermissionWrapper( + 'list_clusters', 'list clusters', 'list clusters' +) +PERMISSION_ADD_CLUSTER = PermissionWrapper( + 'add_cluster', 'add cluster', 'add cluster' +) +PERMISSION_DEL_CLUSTER = PermissionWrapper( + 'del_cluster', 'del cluster', 'del cluster' +) +PERMISSION_LIST_CLUSTER_CONFIG = PermissionWrapper( + 'list_cluster_config', 'list cluster config', 'list cluster config' +) +PERMISSION_ADD_CLUSTER_CONFIG = PermissionWrapper( + 'add_cluster_config', 'add cluster config', 'add cluster config' +) +PERMISSION_DEL_CLUSTER_CONFIG = PermissionWrapper( + 'del_cluster_config', 'del cluster config', 'del cluster config' +) +PERMISSION_UPDATE_CLUSTER_HOSTS = PermissionWrapper( + 'update_cluster_hosts', + 'update cluster hosts', + 'update cluster hosts' +) +PERMISSION_DEL_CLUSTER_HOST = PermissionWrapper( + 'del_clusterhost', 'delete clusterhost', 'delete clusterhost' +) +PERMISSION_REVIEW_CLUSTER = PermissionWrapper( + 'review_cluster', 'review cluster', 'review cluster' +) +PERMISSION_DEPLOY_CLUSTER = PermissionWrapper( + 'deploy_cluster', 'deploy cluster', 'deploy cluster' +) +PERMISSION_DEPLOY_HOST = PermissionWrapper( + 'deploy_host', 'deploy host', 'deploy host' +) +PERMISSION_GET_CLUSTER_STATE = PermissionWrapper( + 'get_cluster_state', 'get cluster state', 'get cluster state' +) +PERMISSION_UPDATE_CLUSTER_STATE = PermissionWrapper( + 'update_cluster_state', 'update cluster state', + 'update cluster state' +) +PERMISSION_LIST_HOSTS = PermissionWrapper( + 'list_hosts', 'list hosts', 'list hosts' +) +PERMISSION_LIST_HOST_CLUSTERS = PermissionWrapper( + 'list_host_clusters', + 'list host clusters', + 'list host clusters' +) +PERMISSION_UPDATE_HOST = PermissionWrapper( + 'update_host', 'update host', 'update host' +) +PERMISSION_DEL_HOST = PermissionWrapper( + 'del_host', 'del host', 'del host' +) +PERMISSION_LIST_HOST_CONFIG = PermissionWrapper( + 'list_host_config', 'list host config', 'list host config' +) +PERMISSION_ADD_HOST_CONFIG = PermissionWrapper( + 'add_host_config', 'add host config', 'add host config' +) +PERMISSION_DEL_HOST_CONFIG = PermissionWrapper( + 'del_host_config', 'del host config', 'del host config' +) +PERMISSION_LIST_HOST_NETWORKS = PermissionWrapper( + 'list_host_networks', + 'list host networks', + 'list host networks' +) +PERMISSION_ADD_HOST_NETWORK = PermissionWrapper( + 'add_host_network', 'add host network', 'add host network' +) +PERMISSION_DEL_HOST_NETWORK = PermissionWrapper( + 'del_host_network', 'del host network', 'del host network' +) +PERMISSION_GET_HOST_STATE = PermissionWrapper( + 'get_host_state', 'get host state', 'get host state' +) +PERMISSION_UPDATE_HOST_STATE = PermissionWrapper( + 'update_host_state', 'update host sate', 'update host state' +) +PERMISSION_LIST_CLUSTERHOSTS = PermissionWrapper( + 'list_clusterhosts', 'list cluster hosts', 'list cluster hosts' +) +PERMISSION_LIST_CLUSTERHOST_CONFIG = PermissionWrapper( + 'list_clusterhost_config', + 'list clusterhost config', + 'list clusterhost config' +) +PERMISSION_ADD_CLUSTERHOST_CONFIG = PermissionWrapper( + 'add_clusterhost_config', + 'add clusterhost config', + 'add clusterhost config' +) +PERMISSION_DEL_CLUSTERHOST_CONFIG = PermissionWrapper( + 'del_clusterhost_config', + 'del clusterhost config', + 'del clusterhost config' +) +PERMISSION_GET_CLUSTERHOST_STATE = PermissionWrapper( + 'get_clusterhost_state', + 'get clusterhost state', + 'get clusterhost state' +) +PERMISSION_UPDATE_CLUSTERHOST_STATE = PermissionWrapper( + 'update_clusterhost_state', + 'update clusterhost state', + 'update clusterhost state' +) +PERMISSION_LIST_HEALTH_REPORT = PermissionWrapper( + 'list_health_reports', + 'list health check report', + 'list health check report' +) +PERMISSION_GET_HEALTH_REPORT = PermissionWrapper( + 'get_health_report', + 'get health report', + 'get health report' +) +PERMISSION_CHECK_CLUSTER_HEALTH = PermissionWrapper( + 'start_check_cluster_health', + 'start check cluster health', + 'start check cluster health' +) +PERMISSION_SET_HEALTH_CHECK_ERROR = PermissionWrapper( + 'set_error_state', + 'set health check into error state', + 'set health check into error state' +) +PERMISSION_DELETE_REPORT = PermissionWrapper( + 'delete_reports', + 'delete health reports', + 'delete health reports' +) +PERMISSIONS = [ + PERMISSION_LIST_PERMISSIONS, + PERMISSION_LIST_SWITCHES, + PERMISSION_ADD_SWITCH, + PERMISSION_DEL_SWITCH, + PERMISSION_LIST_SWITCH_FILTERS, + PERMISSION_UPDATE_SWITCH_FILTERS, + PERMISSION_LIST_SWITCH_MACHINES, + PERMISSION_ADD_SWITCH_MACHINE, + PERMISSION_DEL_SWITCH_MACHINE, + PERMISSION_UPDATE_SWITCH_MACHINES, + PERMISSION_LIST_MACHINES, + PERMISSION_ADD_MACHINE, + PERMISSION_DEL_MACHINE, + PERMISSION_LIST_ADAPTERS, + PERMISSION_LIST_METADATAS, + PERMISSION_LIST_SUBNETS, + PERMISSION_ADD_SUBNET, + PERMISSION_DEL_SUBNET, + PERMISSION_LIST_CLUSTERS, + PERMISSION_ADD_CLUSTER, + PERMISSION_DEL_CLUSTER, + PERMISSION_LIST_CLUSTER_CONFIG, + PERMISSION_ADD_CLUSTER_CONFIG, + PERMISSION_DEL_CLUSTER_CONFIG, + PERMISSION_UPDATE_CLUSTER_HOSTS, + PERMISSION_DEL_CLUSTER_HOST, + PERMISSION_REVIEW_CLUSTER, + PERMISSION_DEPLOY_CLUSTER, + PERMISSION_GET_CLUSTER_STATE, + PERMISSION_UPDATE_CLUSTER_STATE, + PERMISSION_LIST_HOSTS, + PERMISSION_LIST_HOST_CLUSTERS, + PERMISSION_UPDATE_HOST, + PERMISSION_DEL_HOST, + PERMISSION_LIST_HOST_CONFIG, + PERMISSION_ADD_HOST_CONFIG, + PERMISSION_DEL_HOST_CONFIG, + PERMISSION_LIST_HOST_NETWORKS, + PERMISSION_ADD_HOST_NETWORK, + PERMISSION_DEL_HOST_NETWORK, + PERMISSION_GET_HOST_STATE, + PERMISSION_UPDATE_HOST_STATE, + PERMISSION_DEPLOY_HOST, + PERMISSION_LIST_CLUSTERHOSTS, + PERMISSION_LIST_CLUSTERHOST_CONFIG, + PERMISSION_ADD_CLUSTERHOST_CONFIG, + PERMISSION_DEL_CLUSTERHOST_CONFIG, + PERMISSION_GET_CLUSTERHOST_STATE, + PERMISSION_UPDATE_CLUSTERHOST_STATE, + PERMISSION_LIST_HEALTH_REPORT, + PERMISSION_GET_HEALTH_REPORT, + PERMISSION_CHECK_CLUSTER_HEALTH, + PERMISSION_SET_HEALTH_CHECK_ERROR, + PERMISSION_DELETE_REPORT +] + + +@util.deprecated +def list_permissions_internal(session, **filters): + """internal functions used only by other db.api modules.""" + return utils.list_db_objects(session, models.Permission, **filters) + + +@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) +@database.run_in_session() +@user_api.check_user_permission(PERMISSION_LIST_PERMISSIONS) +@utils.wrap_to_dict(RESP_FIELDS) +def list_permissions(user=None, session=None, **filters): + """list permissions.""" + return utils.list_db_objects( + session, models.Permission, **filters + ) + + +def _get_permission(permission_id, session=None, **kwargs): + """Get permission object by the unique key of Permission table.""" + if isinstance(permission_id, (int, long)): + return utils.get_db_object( + session, models.Permission, id=permission_id, **kwargs) + raise exception.InvalidParameter( + 'permission id %s type is not int compatible' % permission_id + ) + + +def get_permission_internal(permission_id, session=None, **kwargs): + return _get_permission(permission_id, session=session, **kwargs) + + +@utils.supported_filters() +@database.run_in_session() +@user_api.check_user_permission(PERMISSION_LIST_PERMISSIONS) +@utils.wrap_to_dict(RESP_FIELDS) +def get_permission( + permission_id, exception_when_missing=True, + user=None, session=None, **kwargs +): + """get permissions.""" + return _get_permission( + permission_id, session=session, + exception_when_missing=exception_when_missing + ) + + +def add_permissions_internal(session=None): + """internal functions used by other db.api modules only.""" + permissions = [] + for permission in PERMISSIONS: + permissions.append( + utils.add_db_object( + session, models.Permission, + True, + permission.name, + alias=permission.alias, + description=permission.description + ) + ) + + return permissions diff --git a/compass-tasks-base/db/api/switch.py b/compass-tasks-base/db/api/switch.py new file mode 100644 index 0000000..647eec0 --- /dev/null +++ b/compass-tasks-base/db/api/switch.py @@ -0,0 +1,1213 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Switch database operations.""" +import logging +import netaddr +import re + +from compass.db.api import database +from compass.db.api import permission +from compass.db.api import user as user_api +from compass.db.api import utils +from compass.db import exception +from compass.db import models +from compass.utils import setting_wrapper as setting +from compass.utils import util + + +SUPPORTED_FIELDS = ['ip_int', 'vendor', 'state'] +SUPPORTED_FILTER_FIELDS = ['ip_int', 'vendor', 'state'] +SUPPORTED_SWITCH_MACHINES_FIELDS = [ + 'switch_ip_int', 'port', 'vlans', 'mac', 'tag', 'location', + 'owner_id' +] +SUPPORTED_MACHINES_FIELDS = [ + 'port', 'vlans', 'mac', 'tag', 'location', 'owner_id' +] +SUPPORTED_SWITCH_MACHINES_HOSTS_FIELDS = [ + 'switch_ip_int', 'port', 'vlans', 'mac', + 'tag', 'location', 'os_name' +] +SUPPORTED_MACHINES_HOSTS_FIELDS = [ + 'port', 'vlans', 'mac', 'tag', 'location', + 'os_name' +] +IGNORE_FIELDS = ['id', 'created_at', 'updated_at'] +ADDED_FIELDS = ['ip'] +OPTIONAL_ADDED_FIELDS = [ + 'credentials', 'vendor', 'state', 'err_msg', 'machine_filters' +] +UPDATED_FIELDS = [ + 'ip', 'credentials', 'vendor', 'state', + 'err_msg', 'put_machine_filters' +] +PATCHED_FIELDS = ['patched_credentials', 'patched_machine_filters'] +UPDATED_FILTERS_FIELDS = ['put_machine_filters'] +PATCHED_FILTERS_FIELDS = ['patched_machine_filters'] +ADDED_MACHINES_FIELDS = ['mac'] +OPTIONAL_ADDED_MACHINES_FIELDS = [ + 'ipmi_credentials', 'tag', 'location', 'owner_id' +] +ADDED_SWITCH_MACHINES_FIELDS = ['port'] +OPTIONAL_ADDED_SWITCH_MACHINES_FIELDS = ['vlans'] +UPDATED_MACHINES_FIELDS = [ + 'ipmi_credentials', + 'tag', 'location' +] +UPDATED_SWITCH_MACHINES_FIELDS = ['port', 'vlans', 'owner_id'] +PATCHED_MACHINES_FIELDS = [ + 'patched_ipmi_credentials', + 'patched_tag', 'patched_location' +] +PATCHED_SWITCH_MACHINES_FIELDS = ['patched_vlans'] +RESP_FIELDS = [ + 'id', 'ip', 'credentials', 'vendor', 'state', 'err_msg', + 'filters', 'created_at', 'updated_at' +] +RESP_FILTERS_FIELDS = [ + 'id', 'ip', 'filters', 'created_at', 'updated_at' +] +RESP_ACTION_FIELDS = [ + 'status', 'details' +] +RESP_MACHINES_FIELDS = [ + 'id', 'switch_id', 'switch_ip', 'machine_id', 'switch_machine_id', + 'port', 'vlans', 'mac', 'owner_id', + 'ipmi_credentials', 'tag', 'location', + 'created_at', 'updated_at' +] +RESP_MACHINES_HOSTS_FIELDS = [ + 'id', 'switch_id', 'switch_ip', 'machine_id', 'switch_machine_id', + 'port', 'vlans', 'mac', + 'ipmi_credentials', 'tag', 'location', 'ip', + 'name', 'hostname', 'os_name', 'owner', + 'os_installer', 'reinstall_os', 'os_installed', + 'clusters', 'created_at', 'updated_at' +] +RESP_CLUSTER_FIELDS = [ + 'name', 'id' +] + + +def _check_machine_filters(machine_filters): + """Check if machine filters format is acceptable.""" + logging.debug('check machine filters: %s', machine_filters) + models.Switch.parse_filters(machine_filters) + + +def _check_vlans(vlans): + """Check vlans format is acceptable.""" + for vlan in vlans: + if not isinstance(vlan, int): + raise exception.InvalidParameter( + 'vlan %s is not int' % vlan + ) + + +@utils.supported_filters( + ADDED_FIELDS, + optional_support_keys=OPTIONAL_ADDED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates( + ip=utils.check_ip, + credentials=utils.check_switch_credentials, + machine_filters=_check_machine_filters +) +@utils.wrap_to_dict(RESP_FIELDS) +def _add_switch( + ip, exception_when_existing=True, + machine_filters=setting.SWITCHES_DEFAULT_FILTERS, + session=None, **kwargs +): + """Add switch by switch ip.""" + ip_int = long(netaddr.IPAddress(ip)) + return utils.add_db_object( + session, models.Switch, exception_when_existing, ip_int, + machine_filters=machine_filters, **kwargs + ) + + +def get_switch_internal( + switch_id, session=None, **kwargs +): + """Get switch by switch id. + + Should only be used by other files under db/api + """ + return _get_switch(switch_id, session=session, **kwargs) + + +def _get_switch(switch_id, session=None, **kwargs): + """Get Switch object switch id.""" + if isinstance(switch_id, (int, long)): + return utils.get_db_object( + session, models.Switch, + id=switch_id, **kwargs + ) + raise exception.InvalidParameter( + 'switch id %s type is not int compatible' % switch_id) + + +def _get_switch_by_ip(switch_ip, session=None, **kwargs): + """Get switch by switch ip.""" + switch_ip_int = long(netaddr.IPAddress(switch_ip)) + return utils.get_db_object( + session, models.Switch, + ip_int=switch_ip_int, **kwargs + ) + + +def _get_switch_machine(switch_id, machine_id, session=None, **kwargs): + """Get switch machine by switch id and machine id.""" + switch = _get_switch(switch_id, session=session) + from compass.db.api import machine as machine_api + machine = machine_api.get_machine_internal(machine_id, session=session) + return utils.get_db_object( + session, models.SwitchMachine, + switch_id=switch.id, machine_id=machine.id, **kwargs + ) + + +def _get_switchmachine(switch_machine_id, session=None, **kwargs): + """Get switch machine by switch_machine_id.""" + if not isinstance(switch_machine_id, (int, long)): + raise exception.InvalidParameter( + 'switch machine id %s type is not int compatible' % ( + switch_machine_id + ) + ) + return utils.get_db_object( + session, models.SwitchMachine, + switch_machine_id=switch_machine_id, **kwargs + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_SWITCHES +) +@utils.wrap_to_dict(RESP_FIELDS) +def get_switch( + switch_id, exception_when_missing=True, + user=None, session=None, **kwargs +): + """get a switch by switch id.""" + return _get_switch( + switch_id, session=session, + exception_when_missing=exception_when_missing + ) + + +@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_SWITCHES +) +@utils.wrap_to_dict(RESP_FIELDS) +def list_switches(user=None, session=None, **filters): + """List switches.""" + # TODO(xicheng): should discuss with weidong. + # If we can deprecate the use of DEFAULT_SWITCH_IP, + # The code will be simpler. + # The UI should use /machines-hosts instead of + # /switches-machines-hosts and can show multi switch ip/port + # under one row of machine info. + switches = utils.list_db_objects( + session, models.Switch, **filters + ) + if 'ip_int' in filters: + return switches + else: + return [ + switch for switch in switches + if switch.ip != setting.DEFAULT_SWITCH_IP + ] + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEL_SWITCH +) +@utils.wrap_to_dict(RESP_FIELDS) +def del_switch(switch_id, user=None, session=None, **kwargs): + """Delete a switch. + + If switch is not the default switch, and the machine under this switch + is only connected to this switch, the machine will be moved to connect + to default switch. Otherwise we can only simply delete the switch + machine. The purpose here to make sure there is no machine not + connecting to any switch. + """ + # TODO(xicheng): Simplify the logic if the default switch feature + # can be deprecated. + switch = _get_switch(switch_id, session=session) + default_switch = _get_switch_by_ip( + setting.DEFAULT_SWITCH_IP, session=session + ) + if switch.id != default_switch.id: + for switch_machine in switch.switch_machines: + machine = switch_machine.machine + if len(machine.switch_machines) <= 1: + utils.add_db_object( + session, models.SwitchMachine, + False, + default_switch.id, machine.id, + port=switch_machine.port + ) + return utils.del_db_object(session, switch) + + +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_SWITCH +) +def add_switch( + exception_when_existing=True, ip=None, + user=None, session=None, **kwargs +): + """Create a switch.""" + return _add_switch( + ip, + exception_when_existing=exception_when_existing, + session=session, **kwargs + ) + + +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_SWITCH +) +def add_switches( + exception_when_existing=False, + data=[], user=None, session=None +): + """Create switches.""" + # TODO(xicheng): simplify the batch api. + switches = [] + fail_switches = [] + for switch_data in data: + switch_object = _get_switch_by_ip( + switch_data['ip'], session=session, + exception_when_missing=False + ) + if switch_object: + logging.error('ip %s exists in switch %s' % ( + switch_data['ip'], switch_object.id + )) + fail_switches.append(switch_data) + else: + switches.append( + _add_switch( + exception_when_existing=exception_when_existing, + session=session, + **switch_data + ) + ) + return { + 'switches': switches, + 'fail_switches': fail_switches + } + + +@utils.wrap_to_dict(RESP_FIELDS) +def _update_switch(switch_id, session=None, **kwargs): + """Update a switch.""" + switch = _get_switch(switch_id, session=session) + return utils.update_db_object(session, switch, **kwargs) + + +# replace machine_filters in kwargs to put_machine_filters, +# which is used to tell db this is a put action for the field. +@utils.replace_filters( + machine_filters='put_machine_filters' +) +@utils.supported_filters( + optional_support_keys=UPDATED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates( + credentials=utils.check_switch_credentials, + put_machine_filters=_check_machine_filters +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_SWITCH +) +def update_switch(switch_id, user=None, session=None, **kwargs): + """Update fields of a switch.""" + return _update_switch(switch_id, session=session, **kwargs) + + +# replace credentials to patched_credentials, +# machine_filters to patched_machine_filters in kwargs. +# This is to tell db they are patch action to the above fields. +@utils.replace_filters( + credentials='patched_credentials', + machine_filters='patched_machine_filters' +) +@utils.supported_filters( + optional_support_keys=PATCHED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates( + patched_machine_filters=_check_machine_filters +) +@database.run_in_session() +@utils.output_validates( + credentials=utils.check_switch_credentials +) +@user_api.check_user_permission( + permission.PERMISSION_ADD_SWITCH +) +def patch_switch(switch_id, user=None, session=None, **kwargs): + """Patch fields of a switch.""" + return _update_switch(switch_id, session=session, **kwargs) + + +@util.deprecated +@utils.supported_filters(optional_support_keys=SUPPORTED_FILTER_FIELDS) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_SWITCH_FILTERS +) +@utils.wrap_to_dict(RESP_FILTERS_FIELDS) +def list_switch_filters(user=None, session=None, **filters): + """List all switches' filters.""" + return utils.list_db_objects( + session, models.Switch, **filters + ) + + +@util.deprecated +@utils.supported_filters() +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_SWITCH_FILTERS +) +@utils.wrap_to_dict(RESP_FILTERS_FIELDS) +def get_switch_filters( + switch_id, exception_when_missing=True, + user=None, session=None, **kwargs +): + """get filters of a switch.""" + return _get_switch( + switch_id, session=session, + exception_when_missing=exception_when_missing + ) + + +@util.deprecated +@utils.replace_filters( + machine_filters='put_machine_filters' +) +@utils.supported_filters( + optional_support_keys=UPDATED_FILTERS_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates(put_machine_filters=_check_machine_filters) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_SWITCH_FILTERS +) +@utils.wrap_to_dict(RESP_FILTERS_FIELDS) +def update_switch_filters(switch_id, user=None, session=None, **kwargs): + """Update filters of a switch.""" + switch = _get_switch(switch_id, session=session) + return utils.update_db_object(session, switch, **kwargs) + + +@util.deprecated +@utils.replace_filters( + machine_filters='patched_machine_filters' +) +@utils.supported_filters( + optional_support_keys=PATCHED_FILTERS_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates(patched_machine_filters=_check_machine_filters) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_SWITCH_FILTERS +) +@utils.wrap_to_dict(RESP_FILTERS_FIELDS) +def patch_switch_filter(switch_id, user=None, session=None, **kwargs): + """Patch filters to a switch.""" + switch = _get_switch(switch_id, session=session) + return utils.update_db_object(session, switch, **kwargs) + + +@util.deprecated +def get_switch_machines_internal(session, **filters): + return utils.list_db_objects( + session, models.SwitchMachine, **filters + ) + + +def _filter_port(port_filter, obj): + """filter switch machines by port. + + supported port_filter keys: [ + 'startswith', 'endswith', 'resp_lt', + 'resp_le', 'resp_gt', 'resp_ge', 'resp_range' + ] + + port_filter examples: + { + 'startswitch': 'ae', 'endswith': '', + 'resp_ge': 20, 'resp_le': 30, + } + """ + port_prefix = port_filter.get('startswith', '') + port_suffix = port_filter.get('endswith', '') + pattern = re.compile(r'%s(\d+)%s' % (port_prefix, port_suffix)) + match = pattern.match(obj) + if not match: + return False + port_number = int(match.group(1)) + if ( + 'resp_lt' in port_filter and + port_number >= port_filter['resp_lt'] + ): + return False + if ( + 'resp_le' in port_filter and + port_number > port_filter['resp_le'] + ): + return False + if ( + 'resp_gt' in port_filter and + port_number <= port_filter['resp_gt'] + ): + return False + if ( + 'resp_ge' in port_filter and + port_number < port_filter['resp_ge'] + ): + return False + if 'resp_range' in port_filter: + resp_range = port_filter['resp_range'] + if not isinstance(resp_range, list): + resp_range = [resp_range] + in_range = False + for port_start, port_end in resp_range: + if port_start <= port_number <= port_end: + in_range = True + break + if not in_range: + return False + return True + + +def _filter_vlans(vlan_filter, obj): + """Filter switch machines by vlan. + + supported keys in vlan_filter: + ['resp_in'] + """ + vlans = set(obj) + if 'resp_in' in vlan_filter: + resp_vlans = set(vlan_filter['resp_in']) + if not (vlans & resp_vlans): + return False + return True + + +@utils.output_filters( + port=_filter_port, vlans=_filter_vlans, + tag=utils.general_filter_callback, + location=utils.general_filter_callback +) +@utils.wrap_to_dict(RESP_MACHINES_FIELDS) +def _filter_switch_machines(switch_machines): + """Get filtered switch machines. + + The filters are defined in each switch. + """ + return [ + switch_machine for switch_machine in switch_machines + if not switch_machine.filtered + ] + + +@utils.output_filters( + missing_ok=True, + port=_filter_port, vlans=_filter_vlans, + tag=utils.general_filter_callback, + location=utils.general_filter_callback, + os_name=utils.general_filter_callback, +) +@utils.wrap_to_dict( + RESP_MACHINES_HOSTS_FIELDS, + clusters=RESP_CLUSTER_FIELDS +) +def _filter_switch_machines_hosts(switch_machines): + """Similar as _filter_switch_machines, but also return host info.""" + filtered_switch_machines = [ + switch_machine for switch_machine in switch_machines + if not switch_machine.filtered + ] + switch_machines_hosts = [] + for switch_machine in filtered_switch_machines: + machine = switch_machine.machine + host = machine.host + if host: + switch_machine_host_dict = host.to_dict() + else: + switch_machine_host_dict = machine.to_dict() + switch_machine_host_dict.update( + switch_machine.to_dict() + ) + switch_machines_hosts.append(switch_machine_host_dict) + return switch_machines_hosts + + +@utils.supported_filters( + optional_support_keys=SUPPORTED_MACHINES_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_SWITCH_MACHINES +) +def list_switch_machines( + switch_id, user=None, session=None, **filters +): + """Get switch machines of a switch.""" + switch = _get_switch(switch_id, session=session) + switch_machines = utils.list_db_objects( + session, models.SwitchMachine, switch_id=switch.id, **filters + ) + if not user.is_admin and len(switch_machines): + switch_machines = [m for m in switch_machines if m.machine.owner_id == user.id] + return _filter_switch_machines(switch_machines) + + +# replace ip_int to switch_ip_int in kwargs +@utils.replace_filters( + ip_int='switch_ip_int' +) +@utils.supported_filters( + optional_support_keys=SUPPORTED_SWITCH_MACHINES_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_SWITCH_MACHINES +) +def list_switchmachines(user=None, session=None, **filters): + """List switch machines.""" + switch_machines = utils.list_db_objects( + session, models.SwitchMachine, **filters + ) + return _filter_switch_machines( + switch_machines + ) + + +@utils.supported_filters( + optional_support_keys=SUPPORTED_MACHINES_HOSTS_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_SWITCH_MACHINES +) +def list_switch_machines_hosts( + switch_id, user=None, session=None, **filters +): + """Get switch machines and possible hosts of a switch.""" + switch = _get_switch(switch_id, session=session) + switch_machines = utils.list_db_objects( + session, models.SwitchMachine, switch_id=switch.id, **filters + ) + return _filter_switch_machines_hosts( + switch_machines + ) + + +# replace ip_int to switch_ip_int in kwargs +@utils.replace_filters( + ip_int='switch_ip_int' +) +@utils.supported_filters( + optional_support_keys=SUPPORTED_SWITCH_MACHINES_HOSTS_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_SWITCH_MACHINES +) +def list_switchmachines_hosts(user=None, session=None, **filters): + """List switch machines hnd possible hosts.""" + switch_machines = utils.list_db_objects( + session, models.SwitchMachine, **filters + ) + if not user.is_admin and len(switch_machines): + switch_machines = [m for m in switch_machines if m.machine.owner_id == user.id] + return _filter_switch_machines_hosts( + switch_machines + ) + + +@utils.supported_filters( + ADDED_MACHINES_FIELDS, + optional_support_keys=OPTIONAL_ADDED_MACHINES_FIELDS, + ignore_support_keys=OPTIONAL_ADDED_SWITCH_MACHINES_FIELDS +) +@utils.input_validates(mac=utils.check_mac) +def _add_machine_if_not_exist(mac=None, session=None, **kwargs): + """Add machine if the mac does not exist in any machine.""" + return utils.add_db_object( + session, models.Machine, False, + mac, **kwargs) + + +@utils.supported_filters( + ADDED_SWITCH_MACHINES_FIELDS, + optional_support_keys=OPTIONAL_ADDED_SWITCH_MACHINES_FIELDS, + ignore_support_keys=OPTIONAL_ADDED_MACHINES_FIELDS +) +@utils.input_validates(vlans=_check_vlans) +def _add_switch_machine_only( + switch, machine, exception_when_existing=True, + session=None, owner_id=None, port=None, **kwargs +): + """add a switch machine.""" + return utils.add_db_object( + session, models.SwitchMachine, + exception_when_existing, + switch.id, machine.id, port=port, + owner_id=owner_id, + **kwargs + ) + + +@utils.supported_filters( + ADDED_MACHINES_FIELDS + ADDED_SWITCH_MACHINES_FIELDS, + optional_support_keys=( + OPTIONAL_ADDED_MACHINES_FIELDS + + OPTIONAL_ADDED_SWITCH_MACHINES_FIELDS + ), + ignore_support_keys=IGNORE_FIELDS +) +@utils.wrap_to_dict(RESP_MACHINES_FIELDS) +def _add_switch_machine( + switch_id, exception_when_existing=True, + mac=None, port=None, session=None, owner_id=None, **kwargs +): + """Add switch machine. + + If underlying machine does not exist, also create the underlying + machine. + """ + switch = _get_switch(switch_id, session=session) + machine = _add_machine_if_not_exist( + mac=mac, session=session, owner_id=owner_id, **kwargs + ) + return _add_switch_machine_only( + switch, machine, + exception_when_existing, + port=port, session=session, **kwargs + ) + + +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_SWITCH_MACHINE +) +def add_switch_machine( + switch_id, exception_when_existing=True, + mac=None, user=None, session=None, + owner_id=None, **kwargs +): + """Add switch machine to a switch.""" + return _add_switch_machine( + switch_id, + exception_when_existing=exception_when_existing, + mac=mac, session=session, owner_id=owner_id, **kwargs + ) + + +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_SWITCH_MACHINE +) +@utils.wrap_to_dict( + [ + 'switches_machines', + 'duplicate_switches_machines', + 'fail_switches_machines' + ], + switches_machines=RESP_MACHINES_FIELDS, + duplicate_switches_machines=RESP_MACHINES_FIELDS +) +def add_switch_machines( + exception_when_existing=False, + data=[], user=None, session=None, owner_id=None +): + """Add switch machines.""" + switch_machines = [] + duplicate_switch_machines = [] + failed_switch_machines = [] + switches_mapping = {} + switch_machines_mapping = {} + switch_ips = [] + for item_data in data: + switch_ip = item_data['switch_ip'] + if switch_ip not in switches_mapping: + switch_object = _get_switch_by_ip( + switch_ip, session=session, + exception_when_missing=False + ) + if switch_object: + switch_ips.append(switch_ip) + switches_mapping[switch_ip] = switch_object + else: + logging.error( + 'switch %s does not exist' % switch_ip + ) + item_data.pop('switch_ip') + failed_switch_machines.append(item_data) + else: + switch_object = switches_mapping[switch_ip] + if switch_object: + item_data.pop('switch_ip') + switch_machines_mapping.setdefault( + switch_object.id, [] + ).append(item_data) + + for switch_ip in switch_ips: + switch_object = switches_mapping[switch_ip] + switch_id = switch_object.id + machines = switch_machines_mapping[switch_id] + for machine in machines: + mac = machine['mac'] + machine_object = _add_machine_if_not_exist( + mac=mac, session=session + ) + switch_machine_object = _get_switch_machine( + switch_id, machine_object.id, session=session, + exception_when_missing=False + ) + if switch_machine_object: + port = machine['port'] + switch_machine_id = switch_machine_object.switch_machine_id + exist_port = switch_machine_object.port + if exist_port != port: + logging.error( + 'switch machine %s exist port %s is ' + 'different from added port %s' % ( + switch_machine_id, + exist_port, port + ) + ) + failed_switch_machines.append(machine) + else: + logging.error( + 'iswitch machine %s is dulicate, ' + 'will not be override' % switch_machine_id + ) + duplicate_switch_machines.append(machine) + else: + del machine['mac'] + switch_machines.append(_add_switch_machine_only( + switch_object, machine_object, + exception_when_existing, + session=session, owner_id=owner_id, **machine + )) + return { + 'switches_machines': switch_machines, + 'duplicate_switches_machines': duplicate_switch_machines, + 'fail_switches_machines': failed_switch_machines + } + + +@utils.supported_filters(optional_support_keys=['find_machines']) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_SWITCH_MACHINES +) +@utils.wrap_to_dict(RESP_ACTION_FIELDS) +def poll_switch(switch_id, user=None, session=None, **kwargs): + """poll switch to get machines.""" + from compass.tasks import client as celery_client + switch = _get_switch(switch_id, session=session) + celery_client.celery.send_task( + 'compass.tasks.pollswitch', + (user.email, switch.ip, switch.credentials), + queue=user.email, + exchange=user.email, + routing_key=user.email + ) + return { + 'status': 'action %s sent' % kwargs, + 'details': { + } + } + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_SWITCH_MACHINES +) +@utils.wrap_to_dict(RESP_MACHINES_FIELDS) +def get_switch_machine( + switch_id, machine_id, exception_when_missing=True, + user=None, session=None, **kwargs +): + """get a switch machine by switch id and machine id.""" + return _get_switch_machine( + switch_id, machine_id, session=session, + exception_when_missing=exception_when_missing + ) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_SWITCH_MACHINES +) +@utils.wrap_to_dict(RESP_MACHINES_FIELDS) +def get_switchmachine( + switch_machine_id, exception_when_missing=True, + user=None, session=None, **kwargs +): + """get a switch machine by switch_machine_id.""" + return _get_switchmachine( + switch_machine_id, session=session, + exception_when_missing=exception_when_missing + ) + + +@utils.supported_filters( + optional_support_keys=( + UPDATED_MACHINES_FIELDS + PATCHED_MACHINES_FIELDS + ), + ignore_support_keys=( + UPDATED_SWITCH_MACHINES_FIELDS + PATCHED_SWITCH_MACHINES_FIELDS + ) +) +def _update_machine_if_necessary( + machine, session=None, **kwargs +): + """Update machine is there is something to update.""" + utils.update_db_object( + session, machine, **kwargs + ) + + +@utils.supported_filters( + optional_support_keys=( + UPDATED_SWITCH_MACHINES_FIELDS + PATCHED_SWITCH_MACHINES_FIELDS + ), + ignore_support_keys=( + UPDATED_MACHINES_FIELDS + PATCHED_MACHINES_FIELDS + ) +) +def _update_switch_machine_only(switch_machine, session=None, **kwargs): + """Update switch machine.""" + return utils.update_db_object( + session, switch_machine, **kwargs + ) + + +def _update_switch_machine( + switch_machine, session=None, **kwargs +): + """Update switch machine. + + If there are some attributes of underlying machine need to update, + also update them in underlying machine. + """ + _update_machine_if_necessary( + switch_machine.machine, session=session, **kwargs + ) + return _update_switch_machine_only( + switch_machine, session=session, **kwargs + ) + + +@utils.supported_filters( + optional_support_keys=( + UPDATED_MACHINES_FIELDS + UPDATED_SWITCH_MACHINES_FIELDS + ), + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates(vlans=_check_vlans) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_SWITCH_MACHINE +) +@utils.wrap_to_dict(RESP_MACHINES_FIELDS) +def update_switch_machine( + switch_id, machine_id, user=None, + session=None, **kwargs +): + """Update switch machine by switch id and machine id.""" + switch_machine = _get_switch_machine( + switch_id, machine_id, session=session + ) + return _update_switch_machine( + switch_machine, + session=session, **kwargs + ) + + +@utils.supported_filters( + optional_support_keys=( + UPDATED_MACHINES_FIELDS + UPDATED_SWITCH_MACHINES_FIELDS + ), + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates(vlans=_check_vlans) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_SWITCH_MACHINE +) +@utils.wrap_to_dict(RESP_MACHINES_FIELDS) +def update_switchmachine(switch_machine_id, user=None, session=None, **kwargs): + """Update switch machine by switch_machine_id.""" + switch_machine = _get_switchmachine( + switch_machine_id, session=session + ) + return _update_switch_machine( + switch_machine, + session=session, **kwargs + ) + + +# replace [vlans, ipmi_credentials, tag, location] to +# [patched_vlans, patched_ipmi_credentials, patched_tag, +# patched_location] in kwargs. It tells db these fields will +# be patched. +@utils.replace_filters( + vlans='patched_vlans', + ipmi_credentials='patched_ipmi_credentials', + tag='patched_tag', + location='patched_location' +) +@utils.supported_filters( + optional_support_keys=( + PATCHED_MACHINES_FIELDS + PATCHED_SWITCH_MACHINES_FIELDS + ), + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates(patched_vlans=_check_vlans) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_SWITCH_MACHINE +) +@utils.wrap_to_dict(RESP_MACHINES_FIELDS) +def patch_switch_machine( + switch_id, machine_id, user=None, + session=None, **kwargs +): + """Patch switch machine by switch_id and machine_id.""" + switch_machine = _get_switch_machine( + switch_id, machine_id, session=session + ) + return _update_switch_machine( + switch_machine, + session=session, **kwargs + ) + + +# replace [vlans, ipmi_credentials, tag, location] to +# [patched_vlans, patched_ipmi_credentials, patched_tag, +# patched_location] in kwargs. It tells db these fields will +# be patched. +@utils.replace_filters( + vlans='patched_vlans', + ipmi_credentials='patched_ipmi_credentials', + tag='patched_tag', + location='patched_location' +) +@utils.supported_filters( + optional_support_keys=( + PATCHED_MACHINES_FIELDS + PATCHED_SWITCH_MACHINES_FIELDS + ), + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates(patched_vlans=_check_vlans) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_ADD_SWITCH_MACHINE +) +@utils.wrap_to_dict(RESP_MACHINES_FIELDS) +def patch_switchmachine(switch_machine_id, user=None, session=None, **kwargs): + """Patch switch machine by switch_machine_id.""" + switch_machine = _get_switchmachine( + switch_machine_id, session=session + ) + return _update_switch_machine( + switch_machine, + session=session, **kwargs + ) + + +def _del_switch_machine( + switch_machine, session=None +): + """Delete switch machine. + + If this is the last switch machine associated to underlying machine, + add a switch machine record to default switch to make the machine + searchable. + """ + default_switch = _get_switch_by_ip( + setting.DEFAULT_SWITCH_IP, session=session + ) + machine = switch_machine.machine + if len(machine.switch_machines) <= 1: + utils.add_db_object( + session, models.SwitchMachine, + False, + default_switch.id, machine.id, + port=switch_machine.port + ) + return utils.del_db_object(session, switch_machine) + + +@utils.supported_filters() +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEL_SWITCH_MACHINE +) +@utils.wrap_to_dict(RESP_MACHINES_FIELDS) +def del_switch_machine( + switch_id, machine_id, user=None, + session=None, **kwargs +): + """Delete switch machine by switch id and machine id.""" + switch_machine = _get_switch_machine( + switch_id, machine_id, session=session + ) + return _del_switch_machine(switch_machine, session=session) + + +@utils.supported_filters() +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_DEL_SWITCH_MACHINE +) +@utils.wrap_to_dict(RESP_MACHINES_FIELDS) +def del_switchmachine(switch_machine_id, user=None, session=None, **kwargs): + """Delete switch machine by switch_machine_id.""" + switch_machine = _get_switchmachine( + switch_machine_id, session=session + ) + return _del_switch_machine(switch_machine, session=session) + + +@utils.supported_filters( + ['machine_id'], + optional_support_keys=UPDATED_SWITCH_MACHINES_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +def _add_machine_to_switch( + switch_id, machine_id, session=None, **kwargs +): + """Add machine to switch.""" + switch = _get_switch(switch_id, session=session) + from compass.db.api import machine as machine_api + machine = machine_api.get_machine_internal( + machine_id, session=session + ) + _add_switch_machine_only( + switch, machine, False, + owner_id=machine.owner_id, **kwargs + ) + + +def _add_machines(switch, machines, session=None): + """Add machines to switch. + + Args: + machines: list of dict which contains attributes to + add machine to switch. + + machines example: + {{'machine_id': 1, 'port': 'ae20'}] + """ + for machine in machines: + _add_machine_to_switch( + switch.id, session=session, **machine + ) + + +def _remove_machines(switch, machines, session=None): + """Remove machines from switch. + + Args: + machines: list of machine id. + + machines example: + [1,2] + """ + utils.del_db_objects( + session, models.SwitchMachine, + switch_id=switch.id, machine_id=machines + ) + + +def _set_machines(switch, machines, session=None): + """Reset machines to a switch. + + Args: + machines: list of dict which contains attributes to + add machine to switch. + + machines example: + {{'machine_id': 1, 'port': 'ae20'}] + """ + utils.del_db_objects( + session, models.SwitchMachine, + switch_id=switch.id + ) + for switch_machine in machines: + _add_machine_to_switch( + switch.id, session=session, **switch_machine + ) + + +@utils.supported_filters( + optional_support_keys=[ + 'add_machines', 'remove_machines', 'set_machines' + ] +) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_UPDATE_SWITCH_MACHINES +) +@utils.wrap_to_dict(RESP_MACHINES_FIELDS) +def update_switch_machines( + switch_id, add_machines=[], remove_machines=[], + set_machines=None, user=None, session=None, **kwargs +): + """update switch's machines""" + switch = _get_switch(switch_id, session=session) + if remove_machines: + _remove_machines( + switch, remove_machines, session=session + ) + if add_machines: + _add_machines( + switch, add_machines, session=session + ) + if set_machines is not None: + _set_machines( + switch, set_machines, session=session + ) + return switch.switch_machines diff --git a/compass-tasks-base/db/api/user.py b/compass-tasks-base/db/api/user.py new file mode 100644 index 0000000..db039eb --- /dev/null +++ b/compass-tasks-base/db/api/user.py @@ -0,0 +1,553 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""User database operations.""" +import datetime +import functools +import logging +import re + +from flask.ext.login import UserMixin + +from compass.db.api import database +from compass.db.api import utils +from compass.db import exception +from compass.db import models + +from compass.utils import setting_wrapper as setting +from compass.utils import util + + +SUPPORTED_FIELDS = ['email', 'is_admin', 'active'] +PERMISSION_SUPPORTED_FIELDS = ['name'] +SELF_UPDATED_FIELDS = ['email', 'firstname', 'lastname', 'password'] +ADMIN_UPDATED_FIELDS = ['is_admin', 'active'] +IGNORE_FIELDS = ['id', 'created_at', 'updated_at'] +UPDATED_FIELDS = [ + 'email', 'firstname', 'lastname', 'password', 'is_admin', 'active' +] +ADDED_FIELDS = ['email', 'password'] +OPTIONAL_ADDED_FIELDS = ['is_admin', 'active'] +PERMISSION_ADDED_FIELDS = ['permission_id'] +RESP_FIELDS = [ + 'id', 'email', 'is_admin', 'active', 'firstname', + 'lastname', 'created_at', 'updated_at' +] +RESP_TOKEN_FIELDS = [ + 'id', 'user_id', 'token', 'expire_timestamp' +] +PERMISSION_RESP_FIELDS = [ + 'id', 'user_id', 'permission_id', 'name', 'alias', 'description', + 'created_at', 'updated_at' +] + + +def _check_email(email): + """Check email is email format.""" + if '@' not in email: + raise exception.InvalidParameter( + 'there is no @ in email address %s.' % email + ) + + +def _check_user_permission(user, permission, session=None): + """Check user has permission.""" + if not user: + logging.info('empty user means the call is from internal') + return + if user.is_admin: + return + + user_permission = utils.get_db_object( + session, models.UserPermission, + False, user_id=user.id, name=permission.name + ) + if not user_permission: + raise exception.Forbidden( + 'user %s does not have permission %s' % ( + user.email, permission.name + ) + ) + + +def check_user_permission(permission): + """Decorator to check user having permission.""" + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + user = kwargs.get('user') + if user is not None: + session = kwargs.get('session') + if session is None: + raise exception.DatabaseException( + 'wrapper check_user_permission does not run in session' + ) + _check_user_permission(user, permission, session=session) + return func(*args, **kwargs) + else: + return func(*args, **kwargs) + return wrapper + return decorator + + +def check_user_admin(): + """Decorator to check user is admin.""" + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + user = kwargs.get('user') + if user is not None: + if not user.is_admin: + raise exception.Forbidden( + 'User %s is not admin.' % ( + user.email + ) + ) + return func(*args, **kwargs) + else: + return func(*args, **kwargs) + return wrapper + return decorator + + +def check_user_admin_or_owner(): + """Decorator to check user is admin or the owner of the resource.""" + def decorator(func): + @functools.wraps(func) + def wrapper(user_id, *args, **kwargs): + user = kwargs.get('user') + if user is not None: + session = kwargs.get('session') + if session is None: + raise exception.DatabaseException( + 'wrapper check_user_admin_or_owner is ' + 'not called in session' + ) + check_user = _get_user(user_id, session=session) + if not user.is_admin and user.id != check_user.id: + raise exception.Forbidden( + 'User %s is not admin or the owner of user %s.' % ( + user.email, check_user.email + ) + ) + + return func( + user_id, *args, **kwargs + ) + else: + return func( + user_id, *args, **kwargs + ) + return wrapper + return decorator + + +def _add_user_permissions(user, session=None, **permission_filters): + """add permissions to a user.""" + from compass.db.api import permission as permission_api + for api_permission in permission_api.list_permissions( + session=session, **permission_filters + ): + utils.add_db_object( + session, models.UserPermission, False, + user.id, api_permission['id'] + ) + + +def _remove_user_permissions(user, session=None, **permission_filters): + """remove permissions from a user.""" + from compass.db.api import permission as permission_api + permission_ids = [ + api_permission['id'] + for api_permission in permission_api.list_permissions( + session=session, **permission_filters + ) + ] + utils.del_db_objects( + session, models.UserPermission, + user_id=user.id, permission_id=permission_ids + ) + + +def _set_user_permissions(user, session=None, **permission_filters): + """set permissions to a user.""" + utils.del_db_objects( + session, models.UserPermission, + user_id=user.id + ) + _add_user_permissions(session, user, **permission_filters) + + +class UserWrapper(UserMixin): + """Wrapper class provided to flask.""" + + def __init__( + self, id, email, crypted_password, + active=True, is_admin=False, + expire_timestamp=None, token='', **kwargs + ): + self.id = id + self.email = email + self.password = crypted_password + self.active = active + self.is_admin = is_admin + self.expire_timestamp = expire_timestamp + if not token: + self.token = self.get_auth_token() + else: + self.token = token + super(UserWrapper, self).__init__() + + def authenticate(self, password): + if not util.encrypt(password, self.password) == self.password: + raise exception.Unauthorized('%s password mismatch' % self.email) + + def get_auth_token(self): + return util.encrypt(self.email) + + def is_active(self): + return self.active + + def get_id(self): + return self.token + + def is_authenticated(self): + current_time = datetime.datetime.now() + return ( + not self.expire_timestamp or + current_time < self.expire_timestamp + ) + + def __str__(self): + return '%s[email:%s,password:%s]' % ( + self.__class__.__name__, self.email, self.password) + + +@database.run_in_session() +def get_user_object(email, session=None, **kwargs): + """get user and convert to UserWrapper object.""" + user = utils.get_db_object( + session, models.User, False, email=email + ) + if not user: + raise exception.Unauthorized( + '%s unauthorized' % email + ) + user_dict = user.to_dict() + user_dict.update(kwargs) + return UserWrapper(**user_dict) + + +@database.run_in_session(exception_when_in_session=False) +def get_user_object_from_token(token, session=None): + """Get user from token and convert to UserWrapper object. + + ::note: + get_user_object_from_token may be called in session. + """ + expire_timestamp = { + 'ge': datetime.datetime.now() + } + user_token = utils.get_db_object( + session, models.UserToken, False, + token=token, expire_timestamp=expire_timestamp + ) + if not user_token: + raise exception.Unauthorized( + 'invalid user token: %s' % token + ) + user_dict = _get_user( + user_token.user_id, session=session + ).to_dict() + user_dict['token'] = token + expire_timestamp = user_token.expire_timestamp + user_dict['expire_timestamp'] = expire_timestamp + return UserWrapper(**user_dict) + + +@utils.supported_filters() +@database.run_in_session() +@utils.wrap_to_dict(RESP_TOKEN_FIELDS) +def record_user_token( + token, expire_timestamp, user=None, session=None +): + """record user token in database.""" + user_token = utils.get_db_object( + session, models.UserToken, False, + user_id=user.id, token=token + ) + if not user_token: + return utils.add_db_object( + session, models.UserToken, True, + token, user_id=user.id, + expire_timestamp=expire_timestamp + ) + elif expire_timestamp > user_token.expire_timestamp: + return utils.update_db_object( + session, user_token, expire_timestamp=expire_timestamp + ) + return user_token + + +@utils.supported_filters() +@database.run_in_session() +@utils.wrap_to_dict(RESP_TOKEN_FIELDS) +def clean_user_token(token, user=None, session=None): + """clean user token in database.""" + return utils.del_db_objects( + session, models.UserToken, + token=token, user_id=user.id + ) + + +def _get_user(user_id, session=None, **kwargs): + """Get user object by user id.""" + if isinstance(user_id, (int, long)): + return utils.get_db_object( + session, models.User, id=user_id, **kwargs + ) + raise exception.InvalidParameter( + 'user id %s type is not int compatible' % user_id + ) + + +@utils.supported_filters() +@database.run_in_session() +@check_user_admin_or_owner() +@utils.wrap_to_dict(RESP_FIELDS) +def get_user( + user_id, exception_when_missing=True, + user=None, session=None, **kwargs +): + """get a user.""" + return _get_user( + user_id, session=session, + exception_when_missing=exception_when_missing + ) + + +@utils.supported_filters() +@database.run_in_session() +@utils.wrap_to_dict(RESP_FIELDS) +def get_current_user( + exception_when_missing=True, user=None, + session=None, **kwargs +): + """get current user.""" + return _get_user( + user.id, session=session, + exception_when_missing=exception_when_missing + ) + + +@utils.supported_filters( + optional_support_keys=SUPPORTED_FIELDS +) +@database.run_in_session() +@check_user_admin() +@utils.wrap_to_dict(RESP_FIELDS) +def list_users(user=None, session=None, **filters): + """List all users.""" + return utils.list_db_objects( + session, models.User, **filters + ) + + +@utils.input_validates(email=_check_email) +@utils.supported_filters( + ADDED_FIELDS, + optional_support_keys=OPTIONAL_ADDED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@check_user_admin() +@utils.wrap_to_dict(RESP_FIELDS) +def add_user( + exception_when_existing=True, user=None, + session=None, email=None, **kwargs +): + """Create a user and return created user object.""" + add_user = utils.add_db_object( + session, models.User, + exception_when_existing, email, + **kwargs) + _add_user_permissions( + add_user, + session=session, + name=setting.COMPASS_DEFAULT_PERMISSIONS + ) + return add_user + + +@utils.supported_filters() +@database.run_in_session() +@check_user_admin() +@utils.wrap_to_dict(RESP_FIELDS) +def del_user(user_id, user=None, session=None, **kwargs): + """delete a user and return the deleted user object.""" + del_user = _get_user(user_id, session=session) + return utils.del_db_object(session, del_user) + + +@utils.supported_filters( + optional_support_keys=UPDATED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates(email=_check_email) +@database.run_in_session() +@utils.wrap_to_dict(RESP_FIELDS) +def update_user(user_id, user=None, session=None, **kwargs): + """Update a user and return the updated user object.""" + update_user = _get_user( + user_id, session=session, + ) + allowed_fields = set() + if user.is_admin: + allowed_fields |= set(ADMIN_UPDATED_FIELDS) + if user.id == update_user.id: + allowed_fields |= set(SELF_UPDATED_FIELDS) + unsupported_fields = set(kwargs) - allowed_fields + if unsupported_fields: + # The user is not allowed to update a user. + raise exception.Forbidden( + 'User %s has no permission to update user %s fields %s.' % ( + user.email, user.email, unsupported_fields + ) + ) + return utils.update_db_object(session, update_user, **kwargs) + + +@utils.supported_filters(optional_support_keys=PERMISSION_SUPPORTED_FIELDS) +@database.run_in_session() +@check_user_admin_or_owner() +@utils.wrap_to_dict(PERMISSION_RESP_FIELDS) +def get_permissions( + user_id, user=None, exception_when_missing=True, + session=None, **kwargs +): + """List permissions of a user.""" + get_user = _get_user( + user_id, session=session, + exception_when_missing=exception_when_missing + ) + return utils.list_db_objects( + session, models.UserPermission, user_id=get_user.id, **kwargs + ) + + +def _get_permission(user_id, permission_id, session=None, **kwargs): + """Get user permission by user id and permission id.""" + user = _get_user(user_id, session=session) + from compass.db.api import permission as permission_api + permission = permission_api.get_permission_internal( + permission_id, session=session + ) + return utils.get_db_object( + session, models.UserPermission, + user_id=user.id, permission_id=permission.id, + **kwargs + ) + + +@utils.supported_filters() +@database.run_in_session() +@check_user_admin_or_owner() +@utils.wrap_to_dict(PERMISSION_RESP_FIELDS) +def get_permission( + user_id, permission_id, exception_when_missing=True, + user=None, session=None, **kwargs +): + """Get a permission of a user.""" + return _get_permission( + user_id, permission_id, + exception_when_missing=exception_when_missing, + session=session, + **kwargs + ) + + +@utils.supported_filters() +@database.run_in_session() +@check_user_admin_or_owner() +@utils.wrap_to_dict(PERMISSION_RESP_FIELDS) +def del_permission(user_id, permission_id, user=None, session=None, **kwargs): + """Delete a permission from a user.""" + user_permission = _get_permission( + user_id, permission_id, + session=session, **kwargs + ) + return utils.del_db_object(session, user_permission) + + +@utils.supported_filters( + PERMISSION_ADDED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@database.run_in_session() +@check_user_admin() +@utils.wrap_to_dict(PERMISSION_RESP_FIELDS) +def add_permission( + user_id, permission_id=None, exception_when_existing=True, + user=None, session=None +): + """Add a permission to a user.""" + get_user = _get_user(user_id, session=session) + from compass.db.api import permission as permission_api + get_permission = permission_api.get_permission_internal( + permission_id, session=session + ) + return utils.add_db_object( + session, models.UserPermission, exception_when_existing, + get_user.id, get_permission.id + ) + + +def _get_permission_filters(permission_ids): + """Helper function to filter permissions.""" + if permission_ids == 'all': + return {} + else: + return {'id': permission_ids} + + +@utils.supported_filters( + optional_support_keys=[ + 'add_permissions', 'remove_permissions', 'set_permissions' + ] +) +@database.run_in_session() +@check_user_admin() +@utils.wrap_to_dict(PERMISSION_RESP_FIELDS) +def update_permissions( + user_id, add_permissions=[], remove_permissions=[], + set_permissions=None, user=None, session=None, **kwargs +): + """update user permissions.""" + update_user = _get_user(user_id, session=session) + if remove_permissions: + _remove_user_permissions( + update_user, session=session, + **_get_permission_filters(remove_permissions) + ) + if add_permissions: + _add_user_permissions( + update_user, session=session, + **_get_permission_filters(add_permissions) + ) + if set_permissions is not None: + _set_user_permissions( + update_user, session=session, + **_get_permission_filters(set_permissions) + ) + return update_user.user_permissions diff --git a/compass-tasks-base/db/api/user_log.py b/compass-tasks-base/db/api/user_log.py new file mode 100644 index 0000000..70de9db --- /dev/null +++ b/compass-tasks-base/db/api/user_log.py @@ -0,0 +1,82 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UserLog database operations.""" +import logging + +from compass.db.api import database +from compass.db.api import user as user_api +from compass.db.api import utils +from compass.db import exception +from compass.db import models + + +SUPPORTED_FIELDS = ['user_email', 'timestamp'] +USER_SUPPORTED_FIELDS = ['timestamp'] +RESP_FIELDS = ['user_id', 'action', 'timestamp'] + + +@database.run_in_session() +def log_user_action(user_id, action, session=None): + """Log user action.""" + utils.add_db_object( + session, models.UserLog, True, user_id=user_id, action=action + ) + + +@utils.supported_filters(optional_support_keys=USER_SUPPORTED_FIELDS) +@database.run_in_session() +@user_api.check_user_admin_or_owner() +@utils.wrap_to_dict(RESP_FIELDS) +def list_user_actions(user_id, user=None, session=None, **filters): + """list user actions of a user.""" + list_user = user_api.get_user(user_id, user=user, session=session) + return utils.list_db_objects( + session, models.UserLog, order_by=['timestamp'], + user_id=list_user['id'], **filters + ) + + +@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) +@user_api.check_user_admin() +@database.run_in_session() +@utils.wrap_to_dict(RESP_FIELDS) +def list_actions(user=None, session=None, **filters): + """list actions of all users.""" + return utils.list_db_objects( + session, models.UserLog, order_by=['timestamp'], **filters + ) + + +@utils.supported_filters() +@database.run_in_session() +@user_api.check_user_admin_or_owner() +@utils.wrap_to_dict(RESP_FIELDS) +def del_user_actions(user_id, user=None, session=None, **filters): + """delete actions of a user.""" + del_user = user_api.get_user(user_id, user=user, session=session) + return utils.del_db_objects( + session, models.UserLog, user_id=del_user['id'], **filters + ) + + +@utils.supported_filters() +@database.run_in_session() +@user_api.check_user_admin() +@utils.wrap_to_dict(RESP_FIELDS) +def del_actions(user=None, session=None, **filters): + """delete actions of all users.""" + return utils.del_db_objects( + session, models.UserLog, **filters + ) diff --git a/compass-tasks-base/db/api/utils.py b/compass-tasks-base/db/api/utils.py new file mode 100644 index 0000000..a44f26e --- /dev/null +++ b/compass-tasks-base/db/api/utils.py @@ -0,0 +1,1286 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utils for database usage.""" + +import functools +import inspect +import logging +import netaddr +import re + +from inspect import isfunction +from sqlalchemy import and_ +from sqlalchemy import or_ + +from compass.db import exception +from compass.db import models +from compass.utils import util + + +def model_query(session, model): + """model query. + + Return sqlalchemy query object. + """ + if not issubclass(model, models.BASE): + raise exception.DatabaseException("model should be sublass of BASE!") + + return session.query(model) + + +def _default_list_condition_func(col_attr, value, condition_func): + """The default condition func for a list of data. + + Given the condition func for single item of data, this function + wrap the condition_func and return another condition func using + or_ to merge the conditions of each single item to deal with a + list of data item. + + Args: + col_attr: the colomn name + value: the column value need to be compared. + condition_func: the sqlalchemy condition object like == + + Examples: + col_attr is name, value is ['a', 'b', 'c'] and + condition_func is ==, the returned condition is + name == 'a' or name == 'b' or name == 'c' + """ + conditions = [] + for sub_value in value: + condition = condition_func(col_attr, sub_value) + if condition is not None: + conditions.append(condition) + if conditions: + return or_(*conditions) + else: + return None + + +def _one_item_list_condition_func(col_attr, value, condition_func): + """The wrapper condition func to deal with one item data list. + + For simplification, it is used to reduce generating too complex + sql conditions. + """ + if value: + return condition_func(col_attr, value[0]) + else: + return None + + +def _model_condition_func( + col_attr, value, + item_condition_func, + list_condition_func=_default_list_condition_func +): + """Return sql condition based on value type.""" + if isinstance(value, list): + if not value: + return None + if len(value) == 1: + return item_condition_func(col_attr, value) + return list_condition_func( + col_attr, value, item_condition_func + ) + else: + return item_condition_func(col_attr, value) + + +def _between_condition(col_attr, value): + """Return sql range condition.""" + if value[0] is not None and value[1] is not None: + return col_attr.between(value[0], value[1]) + if value[0] is not None: + return col_attr >= value[0] + if value[1] is not None: + return col_attr <= value[1] + return None + + +def model_order_by(query, model, order_by): + """append order by into sql query model.""" + if not order_by: + return query + order_by_cols = [] + for key in order_by: + if isinstance(key, tuple): + key, is_desc = key + else: + is_desc = False + if isinstance(key, basestring): + if hasattr(model, key): + col_attr = getattr(model, key) + else: + continue + else: + col_attr = key + if is_desc: + order_by_cols.append(col_attr.desc()) + else: + order_by_cols.append(col_attr) + return query.order_by(*order_by_cols) + + +def _model_condition(col_attr, value): + """Generate condition for one column. + + Example for col_attr is name: + value is 'a': name == 'a' + value is ['a']: name == 'a' + value is ['a', 'b']: name == 'a' or name == 'b' + value is {'eq': 'a'}: name == 'a' + value is {'lt': 'a'}: name < 'a' + value is {'le': 'a'}: name <= 'a' + value is {'gt': 'a'}: name > 'a' + value is {'ge': 'a'}: name >= 'a' + value is {'ne': 'a'}: name != 'a' + value is {'in': ['a', 'b']}: name in ['a', 'b'] + value is {'notin': ['a', 'b']}: name not in ['a', 'b'] + value is {'startswith': 'abc'}: name like 'abc%' + value is {'endswith': 'abc'}: name like '%abc' + value is {'like': 'abc'}: name like '%abc%' + value is {'between': ('a', 'c')}: name >= 'a' and name <= 'c' + value is [{'lt': 'a'}]: name < 'a' + value is [{'lt': 'a'}, {'gt': c'}]: name < 'a' or name > 'c' + value is {'lt': 'c', 'gt': 'a'}: name > 'a' and name < 'c' + + If value is a list, the condition is the or relationship among + conditions of each item. + If value is dict and there are multi keys in the dict, the relationship + is and conditions of each key. + Otherwise the condition is to compare the column with the value. + """ + if isinstance(value, list): + basetype_values = [] + composite_values = [] + for item in value: + if isinstance(item, (list, dict)): + composite_values.append(item) + else: + basetype_values.append(item) + conditions = [] + if basetype_values: + if len(basetype_values) == 1: + condition = (col_attr == basetype_values[0]) + else: + condition = col_attr.in_(basetype_values) + conditions.append(condition) + for composite_value in composite_values: + condition = _model_condition(col_attr, composite_value) + if condition is not None: + conditions.append(condition) + if not conditions: + return None + if len(conditions) == 1: + return conditions[0] + return or_(*conditions) + elif isinstance(value, dict): + conditions = [] + if 'eq' in value: + conditions.append(_model_condition_func( + col_attr, value['eq'], + lambda attr, data: attr == data, + lambda attr, data, item_condition_func: attr.in_(data) + )) + if 'lt' in value: + conditions.append(_model_condition_func( + col_attr, value['lt'], + lambda attr, data: attr < data, + _one_item_list_condition_func + )) + if 'gt' in value: + conditions.append(_model_condition_func( + col_attr, value['gt'], + lambda attr, data: attr > data, + _one_item_list_condition_func + )) + if 'le' in value: + conditions.append(_model_condition_func( + col_attr, value['le'], + lambda attr, data: attr <= data, + _one_item_list_condition_func + )) + if 'ge' in value: + conditions.append(_model_condition_func( + col_attr, value['ge'], + lambda attr, data: attr >= data, + _one_item_list_condition_func + )) + if 'ne' in value: + conditions.append(_model_condition_func( + col_attr, value['ne'], + lambda attr, data: attr != data, + lambda attr, data, item_condition_func: attr.notin_(data) + )) + if 'in' in value: + conditions.append(col_attr.in_(value['in'])) + if 'notin' in value: + conditions.append(col_attr.notin_(value['notin'])) + if 'startswith' in value: + conditions.append(_model_condition_func( + col_attr, value['startswith'], + lambda attr, data: attr.like('%s%%' % data) + )) + if 'endswith' in value: + conditions.append(_model_condition_func( + col_attr, value['endswith'], + lambda attr, data: attr.like('%%%s' % data) + )) + if 'like' in value: + conditions.append(_model_condition_func( + col_attr, value['like'], + lambda attr, data: attr.like('%%%s%%' % data) + )) + if 'between' in value: + conditions.append(_model_condition_func( + col_attr, value['between'], + _between_condition + )) + conditions = [ + condition + for condition in conditions + if condition is not None + ] + if not conditions: + return None + if len(conditions) == 1: + return conditions[0] + return and_(conditions) + else: + condition = (col_attr == value) + return condition + + +def model_filter(query, model, **filters): + """Append conditons to query for each possible column.""" + for key, value in filters.items(): + if isinstance(key, basestring): + if hasattr(model, key): + col_attr = getattr(model, key) + else: + continue + else: + col_attr = key + condition = _model_condition(col_attr, value) + if condition is not None: + query = query.filter(condition) + return query + + +def replace_output(**output_mapping): + """Decorator to recursively relace output by output mapping. + + The replacement detail is described in _replace_output. + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + return _replace_output( + func(*args, **kwargs), **output_mapping + ) + return wrapper + return decorator + + +def _replace_output(data, **output_mapping): + """Helper to replace output data. + + Example: + data = {'a': 'hello'} + output_mapping = {'a': 'b'} + returns: {'b': 'hello'} + + data = {'a': {'b': 'hello'}} + output_mapping = {'a': 'b'} + returns: {'b': {'b': 'hello'}} + + data = {'a': {'b': 'hello'}} + output_mapping = {'a': {'b': 'c'}} + returns: {'a': {'c': 'hello'}} + + data = [{'a': 'hello'}, {'a': 'hi'}] + output_mapping = {'a': 'b'} + returns: [{'b': 'hello'}, {'b': 'hi'}] + """ + if isinstance(data, list): + return [ + _replace_output(item, **output_mapping) + for item in data + ] + if not isinstance(data, dict): + raise exception.InvalidResponse( + '%s type is not dict' % data + ) + info = {} + for key, value in data.items(): + if key in output_mapping: + output_key = output_mapping[key] + if isinstance(output_key, basestring): + info[output_key] = value + else: + info[key] = ( + _replace_output(value, **output_key) + ) + else: + info[key] = value + return info + + +def get_wrapped_func(func): + """Get wrapped function instance. + + Example: + @dec1 + @dec2 + myfunc(*args, **kwargs) + + get_wrapped_func(myfunc) returns function object with + following attributes: + __name__: 'myfunc' + args: args + kwargs: kwargs + otherwise myfunc is function object with following attributes: + __name__: partial object ... + args: ... + kwargs: ... + """ + if func.func_closure: + for closure in func.func_closure: + if isfunction(closure.cell_contents): + return get_wrapped_func(closure.cell_contents) + return func + else: + return func + + +def wrap_to_dict(support_keys=[], **filters): + """Decrator to convert returned object to dict. + + The details is decribed in _wrapper_dict. + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + return _wrapper_dict( + func(*args, **kwargs), support_keys, **filters + ) + return wrapper + return decorator + + +def _wrapper_dict(data, support_keys, **filters): + """Helper for warpping db object into dictionary. + + If data is list, convert it to a list of dict + If data is Base model, convert it to dict + for the data as a dict, filter it with the supported keys. + For each filter_key, filter_value in filters, also filter + data[filter_key] by filter_value recursively if it exists. + + Example: + data is models.Switch, it will be converted to + { + 'id': 1, 'ip': '10.0.0.1', 'ip_int': 123456, + 'credentials': {'version': 2, 'password': 'abc'} + } + Then if support_keys are ['id', 'ip', 'credentials'], + it will be filtered to { + 'id': 1, 'ip': '10.0.0.1', + 'credentials': {'version': 2, 'password': 'abc'} + } + Then if filters is {'credentials': ['version']}, + it will be filtered to { + 'id': 1, 'ip': '10.0.0.1', + 'credentials': {'version': 2} + } + """ + logging.debug( + 'wrap dict %s by support_keys=%s filters=%s', + data, support_keys, filters + ) + if isinstance(data, list): + return [ + _wrapper_dict(item, support_keys, **filters) + for item in data + ] + if isinstance(data, models.HelperMixin): + data = data.to_dict() + if not isinstance(data, dict): + raise exception.InvalidResponse( + 'response %s type is not dict' % data + ) + info = {} + try: + for key in support_keys: + if key in data and data[key] is not None: + if key in filters: + filter_keys = filters[key] + if isinstance(filter_keys, dict): + info[key] = _wrapper_dict( + data[key], filter_keys.keys(), + **filter_keys + ) + else: + info[key] = _wrapper_dict( + data[key], filter_keys + ) + else: + info[key] = data[key] + return info + except Exception as error: + logging.exception(error) + raise error + + +def replace_filters(**kwarg_mapping): + """Decorator to replace kwargs. + + Examples: + kwargs: {'a': 'b'}, kwarg_mapping: {'a': 'c'} + replaced kwargs to decorated func: + {'c': 'b'} + + replace_filters is used to replace caller's input + to make it understandable by models.py. + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + replaced_kwargs = {} + for key, value in kwargs.items(): + if key in kwarg_mapping: + replaced_kwargs[kwarg_mapping[key]] = value + else: + replaced_kwargs[key] = value + return func(*args, **replaced_kwargs) + return wrapper + return decorator + + +def supported_filters( + support_keys=[], + optional_support_keys=[], + ignore_support_keys=[], +): + """Decorator to check kwargs keys. + + keys in kwargs and in ignore_support_keys will be removed. + If any unsupported keys found, a InvalidParameter + exception raises. + + Args: + support_keys: keys that must exist. + optional_support_keys: keys that may exist. + ignore_support_keys: keys should be ignored. + + Assumption: args without default value is supposed to exist. + You can add them in support_keys or not but we will make sure + it appears when we call the decorated function. + We do best match on both args and kwargs to make sure if the + key appears or not. + + Examples: + decorated func: func(a, b, c=3, d=4, **kwargs) + + support_keys=['e'] and call func(e=5): + raises: InvalidParameter: missing declared arg + support_keys=['e'] and call func(1,2,3,4,5,e=6): + raises: InvalidParameter: caller sending more args + support_keys=['e'] and call func(1,2): + raises: InvalidParameter: supported keys ['e'] missing + support_keys=['d', 'e'] and call func(1,2,e=3): + raises: InvalidParameter: supported keys ['d'] missing + support_keys=['d', 'e'] and call func(1,2,d=4, e=3): + passed + support_keys=['d'], optional_support_keys=['e'] + and call func(1,2, d=3): + passed + support_keys=['d'], optional_support_keys=['e'] + and call func(1,2, d=3, e=4, f=5): + raises: InvalidParameter: unsupported keys ['f'] + support_keys=['d'], optional_support_keys=['e'], + ignore_support_keys=['f'] + and call func(1,2, d=3, e=4, f=5): + passed to decorated keys: func(1,2, d=3, e=4) + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **filters): + wrapped_func = get_wrapped_func(func) + argspec = inspect.getargspec(wrapped_func) + wrapped_args = argspec.args + args_defaults = argspec.defaults + # wrapped_must_args are positional args caller must pass in. + if args_defaults: + wrapped_must_args = wrapped_args[:-len(args_defaults)] + else: + wrapped_must_args = wrapped_args[:] + # make sure any positional args without default value in + # decorated function should appear in args or filters. + if len(args) < len(wrapped_must_args): + remain_args = wrapped_must_args[len(args):] + for remain_arg in remain_args: + if remain_arg not in filters: + raise exception.InvalidParameter( + 'function missing declared arg %s ' + 'while caller sends args %s' % ( + remain_arg, args + ) + ) + # make sure args should be no more than positional args + # declared in decorated function. + if len(args) > len(wrapped_args): + raise exception.InvalidParameter( + 'function definition args %s while the caller ' + 'sends args %s' % ( + wrapped_args, args + ) + ) + # exist_args are positional args caller has given. + exist_args = dict(zip(wrapped_args, args)).keys() + must_support_keys = set(support_keys) + all_support_keys = must_support_keys | set(optional_support_keys) + wrapped_supported_keys = set(filters) | set(exist_args) + unsupported_keys = ( + set(filters) - set(wrapped_args) - + all_support_keys - set(ignore_support_keys) + ) + # unsupported_keys are the keys that are not in support_keys, + # optional_support_keys, ignore_support_keys and are not passed in + # by positional args. It means the decorated function may + # not understand these parameters. + if unsupported_keys: + raise exception.InvalidParameter( + 'filter keys %s are not supported for %s' % ( + list(unsupported_keys), wrapped_func + ) + ) + # missing_keys are the keys that must exist but missing in + # both positional args or kwargs. + missing_keys = must_support_keys - wrapped_supported_keys + if missing_keys: + raise exception.InvalidParameter( + 'filter keys %s not found for %s' % ( + list(missing_keys), wrapped_func + ) + ) + # We filter kwargs to eliminate ignore_support_keys in kwargs + # passed to decorated function. + filtered_filters = dict([ + (key, value) + for key, value in filters.items() + if key not in ignore_support_keys + ]) + return func(*args, **filtered_filters) + return wrapper + return decorator + + +def input_filters( + **filters +): + """Decorator to filter kwargs. + + For key in kwargs, if the key exists and filters + and the return of call filters[key] is False, the key + will be removed from kwargs. + + The function definition of filters[key] is + func(value, *args, **kwargs) compared with decorated + function func(*args, **kwargs) + + The function is used to filter kwargs in case some + kwargs should be removed conditionally depends on the + related filters. + + Examples: + filters={'a': func(value, *args, **kwargs)} + @input_filters(**filters) + decorated_func(*args, **kwargs) + func returns False. + Then when call decorated_func(a=1, b=2) + it will be actually called the decorated func with + b=2. a=1 will be removed since it does not pass filtering. + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + filtered_kwargs = {} + for key, value in kwargs.items(): + if key in filters: + if filters[key](value, *args, **kwargs): + filtered_kwargs[key] = value + else: + logging.debug( + 'ignore filtered input key %s' % key + ) + else: + filtered_kwargs[key] = value + return func(*args, **filtered_kwargs) + return wrapper + return decorator + + +def _obj_equal_or_subset(check, obj): + """Used by output filter to check if obj is in check.""" + if check == obj: + return True + if not issubclass(obj.__class__, check.__class__): + return False + if isinstance(obj, dict): + return _dict_equal_or_subset(check, obj) + elif isinstance(obj, list): + return _list_equal_or_subset(check, obj) + else: + return False + + +def _list_equal_or_subset(check_list, obj_list): + """Used by output filter to check if obj_list is in check_list""" + if not isinstance(check_list, list): + return False + return set(check_list).issubset(set(obj_list)) + + +def _dict_equal_or_subset(check_dict, obj_dict): + """Used by output filter to check if obj_dict in check_dict.""" + if not isinstance(check_dict, dict): + return False + for key, value in check_dict.items(): + if ( + key not in obj_dict or + not _obj_equal_or_subset(check_dict[key], obj_dict[key]) + ): + return False + return True + + +def general_filter_callback(general_filter, obj): + """General filter function to filter output. + + Since some fields stored in database is json encoded and + we want to do the deep match for the json encoded field to + do the filtering in some cases, we introduces the output_filters + and general_filter_callback to deal with this kind of cases. + + We do special treatment for key 'resp_eq' to check if + obj is the recursively subset of general_filter['resp_eq'] + + + Example: + obj: 'b' + general_filter: {} + returns: True + + obj: 'b' + general_filter: {'resp_in': ['a', 'b']} + returns: True + + obj: 'b' + general_filter: {'resp_in': ['a']} + returns: False + + obj: 'b' + general_filter: {'resp_eq': 'b'} + returns: True + + obj: 'b' + general_filter: {'resp_eq': 'a'} + returns: False + + obj: 'b' + general_filter: {'resp_range': ('a', 'c')} + returns: True + + obj: 'd' + general_filter: {'resp_range': ('a', 'c')} + returns: False + + If there are multi keys in dict, the output is filtered + by and relationship. + + If the general_filter is a list, the output is filtered + by or relationship. + + Supported general filters: [ + 'resp_eq', 'resp_in', 'resp_lt', + 'resp_le', 'resp_gt', 'resp_ge', + 'resp_match', 'resp_range' + ] + """ + if isinstance(general_filter, list): + if not general_filter: + return True + return any([ + general_filter_callback(item, obj) + for item in general_filter + ]) + elif isinstance(general_filter, dict): + if 'resp_eq' in general_filter: + if not _obj_equal_or_subset( + general_filter['resp_eq'], obj + ): + return False + if 'resp_in' in general_filter: + in_filters = general_filter['resp_in'] + if not any([ + _obj_equal_or_subset(in_filer, obj) + for in_filer in in_filters + ]): + return False + if 'resp_lt' in general_filter: + if obj >= general_filter['resp_lt']: + return False + if 'resp_le' in general_filter: + if obj > general_filter['resp_le']: + return False + if 'resp_gt' in general_filter: + if obj <= general_filter['resp_gt']: + return False + if 'resp_ge' in general_filter: + if obj < general_filter['resp_gt']: + return False + if 'resp_match' in general_filter: + if not re.match(general_filter['resp_match'], obj): + return False + if 'resp_range' in general_filter: + resp_range = general_filter['resp_range'] + if not isinstance(resp_range, list): + resp_range = [resp_range] + in_range = False + for range_start, range_end in resp_range: + if range_start <= obj <= range_end: + in_range = True + if not in_range: + return False + return True + else: + return True + + +def filter_output(filter_callbacks, kwargs, obj, missing_ok=False): + """Filter ouput. + + For each key in filter_callbacks, if it exists in kwargs, + kwargs[key] tells what we need to filter. If the call of + filter_callbacks[key] returns False, it tells the obj should be + filtered out of output. + """ + for callback_key, callback_value in filter_callbacks.items(): + if callback_key not in kwargs: + continue + if callback_key not in obj: + if missing_ok: + continue + else: + raise exception.InvalidResponse( + '%s is not in %s' % (callback_key, obj) + ) + if not callback_value( + kwargs[callback_key], obj[callback_key] + ): + return False + return True + + +def output_filters(missing_ok=False, **filter_callbacks): + """Decorator to filter output list. + + Each filter_callback should have the definition like: + func({'resp_eq': 'a'}, 'a') + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + filtered_obj_list = [] + obj_list = func(*args, **kwargs) + for obj in obj_list: + if filter_output( + filter_callbacks, kwargs, obj, missing_ok + ): + filtered_obj_list.append(obj) + return filtered_obj_list + return wrapper + return decorator + + +def _input_validates(args_validators, kwargs_validators, *args, **kwargs): + """Used by input_validators to validate inputs.""" + for i, value in enumerate(args): + if i < len(args_validators) and args_validators[i]: + args_validators[i](value) + for key, value in kwargs.items(): + if kwargs_validators.get(key): + kwargs_validators[key](value) + + +def input_validates(*args_validators, **kwargs_validators): + """Decorator to validate input. + + Each validator should have definition like: + func('00:01:02:03:04:05') + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + _input_validates( + args_validators, kwargs_validators, + *args, **kwargs + ) + return func(*args, **kwargs) + return wrapper + return decorator + + +def _input_validates_with_args( + args_validators, kwargs_validators, *args, **kwargs +): + """Validate input with validators. + + Each validator takes the arguments of the decorated function + as its arguments. The function definition is like: + func(value, *args, **kwargs) compared with the decorated + function func(*args, **kwargs). + """ + for i, value in enumerate(args): + if i < len(args_validators) and args_validators[i]: + args_validators[i](value, *args, **kwargs) + for key, value in kwargs.items(): + if kwargs_validators.get(key): + kwargs_validators[key](value, *args, **kwargs) + + +def input_validates_with_args( + *args_validators, **kwargs_validators +): + """Decorator to validate input.""" + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + _input_validates_with_args( + args_validators, kwargs_validators, + *args, **kwargs + ) + return func(*args, **kwargs) + return wrapper + return decorator + + +def _output_validates_with_args( + kwargs_validators, obj, *args, **kwargs +): + """Validate output with validators. + + Each validator takes the arguments of the decorated function + as its arguments. The function definition is like: + func(value, *args, **kwargs) compared with the decorated + function func(*args, **kwargs). + """ + if isinstance(obj, list): + for item in obj: + _output_validates_with_args( + kwargs_validators, item, *args, **kwargs + ) + return + if isinstance(obj, models.HelperMixin): + obj = obj.to_dict() + if not isinstance(obj, dict): + raise exception.InvalidResponse( + 'response %s type is not dict' % str(obj) + ) + try: + for key, value in obj.items(): + if key in kwargs_validators: + kwargs_validators[key](value, *args, **kwargs) + except Exception as error: + logging.exception(error) + raise error + + +def output_validates_with_args(**kwargs_validators): + """Decorator to validate output. + + The validator can take the arguments of the decorated + function as its arguments. + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + obj = func(*args, **kwargs) + if isinstance(obj, list): + for obj_item in obj: + _output_validates_with_args( + kwargs_validators, obj_item, + *args, **kwargs + ) + else: + _output_validates_with_args( + kwargs_validators, obj, + *args, **kwargs + ) + return obj + return wrapper + return decorator + + +def _output_validates(kwargs_validators, obj): + """Validate output. + + Each validator has following signature: + func(value) + """ + if isinstance(obj, list): + for item in obj: + _output_validates(kwargs_validators, item) + return + if isinstance(obj, models.HelperMixin): + obj = obj.to_dict() + if not isinstance(obj, dict): + raise exception.InvalidResponse( + 'response %s type is not dict' % str(obj) + ) + try: + for key, value in obj.items(): + if key in kwargs_validators: + kwargs_validators[key](value) + except Exception as error: + logging.exception(error) + raise error + + +def output_validates(**kwargs_validators): + """Decorator to validate output.""" + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + obj = func(*args, **kwargs) + if isinstance(obj, list): + for obj_item in obj: + _output_validates(kwargs_validators, obj_item) + else: + _output_validates(kwargs_validators, obj) + return obj + return wrapper + return decorator + + +def get_db_object(session, table, exception_when_missing=True, **kwargs): + """Get db object. + + If not exception_when_missing and the db object can not be found, + return None instead of raising exception. + """ + if not session: + raise exception.DatabaseException('session param is None') + with session.begin(subtransactions=True): + logging.debug( + 'session %s get db object %s from table %s', + id(session), kwargs, table.__name__) + db_object = model_filter( + model_query(session, table), table, **kwargs + ).first() + logging.debug( + 'session %s got db object %s', id(session), db_object + ) + if db_object: + return db_object + + if not exception_when_missing: + return None + + raise exception.RecordNotExists( + 'Cannot find the record in table %s: %s' % ( + table.__name__, kwargs + ) + ) + + +def add_db_object(session, table, exception_when_existing=True, + *args, **kwargs): + """Create db object. + + If not exception_when_existing and the db object exists, + Instead of raising exception, updating the existing db object. + """ + if not session: + raise exception.DatabaseException('session param is None') + with session.begin(subtransactions=True): + logging.debug( + 'session %s add object %s atributes %s to table %s', + id(session), args, kwargs, table.__name__) + argspec = inspect.getargspec(table.__init__) + arg_names = argspec.args[1:] + arg_defaults = argspec.defaults + if not arg_defaults: + arg_defaults = [] + if not ( + len(arg_names) - len(arg_defaults) <= len(args) <= len(arg_names) + ): + raise exception.InvalidParameter( + 'arg names %s does not match arg values %s' % ( + arg_names, args) + ) + db_keys = dict(zip(arg_names, args)) + if db_keys: + db_object = session.query(table).filter_by(**db_keys).first() + else: + db_object = None + + new_object = False + if db_object: + logging.debug( + 'got db object %s: %s', db_keys, db_object + ) + if exception_when_existing: + raise exception.DuplicatedRecord( + '%s exists in table %s' % (db_keys, table.__name__) + ) + else: + db_object = table(**db_keys) + new_object = True + + for key, value in kwargs.items(): + setattr(db_object, key, value) + + if new_object: + session.add(db_object) + session.flush() + db_object.initialize() + db_object.validate() + logging.debug( + 'session %s db object %s added', id(session), db_object + ) + return db_object + + +def list_db_objects(session, table, order_by=[], **filters): + """List db objects. + + If order by given, the db objects should be sorted by the ordered keys. + """ + if not session: + raise exception.DatabaseException('session param is None') + with session.begin(subtransactions=True): + logging.debug( + 'session %s list db objects by filters %s in table %s', + id(session), filters, table.__name__ + ) + db_objects = model_order_by( + model_filter( + model_query(session, table), + table, + **filters + ), + table, + order_by + ).all() + logging.debug( + 'session %s got listed db objects: %s', + id(session), db_objects + ) + return db_objects + + +def del_db_objects(session, table, **filters): + """delete db objects.""" + if not session: + raise exception.DatabaseException('session param is None') + with session.begin(subtransactions=True): + logging.debug( + 'session %s delete db objects by filters %s in table %s', + id(session), filters, table.__name__ + ) + query = model_filter( + model_query(session, table), table, **filters + ) + db_objects = query.all() + query.delete(synchronize_session=False) + logging.debug( + 'session %s db objects %s deleted', id(session), db_objects + ) + return db_objects + + +def update_db_objects(session, table, updates={}, **filters): + """Update db objects.""" + if not session: + raise exception.DatabaseException('session param is None') + with session.begin(subtransactions=True): + logging.debug( + 'session %s update db objects by filters %s in table %s', + id(session), filters, table.__name__) + db_objects = model_filter( + model_query(session, table), table, **filters + ).all() + for db_object in db_objects: + logging.debug('update db object %s: %s', db_object, updates) + update_db_object(session, db_object, **updates) + logging.debug( + 'session %s db objects %s updated', + id(session), db_objects + ) + return db_objects + + +def update_db_object(session, db_object, **kwargs): + """Update db object.""" + if not session: + raise exception.DatabaseException('session param is None') + with session.begin(subtransactions=True): + logging.debug( + 'session %s update db object %s by value %s', + id(session), db_object, kwargs + ) + for key, value in kwargs.items(): + setattr(db_object, key, value) + session.flush() + db_object.update() + db_object.validate() + logging.debug( + 'session %s db object %s updated', + id(session), db_object + ) + return db_object + + +def del_db_object(session, db_object): + """Delete db object.""" + if not session: + raise exception.DatabaseException('session param is None') + with session.begin(subtransactions=True): + logging.debug( + 'session %s delete db object %s', + id(session), db_object + ) + session.delete(db_object) + logging.debug( + 'session %s db object %s deleted', + id(session), db_object + ) + return db_object + + +def check_ip(ip): + """Check ip is ip address formatted.""" + try: + netaddr.IPAddress(ip) + except Exception as error: + logging.exception(error) + raise exception.InvalidParameter( + 'ip address %s format uncorrect' % ip + ) + + +def check_mac(mac): + """Check mac is mac address formatted.""" + try: + netaddr.EUI(mac) + except Exception as error: + logging.exception(error) + raise exception.InvalidParameter( + 'invalid mac address %s' % mac + ) + + +NAME_PATTERN = re.compile(r'[a-zA-Z0-9][a-zA-Z0-9_-]*') + + +def check_name(name): + """Check name meeting name format requirement.""" + if not NAME_PATTERN.match(name): + raise exception.InvalidParameter( + 'name %s does not match the pattern %s' % ( + name, NAME_PATTERN.pattern + ) + ) + + +def _check_ipmi_credentials_ip(ip): + check_ip(ip) + + +def check_ipmi_credentials(ipmi_credentials): + """Check ipmi credentials format is correct.""" + if not ipmi_credentials: + return + if not isinstance(ipmi_credentials, dict): + raise exception.InvalidParameter( + 'invalid ipmi credentials %s' % ipmi_credentials + + ) + for key in ipmi_credentials: + if key not in ['ip', 'username', 'password']: + raise exception.InvalidParameter( + 'unrecognized field %s in ipmi credentials %s' % ( + key, ipmi_credentials + ) + ) + for key in ['ip', 'username', 'password']: + if key not in ipmi_credentials: + raise exception.InvalidParameter( + 'no field %s in ipmi credentials %s' % ( + key, ipmi_credentials + ) + ) + check_ipmi_credential_field = '_check_ipmi_credentials_%s' % key + this_module = globals() + if check_ipmi_credential_field in this_module: + this_module[check_ipmi_credential_field]( + ipmi_credentials[key] + ) + else: + logging.debug( + 'function %s is not defined', check_ipmi_credential_field + ) + + +def _check_switch_credentials_version(version): + if version not in ['1', '2c', '3']: + raise exception.InvalidParameter( + 'unknown snmp version %s' % version + ) + + +def check_switch_credentials(credentials): + """Check switch credentials format is correct.""" + if not credentials: + return + if not isinstance(credentials, dict): + raise exception.InvalidParameter( + 'credentials %s is not dict' % credentials + ) + for key in credentials: + if key not in ['version', 'community']: + raise exception.InvalidParameter( + 'unrecognized key %s in credentials %s' % (key, credentials) + ) + for key in ['version', 'community']: + if key not in credentials: + raise exception.InvalidParameter( + 'there is no %s field in credentials %s' % (key, credentials) + ) + + key_check_func_name = '_check_switch_credentials_%s' % key + this_module = globals() + if key_check_func_name in this_module: + this_module[key_check_func_name]( + credentials[key] + ) + else: + logging.debug( + 'function %s is not defined', + key_check_func_name + ) diff --git a/compass-tasks-base/db/callback.py b/compass-tasks-base/db/callback.py new file mode 100644 index 0000000..35798bc --- /dev/null +++ b/compass-tasks-base/db/callback.py @@ -0,0 +1,204 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Metadata Callback methods.""" +import logging +import netaddr +import random +import re +import socket + +from compass.db import exception +from compass.utils import setting_wrapper as setting +from compass.utils import util + + +CALLBACK_GLOBALS = globals() +CALLBACK_LOCALS = locals() +CALLBACK_CONFIGS = util.load_configs( + setting.CALLBACK_DIR, + config_name_suffix='.py', + env_globals=CALLBACK_GLOBALS, + env_locals=CALLBACK_LOCALS +) +for callback_config in CALLBACK_CONFIGS: + CALLBACK_LOCALS.update(callback_config) + + +def default_proxy(name, **kwargs): + return setting.COMPASS_SUPPORTED_PROXY + + +def proxy_options(name, **kwargs): + return [setting.COMPASS_SUPPORTED_PROXY] + + +def default_noproxy(name, **kwargs): + return setting.COMPASS_SUPPORTED_DEFAULT_NOPROXY + + +def noproxy_options(name, **kwargs): + return setting.COMPASS_SUPPORTED_DEFAULT_NOPROXY + + +def default_ntp_server(name, **kwargs): + return setting.COMPASS_SUPPORTED_NTP_SERVER + + +def ntp_server_options(name, **kwargs): + return setting.COMPASS_SUPPORTED_NTP_SERVER + + +def default_dns_servers(name, **kwargs): + return setting.COMPASS_SUPPORTED_DNS_SERVERS + + +def dns_servers_options(name, **kwargs): + return setting.COMPASS_SUPPORTED_DNS_SERVERS + + +def default_domain(name, **kwargs): + if setting.COMPASS_SUPPORTED_DOMAINS: + return setting.COMPASS_SUPPORTED_DOMAINS[0] + else: + return None + + +def domain_options(name, **kwargs): + return setting.COMPASS_SUPPORTED_DOMAINS + + +def default_search_path(name, **kwargs): + return setting.COMPASS_SUPPORTED_DOMAINS + + +def search_path_options(name, **kwargs): + return setting.COMPASS_SUPPORTED_DOMAINS + + +def default_gateway(name, **kwargs): + return setting.COMPASS_SUPPORTED_DEFAULT_GATEWAY + + +def default_gateway_options(name, **kwargs): + return [setting.COMPASS_SUPPORTED_DEFAULT_GATEWAY] + + +def default_localrepo(name, **kwargs): + return setting.COMPASS_SUPPORTED_LOCAL_REPO + + +def default_localrepo_options(name, **kwargs): + return [setting.COMPASS_SUPPORTED_LOCAL_REPO] + + +def autofill_callback_default(name, config, **kwargs): + if config is None: + if ( + 'autofill_types' not in kwargs or + not (set(kwargs['autofill_types']) & set(kwargs)) + ): + return None + if 'default_value' not in kwargs: + return None + return kwargs['default_value'] + return config + + +def autofill_callback_random_option(name, config, **kwargs): + if config is None: + if ( + 'autofill_types' not in kwargs or + not (set(kwargs['autofill_types']) & set(kwargs)) + ): + return None + if 'options' not in kwargs or not kwargs['options']: + return None + return random.choice(kwargs['options']) + return config + + +def autofill_no_proxy(name, config, **kwargs): + logging.debug( + 'autofill %s config %s by params %s', + name, config, kwargs + ) + if 'cluster' in kwargs: + if config is None: + config = [] + if 'default_value' in kwargs: + for default_no_proxy in kwargs['default_value']: + if default_no_proxy and default_no_proxy not in config: + config.append(default_no_proxy) + cluster = kwargs['cluster'] + for clusterhost in cluster.clusterhosts: + host = clusterhost.host + hostname = host.name + if hostname not in config: + config.append(hostname) + for host_network in host.host_networks: + if host_network.is_mgmt: + ip = host_network.ip + if ip not in config: + config.append(ip) + if not config: + return config + return [no_proxy for no_proxy in config if no_proxy] + + +def autofill_network_mapping(name, config, **kwargs): + logging.debug( + 'autofill %s config %s by params %s', + name, config, kwargs + ) + if not config: + return config + if isinstance(config, basestring): + config = { + 'interface': config, + 'subnet': None + } + if not isinstance(config, dict): + return config + if 'interface' not in config: + return config + subnet = None + interface = config['interface'] + if 'cluster' in kwargs: + cluster = kwargs['cluster'] + for clusterhost in cluster.clusterhosts: + host = clusterhost.host + for host_network in host.host_networks: + if host_network.interface == interface: + subnet = host_network.subnet.subnet + elif 'clusterhost' in kwargs: + clusterhost = kwargs['clusterhost'] + host = clusterhost.host + for host_network in host.host_networks: + if host_network.interface == interface: + subnet = host_network.subnet.subnet + if not subnet: + raise exception.InvalidParameter( + 'interface %s not found in host(s)' % interface + ) + if 'subnet' not in config or not config['subnet']: + config['subnet'] = subnet + else: + if config['subnet'] != subnet: + raise exception.InvalidParameter( + 'subnet %s in config is not equal to subnet %s in hosts' % ( + config['subnet'], subnet + ) + ) + return config diff --git a/compass-tasks-base/db/config_validation/__init__.py b/compass-tasks-base/db/config_validation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/compass-tasks-base/db/config_validation/default_validator.py b/compass-tasks-base/db/config_validation/default_validator.py new file mode 100644 index 0000000..224447f --- /dev/null +++ b/compass-tasks-base/db/config_validation/default_validator.py @@ -0,0 +1,131 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Default config validation function.""" + +from sqlalchemy import or_ + +from compass.db.models import OSConfigField +from compass.db.models import OSConfigMetadata +from compass.db import validator + +MAPPER = { + "os_id": { + "metaTable": OSConfigMetadata, + "metaFieldTable": OSConfigField + } + # "adapter_id": { + # "metaTable": AdapterConfigMetadata, + # "metaFieldTable": AdapterConfigField + # } +} + + +def validate_config(session, config, id_name, id_value, patch=True): + """Validates config. + + Validates the given config value according to the config + metadata of the asscoiated os_id or adapter_id. Returns + a tuple (status, message). + """ + if id_name not in MAPPER.keys(): + return (False, "Invalid id type %s" % id_name) + + meta_table = MAPPER[id_name]['metaTable'] + metafield_table = MAPPER[id_name]['metaFieldTable'] + with session.begin(subtransactions=True): + name_col = name_col = getattr(meta_table, 'name') + id_col = getattr(meta_table, id_name) + + return _validate_config_helper(session, config, + name_col, id_col, id_value, + meta_table, metafield_table, + patch) + + +def _validate_config_helper(session, config, + name_col, id_col, id_value, + meta_table, metafield_table, patch=True): + + with session.begin(subtransactions=True): + for elem in config: + + obj = session.query(meta_table).filter(name_col == elem)\ + .filter(or_(id_col is None, + id_col == id_value)).first() + + if not obj and "_type" not in config[elem]: + return (False, "Invalid metadata '%s'!" % elem) + + if "_type" in config[elem]: + # Metadata is a variable + metadata_name = config[elem]['_type'] + obj = session.query(meta_table).filter_by(name=metadata_name)\ + .first() + + if not obj: + err_msg = ("Invalid metatdata '%s' or missing '_type'" + "to indicate this is a variable metatdata." + % elem) + return (False, err_msg) + + # TODO(Grace): validate metadata here + del config[elem]['_type'] + + fields = obj.fields + + if not fields: + is_valid, message = _validate_config_helper(session, + config[elem], + name_col, id_col, + id_value, + meta_table, + metafield_table, + patch) + if not is_valid: + return (False, message) + + else: + field_config = config[elem] + for key in field_config: + field = session.query(metafield_table)\ + .filter_by(field=key).first() + if not field: + # The field is not in schema + return (False, "Invalid field '%s'!" % key) + + value = field_config[key] + if field.is_required and value is None: + # The value of this field is required + # and cannot be none + err = "The value of field '%s' cannot be null!" % key + return (False, err) + + if field.validator: + func = getattr(validator, field.validator) + if not func or not func(value): + err_msg = ("The value of the field '%s' is " + "invalid format or None!" % key) + return (False, err_msg) + + # This is a PUT request. We need to check presence of all + # required fields. + if not patch: + for field in fields: + name = field.field + if field.is_required and name not in field_config: + return (False, + "Missing required field '%s'" % name) + + return (True, None) diff --git a/compass-tasks-base/db/config_validation/extension/__init__.py b/compass-tasks-base/db/config_validation/extension/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/compass-tasks-base/db/config_validation/extension/openstack.py b/compass-tasks-base/db/config_validation/extension/openstack.py new file mode 100644 index 0000000..6b3af69 --- /dev/null +++ b/compass-tasks-base/db/config_validation/extension/openstack.py @@ -0,0 +1,18 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def validate_cluster_config(): + # TODO(xiaodong): Add openstack specific validation here. + pass diff --git a/compass-tasks-base/db/exception.py b/compass-tasks-base/db/exception.py new file mode 100644 index 0000000..44556c9 --- /dev/null +++ b/compass-tasks-base/db/exception.py @@ -0,0 +1,116 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Custom exception""" +import traceback + + +class DatabaseException(Exception): + """Base class for all database exceptions.""" + def __init__(self, message): + super(DatabaseException, self).__init__(message) + self.traceback = traceback.format_exc() + self.status_code = 400 + + def to_dict(self): + return {'message': str(self)} + + +class RecordNotExists(DatabaseException): + """Define the exception for referring non-existing object in DB.""" + def __init__(self, message): + super(RecordNotExists, self).__init__(message) + self.status_code = 404 + + +class DuplicatedRecord(DatabaseException): + """Define the exception for trying to insert an existing object in DB.""" + def __init__(self, message): + super(DuplicatedRecord, self).__init__(message) + self.status_code = 409 + + +class Unauthorized(DatabaseException): + """Define the exception for invalid user login.""" + def __init__(self, message): + super(Unauthorized, self).__init__(message) + self.status_code = 401 + + +class UserDisabled(DatabaseException): + """Define the exception that a disabled user tries to do some operations. + + """ + def __init__(self, message): + super(UserDisabled, self).__init__(message) + self.status_code = 403 + + +class Forbidden(DatabaseException): + """Define the exception that a user is trying to make some action + + without the right permission. + + """ + def __init__(self, message): + super(Forbidden, self).__init__(message) + self.status_code = 403 + + +class NotAcceptable(DatabaseException): + """The data is not acceptable.""" + def __init__(self, message): + super(NotAcceptable, self).__init__(message) + self.status_code = 406 + + +class InvalidParameter(DatabaseException): + """Define the exception that the request has invalid or missing parameters. + + """ + def __init__(self, message): + super(InvalidParameter, self).__init__(message) + self.status_code = 400 + + +class InvalidResponse(DatabaseException): + """Define the exception that the response is invalid. + + """ + def __init__(self, message): + super(InvalidResponse, self).__init__(message) + self.status_code = 400 + + +class MultiDatabaseException(DatabaseException): + """Define the exception composites with multi exceptions.""" + def __init__(self, exceptions): + super(MultiDatabaseException, self).__init__('multi exceptions') + self.exceptions = exceptions + self.status_code = 400 + + @property + def traceback(self): + tracebacks = [] + for exception in self.exceptions: + tracebacks.append(exception.trackback) + + def to_dict(self): + dict_info = super(MultiDatabaseException, self).to_dict() + dict_info.update({ + 'exceptions': [ + exception.to_dict() for exception in self.exceptions + ] + }) + return dict_info diff --git a/compass-tasks-base/db/models.py b/compass-tasks-base/db/models.py new file mode 100644 index 0000000..d4b0324 --- /dev/null +++ b/compass-tasks-base/db/models.py @@ -0,0 +1,1924 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Database model""" +import copy +import datetime +import logging +import netaddr +import re +import simplejson as json + +from sqlalchemy import BigInteger +from sqlalchemy import Boolean +from sqlalchemy import Column +from sqlalchemy import ColumnDefault +from sqlalchemy import DateTime +from sqlalchemy import Enum +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy import Float +from sqlalchemy import ForeignKey +from sqlalchemy import Integer +from sqlalchemy.orm import relationship, backref +from sqlalchemy import String +from sqlalchemy import Table +from sqlalchemy import Text +from sqlalchemy.types import TypeDecorator +from sqlalchemy import UniqueConstraint + +from compass.db import exception +from compass.utils import util + + +BASE = declarative_base() + + +class JSONEncoded(TypeDecorator): + """Represents an immutable structure as a json-encoded string.""" + + impl = Text + + def process_bind_param(self, value, dialect): + if value is not None: + value = json.dumps(value) + return value + + def process_result_value(self, value, dialect): + if value is not None: + value = json.loads(value) + return value + + +class TimestampMixin(object): + """Provides table fields for each row created/updated timestamp.""" + created_at = Column(DateTime, default=lambda: datetime.datetime.now()) + updated_at = Column(DateTime, default=lambda: datetime.datetime.now(), + onupdate=lambda: datetime.datetime.now()) + + +class HelperMixin(object): + """Provides general fuctions for all compass table models.""" + + def initialize(self): + self.update() + + def update(self): + pass + + @staticmethod + def type_compatible(value, column_type): + """Check if value type is compatible with the column type.""" + if value is None: + return True + if not hasattr(column_type, 'python_type'): + return True + column_python_type = column_type.python_type + if isinstance(value, column_python_type): + return True + if issubclass(column_python_type, basestring): + return isinstance(value, basestring) + if column_python_type in [int, long]: + return type(value) in [int, long] + if column_python_type in [float]: + return type(value) in [float] + if column_python_type in [bool]: + return type(value) in [bool] + return False + + def validate(self): + """Generate validate function to make sure the record is legal.""" + columns = self.__mapper__.columns + for key, column in columns.items(): + value = getattr(self, key) + if not self.type_compatible(value, column.type): + raise exception.InvalidParameter( + 'column %s value %r type is unexpected: %s' % ( + key, value, column.type + ) + ) + + def to_dict(self): + """General function to convert record to dict. + + Convert all columns not starting with '_' to + {: } + """ + keys = self.__mapper__.columns.keys() + dict_info = {} + for key in keys: + if key.startswith('_'): + continue + value = getattr(self, key) + if value is not None: + if isinstance(value, datetime.datetime): + value = util.format_datetime(value) + dict_info[key] = value + return dict_info + + +class StateMixin(TimestampMixin, HelperMixin): + """Provides general fields and functions for state related table.""" + + state = Column( + Enum( + 'UNINITIALIZED', 'INITIALIZED', 'UPDATE_PREPARING', + 'INSTALLING', 'SUCCESSFUL', 'ERROR' + ), + ColumnDefault('UNINITIALIZED') + ) + percentage = Column(Float, default=0.0) + message = Column(Text, default='') + severity = Column( + Enum('INFO', 'WARNING', 'ERROR'), + ColumnDefault('INFO') + ) + ready = Column(Boolean, default=False) + + def update(self): + # In state table, some field information is redundant. + # The update function to make sure all related fields + # are set to correct state. + if self.ready: + self.state = 'SUCCESSFUL' + if self.state in ['UNINITIALIZED', 'INITIALIZED']: + self.percentage = 0.0 + self.severity = 'INFO' + self.message = '' + if self.state == 'INSTALLING': + if self.severity == 'ERROR': + self.state = 'ERROR' + elif self.percentage >= 1.0: + self.state = 'SUCCESSFUL' + self.percentage = 1.0 + if self.state == 'SUCCESSFUL': + self.percentage = 1.0 + super(StateMixin, self).update() + + +class LogHistoryMixin(TimestampMixin, HelperMixin): + """Provides general fields and functions for LogHistory related tables.""" + position = Column(Integer, default=0) + partial_line = Column(Text, default='') + percentage = Column(Float, default=0.0) + message = Column(Text, default='') + severity = Column( + Enum('ERROR', 'WARNING', 'INFO'), + ColumnDefault('INFO') + ) + line_matcher_name = Column( + String(80), default='start' + ) + + def validate(self): + # TODO(xicheng): some validation can be moved to column. + if not self.filename: + raise exception.InvalidParameter( + 'filename is not set in %s' % self.id + ) + + +class HostNetwork(BASE, TimestampMixin, HelperMixin): + """Host network table.""" + __tablename__ = 'host_network' + + id = Column(Integer, primary_key=True) + host_id = Column( + Integer, + ForeignKey('host.id', onupdate='CASCADE', ondelete='CASCADE') + ) + interface = Column( + String(80), nullable=False) + subnet_id = Column( + Integer, + ForeignKey('subnet.id', onupdate='CASCADE', ondelete='CASCADE') + ) + user_id = Column(Integer, ForeignKey('user.id')) + ip_int = Column(BigInteger, nullable=False) + is_mgmt = Column(Boolean, default=False) + is_promiscuous = Column(Boolean, default=False) + + __table_args__ = ( + UniqueConstraint('host_id', 'interface', name='interface_constraint'), + UniqueConstraint('ip_int', 'user_id', name='ip_constraint') + ) + + def __init__(self, host_id, interface, user_id, **kwargs): + self.host_id = host_id + self.interface = interface + self.user_id = user_id + super(HostNetwork, self).__init__(**kwargs) + + def __str__(self): + return 'HostNetwork[%s=%s]' % (self.interface, self.ip) + + @property + def ip(self): + return str(netaddr.IPAddress(self.ip_int)) + + @ip.setter + def ip(self, value): + self.ip_int = int(netaddr.IPAddress(value)) + + @property + def netmask(self): + return str(netaddr.IPNetwork(self.subnet.subnet).netmask) + + def update(self): + self.host.config_validated = False + + def validate(self): + # TODO(xicheng): some validation can be moved to column. + super(HostNetwork, self).validate() + if not self.subnet: + raise exception.InvalidParameter( + 'subnet is not set in %s interface %s' % ( + self.host_id, self.interface + ) + ) + if not self.ip_int: + raise exception.InvalidParameter( + 'ip is not set in %s interface %s' % ( + self.host_id, self.interface + ) + ) + ip = netaddr.IPAddress(self.ip_int) + subnet = netaddr.IPNetwork(self.subnet.subnet) + if ip not in subnet: + raise exception.InvalidParameter( + 'ip %s is not in subnet %s' % ( + str(ip), str(subnet) + ) + ) + + def to_dict(self): + dict_info = super(HostNetwork, self).to_dict() + dict_info['ip'] = self.ip + dict_info['interface'] = self.interface + dict_info['netmask'] = self.netmask + dict_info['subnet'] = self.subnet.subnet + dict_info['user_id'] = self.user_id + return dict_info + + +class ClusterHostLogHistory(BASE, LogHistoryMixin): + """clusterhost installing log history for each file. + + """ + __tablename__ = 'clusterhost_log_history' + + clusterhost_id = Column( + 'id', Integer, + ForeignKey('clusterhost.id', onupdate='CASCADE', ondelete='CASCADE'), + primary_key=True + ) + filename = Column(String(80), primary_key=True, nullable=False) + cluster_id = Column( + Integer, + ForeignKey('cluster.id') + ) + host_id = Column( + Integer, + ForeignKey('host.id') + ) + + def __init__(self, clusterhost_id, filename, **kwargs): + self.clusterhost_id = clusterhost_id + self.filename = filename + super(ClusterHostLogHistory, self).__init__(**kwargs) + + def __str__(self): + return 'ClusterHostLogHistory[%s:%s]' % ( + self.clusterhost_id, self.filename + ) + + def initialize(self): + self.cluster_id = self.clusterhost.cluster_id + self.host_id = self.clusterhost.host_id + super(ClusterHostLogHistory, self).initialize() + + +class HostLogHistory(BASE, LogHistoryMixin): + """host installing log history for each file. + + """ + __tablename__ = 'host_log_history' + + id = Column( + Integer, + ForeignKey('host.id', onupdate='CASCADE', ondelete='CASCADE'), + primary_key=True) + filename = Column(String(80), primary_key=True, nullable=False) + + def __init__(self, id, filename, **kwargs): + self.id = id + self.filename = filename + super(HostLogHistory, self).__init__(**kwargs) + + def __str__(self): + return 'HostLogHistory[%s:%s]' % (self.id, self.filename) + + +class ClusterHostState(BASE, StateMixin): + """ClusterHost state table.""" + __tablename__ = 'clusterhost_state' + + id = Column( + Integer, + ForeignKey( + 'clusterhost.id', + onupdate='CASCADE', ondelete='CASCADE' + ), + primary_key=True + ) + + def __str__(self): + return 'ClusterHostState[%s state %s percentage %s]' % ( + self.id, self.state, self.percentage + ) + + def update(self): + """Update clusterhost state. + + When clusterhost state is updated, the underlying host state + may be updated accordingly. + """ + super(ClusterHostState, self).update() + host_state = self.clusterhost.host.state + if self.state == 'INITIALIZED': + if host_state.state in ['UNINITIALIZED', 'UPDATE_PREPARING']: + host_state.state = 'INITIALIZED' + host_state.update() + elif self.state == 'INSTALLING': + if host_state.state in [ + 'UNINITIALIZED', 'UPDATE_PREPARING', 'INITIALIZED' + ]: + host_state.state = 'INSTALLING' + host_state.update() + elif self.state == 'SUCCESSFUL': + if host_state.state != 'SUCCESSFUL': + host_state.state = 'SUCCESSFUL' + host_state.update() + + +class ClusterHost(BASE, TimestampMixin, HelperMixin): + """ClusterHost table.""" + __tablename__ = 'clusterhost' + + clusterhost_id = Column('id', Integer, primary_key=True) + cluster_id = Column( + Integer, + ForeignKey('cluster.id', onupdate='CASCADE', ondelete='CASCADE') + ) + host_id = Column( + Integer, + ForeignKey('host.id', onupdate='CASCADE', ondelete='CASCADE') + ) + # the list of role names. + _roles = Column('roles', JSONEncoded, default=[]) + _patched_roles = Column('patched_roles', JSONEncoded, default=[]) + config_step = Column(String(80), default='') + package_config = Column(JSONEncoded, default={}) + config_validated = Column(Boolean, default=False) + deployed_package_config = Column(JSONEncoded, default={}) + + log_histories = relationship( + ClusterHostLogHistory, + passive_deletes=True, passive_updates=True, + cascade='all, delete-orphan', + backref=backref('clusterhost') + ) + + __table_args__ = ( + UniqueConstraint('cluster_id', 'host_id', name='constraint'), + ) + + state = relationship( + ClusterHostState, + uselist=False, + passive_deletes=True, passive_updates=True, + cascade='all, delete-orphan', + backref=backref('clusterhost') + ) + + def __init__(self, cluster_id, host_id, **kwargs): + self.cluster_id = cluster_id + self.host_id = host_id + self.state = ClusterHostState() + super(ClusterHost, self).__init__(**kwargs) + + def __str__(self): + return 'ClusterHost[%s:%s]' % (self.clusterhost_id, self.name) + + def update(self): + if self.host.reinstall_os: + if self.state in ['SUCCESSFUL', 'ERROR']: + if self.config_validated: + self.state.state = 'INITIALIZED' + else: + self.state.state = 'UNINITIALIZED' + self.cluster.update() + self.host.update() + self.state.update() + super(ClusterHost, self).update() + + @property + def name(self): + return '%s.%s' % (self.host.name, self.cluster.name) + + @property + def patched_package_config(self): + return self.package_config + + @patched_package_config.setter + def patched_package_config(self, value): + package_config = copy.deepcopy(self.package_config) + self.package_config = util.merge_dict(package_config, value) + logging.debug( + 'patch clusterhost %s package_config: %s', + self.clusterhost_id, value + ) + self.config_validated = False + + @property + def put_package_config(self): + return self.package_config + + @put_package_config.setter + def put_package_config(self, value): + package_config = copy.deepcopy(self.package_config) + package_config.update(value) + self.package_config = package_config + logging.debug( + 'put clusterhost %s package_config: %s', + self.clusterhost_id, value + ) + self.config_validated = False + + @property + def patched_os_config(self): + return self.host.os_config + + @patched_os_config.setter + def patched_os_config(self, value): + host = self.host + host.patched_os_config = value + + @property + def put_os_config(self): + return self.host.os_config + + @put_os_config.setter + def put_os_config(self, value): + host = self.host + host.put_os_config = value + + @property + def deployed_os_config(self): + return self.host.deployed_os_config + + @deployed_os_config.setter + def deployed_os_config(self, value): + host = self.host + host.deployed_os_config = value + + @hybrid_property + def os_name(self): + return self.host.os_name + + @os_name.expression + def os_name(cls): + return cls.host.os_name + + @hybrid_property + def clustername(self): + return self.cluster.name + + @clustername.expression + def clustername(cls): + return cls.cluster.name + + @hybrid_property + def hostname(self): + return self.host.hostname + + @hostname.expression + def hostname(cls): + return Host.hostname + + @property + def distributed_system_installed(self): + return self.state.state == 'SUCCESSFUL' + + @property + def resintall_os(self): + return self.host.reinstall_os + + @property + def reinstall_distributed_system(self): + return self.cluster.reinstall_distributed_system + + @property + def os_installed(self): + return self.host.os_installed + + @property + def roles(self): + # only the role exists in flavor roles will be returned. + # the role will be sorted as the order defined in flavor + # roles. + # duplicate role names will be removed. + # The returned value is a list of dict like + # [{'name': 'allinone', 'optional': False}] + role_names = list(self._roles) + if not role_names: + return [] + cluster_roles = self.cluster.flavor['roles'] + if not cluster_roles: + return [] + roles = [] + for cluster_role in cluster_roles: + if cluster_role['name'] in role_names: + roles.append(cluster_role) + return roles + + @roles.setter + def roles(self, value): + """value should be a list of role name.""" + self._roles = list(value) + self.config_validated = False + + @property + def patched_roles(self): + patched_role_names = list(self._patched_roles) + if not patched_role_names: + return [] + cluster_roles = self.cluster.flavor['roles'] + if not cluster_roles: + return [] + roles = [] + for cluster_role in cluster_roles: + if cluster_role['name'] in patched_role_names: + roles.append(cluster_role) + return roles + + @patched_roles.setter + def patched_roles(self, value): + """value should be a list of role name.""" + # if value is an empty list, we empty the field + if value: + roles = list(self._roles) + roles.extend(value) + self._roles = roles + patched_roles = list(self._patched_roles) + patched_roles.extend(value) + self._patched_roles = patched_roles + self.config_validated = False + else: + self._patched_roles = list(value) + self.config_validated = False + + @hybrid_property + def owner(self): + return self.cluster.owner + + @owner.expression + def owner(cls): + return cls.cluster.owner + + def state_dict(self): + """Get clusterhost state dict. + + The clusterhost state_dict is different from + clusterhost.state.to_dict. The main difference is state_dict + show the progress of both installing os on host and installing + distributed system on clusterhost. While clusterhost.state.to_dict + only shows the progress of installing distributed system on + clusterhost. + """ + cluster = self.cluster + host = self.host + host_state = host.state_dict() + if not cluster.flavor_name: + return host_state + clusterhost_state = self.state.to_dict() + if clusterhost_state['state'] in ['ERROR', 'SUCCESSFUL']: + return clusterhost_state + if ( + clusterhost_state['state'] in 'INSTALLING' and + clusterhost_state['percentage'] > 0 + ): + clusterhost_state['percentage'] = min( + 1.0, ( + 0.5 + clusterhost_state['percentage'] / 2 + ) + ) + return clusterhost_state + + host_state['percentage'] = host_state['percentage'] / 2 + if host_state['state'] == 'SUCCESSFUL': + host_state['state'] = 'INSTALLING' + return host_state + + def to_dict(self): + dict_info = self.host.to_dict() + dict_info.update(super(ClusterHost, self).to_dict()) + state_dict = self.state_dict() + dict_info.update({ + 'distributed_system_installed': self.distributed_system_installed, + 'reinstall_distributed_system': self.reinstall_distributed_system, + 'owner': self.owner, + 'clustername': self.clustername, + 'name': self.name, + 'state': state_dict['state'] + }) + dict_info['roles'] = self.roles + dict_info['patched_roles'] = self.patched_roles + return dict_info + + +class HostState(BASE, StateMixin): + """Host state table.""" + __tablename__ = 'host_state' + + id = Column( + Integer, + ForeignKey('host.id', onupdate='CASCADE', ondelete='CASCADE'), + primary_key=True + ) + + def __str__(self): + return 'HostState[%s state %s percentage %s]' % ( + self.id, self.state, self.percentage + ) + + def update(self): + """Update host state. + + When host state is updated, all clusterhosts on the + host will update their state if necessary. + """ + super(HostState, self).update() + host = self.host + if self.state == 'INSTALLING': + host.reinstall_os = False + for clusterhost in self.host.clusterhosts: + if clusterhost.state in [ + 'SUCCESSFUL', 'ERROR' + ]: + clusterhost.state = 'INSTALLING' + clusterhost.state.update() + elif self.state == 'UNINITIALIZED': + for clusterhost in self.host.clusterhosts: + if clusterhost.state in [ + 'INITIALIZED', 'INSTALLING', 'SUCCESSFUL', 'ERROR' + ]: + clusterhost.state = 'UNINITIALIZED' + clusterhost.state.update() + elif self.state == 'UPDATE_PREPARING': + for clusterhost in self.host.clusterhosts: + if clusterhost.state in [ + 'INITIALIZED', 'INSTALLING', 'SUCCESSFUL', 'ERROR' + ]: + clusterhost.state = 'UPDATE_PREPARING' + clusterhost.state.update() + elif self.state == 'INITIALIZED': + for clusterhost in self.host.clusterhosts: + if clusterhost.state in [ + 'INSTALLING', 'SUCCESSFUL', 'ERROR' + ]: + clusterhost.state = 'INITIALIZED' + clusterhost.state.update() + + +class Host(BASE, TimestampMixin, HelperMixin): + """Host table.""" + __tablename__ = 'host' + + name = Column(String(80), nullable=True) + config_step = Column(String(80), default='') + os_config = Column(JSONEncoded, default={}) + config_validated = Column(Boolean, default=False) + deployed_os_config = Column(JSONEncoded, default={}) + os_name = Column(String(80)) + creator_id = Column(Integer, ForeignKey('user.id')) + owner = Column(String(80)) + os_installer = Column(JSONEncoded, default={}) + + __table_args__ = ( + UniqueConstraint('name', 'owner', name='constraint'), + ) + + id = Column( + Integer, + ForeignKey('machine.id', onupdate='CASCADE', ondelete='CASCADE'), + primary_key=True + ) + reinstall_os = Column(Boolean, default=True) + + host_networks = relationship( + HostNetwork, + passive_deletes=True, passive_updates=True, + cascade='all, delete-orphan', + backref=backref('host') + ) + clusterhosts = relationship( + ClusterHost, + passive_deletes=True, passive_updates=True, + cascade='all, delete-orphan', + backref=backref('host') + ) + state = relationship( + HostState, + uselist=False, + passive_deletes=True, passive_updates=True, + cascade='all, delete-orphan', + backref=backref('host') + ) + log_histories = relationship( + HostLogHistory, + passive_deletes=True, passive_updates=True, + cascade='all, delete-orphan', + backref=backref('host') + ) + + def __str__(self): + return 'Host[%s:%s]' % (self.id, self.name) + + @hybrid_property + def mac(self): + machine = self.machine + if machine: + return machine.mac + else: + return None + + @property + def os_id(self): + return self.os_name + + @os_id.setter + def os_id(self, value): + self.os_name = value + + @hybrid_property + def hostname(self): + return self.name + + @hostname.expression + def hostname(cls): + return cls.name + + @property + def patched_os_config(self): + return self.os_config + + @patched_os_config.setter + def patched_os_config(self, value): + os_config = copy.deepcopy(self.os_config) + self.os_config = util.merge_dict(os_config, value) + logging.debug('patch host os config in %s: %s', self.id, value) + self.config_validated = False + + @property + def put_os_config(self): + return self.os_config + + @put_os_config.setter + def put_os_config(self, value): + os_config = copy.deepcopy(self.os_config) + os_config.update(value) + self.os_config = os_config + logging.debug('put host os config in %s: %s', self.id, value) + self.config_validated = False + + def __init__(self, id, **kwargs): + self.id = id + self.state = HostState() + super(Host, self).__init__(**kwargs) + + def update(self): + creator = self.creator + if creator: + self.owner = creator.email + if self.reinstall_os: + if self.state in ['SUCCESSFUL', 'ERROR']: + if self.config_validated: + self.state.state = 'INITIALIZED' + else: + self.state.state = 'UNINITIALIZED' + self.state.update() + self.state.update() + super(Host, self).update() + + def validate(self): + # TODO(xicheng): some validation can be moved to the column in future. + super(Host, self).validate() + creator = self.creator + if not creator: + raise exception.InvalidParameter( + 'creator is not set in host %s' % self.id + ) + os_name = self.os_name + if not os_name: + raise exception.InvalidParameter( + 'os is not set in host %s' % self.id + ) + os_installer = self.os_installer + if not os_installer: + raise exception.Invalidparameter( + 'os_installer is not set in host %s' % self.id + ) + + @property + def os_installed(self): + return self.state.state == 'SUCCESSFUL' + + @property + def clusters(self): + return [clusterhost.cluster for clusterhost in self.clusterhosts] + + def state_dict(self): + return self.state.to_dict() + + def to_dict(self): + """Host dict contains its underlying machine dict.""" + dict_info = self.machine.to_dict() + dict_info.update(super(Host, self).to_dict()) + state_dict = self.state_dict() + ip = None + for host_network in self.host_networks: + if host_network.is_mgmt: + ip = host_network.ip + dict_info.update({ + 'machine_id': self.machine.id, + 'os_installed': self.os_installed, + 'hostname': self.hostname, + 'ip': ip, + 'networks': [ + host_network.to_dict() + for host_network in self.host_networks + ], + 'os_id': self.os_id, + 'clusters': [cluster.to_dict() for cluster in self.clusters], + 'state': state_dict['state'] + }) + return dict_info + + +class ClusterState(BASE, StateMixin): + """Cluster state table.""" + __tablename__ = 'cluster_state' + + id = Column( + Integer, + ForeignKey('cluster.id', onupdate='CASCADE', ondelete='CASCADE'), + primary_key=True + ) + total_hosts = Column( + Integer, + default=0 + ) + installing_hosts = Column( + Integer, + default=0 + ) + completed_hosts = Column( + Integer, + default=0 + ) + failed_hosts = Column( + Integer, + default=0 + ) + + def __init__(self, **kwargs): + super(ClusterState, self).__init__(**kwargs) + + def __str__(self): + return 'ClusterState[%s state %s percentage %s]' % ( + self.id, self.state, self.percentage + ) + + def to_dict(self): + dict_info = super(ClusterState, self).to_dict() + dict_info['status'] = { + 'total_hosts': self.total_hosts, + 'installing_hosts': self.installing_hosts, + 'completed_hosts': self.completed_hosts, + 'failed_hosts': self.failed_hosts + } + return dict_info + + def update(self): + # all fields of cluster state should be calculated by + # its each underlying clusterhost state. + cluster = self.cluster + clusterhosts = cluster.clusterhosts + self.total_hosts = len(clusterhosts) + self.installing_hosts = 0 + self.failed_hosts = 0 + self.completed_hosts = 0 + if not cluster.flavor_name: + for clusterhost in clusterhosts: + host = clusterhost.host + host_state = host.state.state + if host_state == 'INSTALLING': + self.installing_hosts += 1 + elif host_state == 'ERROR': + self.failed_hosts += 1 + elif host_state == 'SUCCESSFUL': + self.completed_hosts += 1 + else: + for clusterhost in clusterhosts: + clusterhost_state = clusterhost.state.state + if clusterhost_state == 'INSTALLING': + self.installing_hosts += 1 + elif clusterhost_state == 'ERROR': + self.failed_hosts += 1 + elif clusterhost_state == 'SUCCESSFUL': + self.completed_hosts += 1 + if self.total_hosts: + if self.completed_hosts == self.total_hosts: + self.percentage = 1.0 + else: + self.percentage = ( + float(self.completed_hosts) + / + float(self.total_hosts) + ) + if self.state == 'SUCCESSFUL': + self.state = 'INSTALLING' + self.ready = False + self.message = ( + 'total %s, installing %s, completed: %s, error %s' + ) % ( + self.total_hosts, self.installing_hosts, + self.completed_hosts, self.failed_hosts + ) + if self.failed_hosts: + self.severity = 'ERROR' + + super(ClusterState, self).update() + if self.state == 'INSTALLING': + cluster.reinstall_distributed_system = False + + +class Cluster(BASE, TimestampMixin, HelperMixin): + """Cluster table.""" + __tablename__ = 'cluster' + + id = Column(Integer, primary_key=True) + name = Column(String(80), nullable=False) + reinstall_distributed_system = Column(Boolean, default=True) + config_step = Column(String(80), default='') + os_name = Column(String(80)) + flavor_name = Column(String(80), nullable=True) + # flavor dict got from flavor id. + flavor = Column(JSONEncoded, default={}) + os_config = Column(JSONEncoded, default={}) + package_config = Column(JSONEncoded, default={}) + deployed_os_config = Column(JSONEncoded, default={}) + deployed_package_config = Column(JSONEncoded, default={}) + config_validated = Column(Boolean, default=False) + adapter_name = Column(String(80)) + creator_id = Column(Integer, ForeignKey('user.id')) + owner = Column(String(80)) + clusterhosts = relationship( + ClusterHost, + passive_deletes=True, passive_updates=True, + cascade='all, delete-orphan', + backref=backref('cluster') + ) + state = relationship( + ClusterState, + uselist=False, + passive_deletes=True, passive_updates=True, + cascade='all, delete-orphan', + backref=backref('cluster') + ) + __table_args__ = ( + UniqueConstraint('name', 'creator_id', name='constraint'), + ) + + def __init__(self, name, creator_id, **kwargs): + self.name = name + self.creator_id = creator_id + self.state = ClusterState() + super(Cluster, self).__init__(**kwargs) + + def __str__(self): + return 'Cluster[%s:%s]' % (self.id, self.name) + + def update(self): + creator = self.creator + if creator: + self.owner = creator.email + if self.reinstall_distributed_system: + if self.state in ['SUCCESSFUL', 'ERROR']: + if self.config_validated: + self.state.state = 'INITIALIZED' + else: + self.state.state = 'UNINITIALIZED' + self.state.update() + self.state.update() + super(Cluster, self).update() + + def validate(self): + # TODO(xicheng): some validation can be moved to column. + super(Cluster, self).validate() + creator = self.creator + if not creator: + raise exception.InvalidParameter( + 'creator is not set in cluster %s' % self.id + ) + os_name = self.os_name + if not os_name: + raise exception.InvalidParameter( + 'os is not set in cluster %s' % self.id + ) + adapter_name = self.adapter_name + if not adapter_name: + raise exception.InvalidParameter( + 'adapter is not set in cluster %s' % self.id + ) + flavor_name = self.flavor_name + if flavor_name: + if 'name' not in self.flavor: + raise exception.InvalidParameter( + 'key name does not exist in flavor %s' % ( + self.flavor + ) + ) + if flavor_name != self.flavor['name']: + raise exception.InvalidParameter( + 'flavor name %s is not match ' + 'the name key in flavor %s' % ( + flavor_name, self.flavor + ) + ) + else: + if self.flavor: + raise exception.InvalidParameter( + 'flavor %s is not empty' % self.flavor + ) + + @property + def os_id(self): + return self.os_name + + @os_id.setter + def os_id(self, value): + self.os_name = value + + @property + def adapter_id(self): + return self.adapter_name + + @adapter_id.setter + def adapter_id(self, value): + self.adapter_name = value + + @property + def flavor_id(self): + if self.flavor_name: + return '%s:%s' % (self.adapter_name, self.flavor_name) + else: + return None + + @flavor_id.setter + def flavor_id(self, value): + if value: + _, flavor_name = value.split(':', 1) + self.flavor_name = flavor_name + else: + self.flavor_name = value + + @property + def patched_os_config(self): + return self.os_config + + @patched_os_config.setter + def patched_os_config(self, value): + os_config = copy.deepcopy(self.os_config) + self.os_config = util.merge_dict(os_config, value) + logging.debug('patch cluster %s os config: %s', self.id, value) + self.config_validated = False + + @property + def put_os_config(self): + return self.os_config + + @put_os_config.setter + def put_os_config(self, value): + os_config = copy.deepcopy(self.os_config) + os_config.update(value) + self.os_config = os_config + logging.debug('put cluster %s os config: %s', self.id, value) + self.config_validated = False + + @property + def patched_package_config(self): + return self.package_config + + @patched_package_config.setter + def patched_package_config(self, value): + package_config = copy.deepcopy(self.package_config) + self.package_config = util.merge_dict(package_config, value) + logging.debug('patch cluster %s package config: %s', self.id, value) + self.config_validated = False + + @property + def put_package_config(self): + return self.package_config + + @put_package_config.setter + def put_package_config(self, value): + package_config = dict(self.package_config) + package_config.update(value) + self.package_config = package_config + logging.debug('put cluster %s package config: %s', self.id, value) + self.config_validated = False + + @property + def distributed_system_installed(self): + return self.state.state == 'SUCCESSFUL' + + def state_dict(self): + return self.state.to_dict() + + def to_dict(self): + dict_info = super(Cluster, self).to_dict() + dict_info['distributed_system_installed'] = ( + self.distributed_system_installed + ) + dict_info['os_id'] = self.os_id + dict_info['adapter_id'] = self.adapter_id + dict_info['flavor_id'] = self.flavor_id + return dict_info + + +# User, Permission relation table +class UserPermission(BASE, HelperMixin, TimestampMixin): + """User permission table.""" + __tablename__ = 'user_permission' + id = Column(Integer, primary_key=True) + user_id = Column( + Integer, + ForeignKey('user.id', onupdate='CASCADE', ondelete='CASCADE') + ) + permission_id = Column( + Integer, + ForeignKey('permission.id', onupdate='CASCADE', ondelete='CASCADE') + ) + __table_args__ = ( + UniqueConstraint('user_id', 'permission_id', name='constraint'), + ) + + def __init__(self, user_id, permission_id, **kwargs): + self.user_id = user_id + self.permission_id = permission_id + + def __str__(self): + return 'UserPermission[%s:%s]' % (self.id, self.name) + + @hybrid_property + def name(self): + return self.permission.name + + def to_dict(self): + dict_info = self.permission.to_dict() + dict_info.update(super(UserPermission, self).to_dict()) + return dict_info + + +class Permission(BASE, HelperMixin, TimestampMixin): + """Permission table.""" + __tablename__ = 'permission' + + id = Column(Integer, primary_key=True) + name = Column(String(80), unique=True, nullable=False) + alias = Column(String(100)) + description = Column(Text) + user_permissions = relationship( + UserPermission, + passive_deletes=True, passive_updates=True, + cascade='all, delete-orphan', + backref=backref('permission') + ) + + def __init__(self, name, **kwargs): + self.name = name + super(Permission, self).__init__(**kwargs) + + def __str__(self): + return 'Permission[%s:%s]' % (self.id, self.name) + + +class UserToken(BASE, HelperMixin): + """user token table.""" + __tablename__ = 'user_token' + + id = Column(Integer, primary_key=True) + user_id = Column( + Integer, + ForeignKey('user.id', onupdate='CASCADE', ondelete='CASCADE') + ) + token = Column(String(256), unique=True, nullable=False) + expire_timestamp = Column(DateTime, nullable=True) + + def __init__(self, token, **kwargs): + self.token = token + super(UserToken, self).__init__(**kwargs) + + def validate(self): + # TODO(xicheng): some validation can be moved to column. + super(UserToken, self).validate() + if not self.user: + raise exception.InvalidParameter( + 'user is not set in token: %s' % self.token + ) + + +class UserLog(BASE, HelperMixin): + """User log table.""" + __tablename__ = 'user_log' + + id = Column(Integer, primary_key=True) + user_id = Column( + Integer, + ForeignKey('user.id', onupdate='CASCADE', ondelete='CASCADE') + ) + action = Column(Text) + timestamp = Column(DateTime, default=lambda: datetime.datetime.now()) + + @hybrid_property + def user_email(self): + return self.user.email + + def validate(self): + # TODO(xicheng): some validation can be moved to column. + super(UserLog, self).validate() + if not self.user: + raise exception.InvalidParameter( + 'user is not set in user log: %s' % self.id + ) + + +class User(BASE, HelperMixin, TimestampMixin): + """User table.""" + __tablename__ = 'user' + + id = Column(Integer, primary_key=True) + email = Column(String(80), unique=True, nullable=False) + crypted_password = Column('password', String(225)) + firstname = Column(String(80)) + lastname = Column(String(80)) + is_admin = Column(Boolean, default=False) + active = Column(Boolean, default=True) + user_permissions = relationship( + UserPermission, + passive_deletes=True, passive_updates=True, + cascade='all, delete-orphan', + backref=backref('user') + ) + user_logs = relationship( + UserLog, + passive_deletes=True, passive_updates=True, + cascade='all, delete-orphan', + backref=backref('user') + ) + user_tokens = relationship( + UserToken, + passive_deletes=True, passive_updates=True, + cascade='all, delete-orphan', + backref=backref('user') + ) + clusters = relationship( + Cluster, + backref=backref('creator') + ) + hosts = relationship( + Host, + backref=backref('creator') + ) + + def __init__(self, email, **kwargs): + self.email = email + super(User, self).__init__(**kwargs) + + def __str__(self): + return 'User[%s]' % self.email + + def validate(self): + # TODO(xicheng): some validation can be moved to column. + super(User, self).validate() + if not self.crypted_password: + raise exception.InvalidParameter( + 'password is not set in user : %s' % self.email + ) + + @property + def password(self): + return '***********' + + @password.setter + def password(self, password): + # password stored in database is crypted. + self.crypted_password = util.encrypt(password) + + @hybrid_property + def permissions(self): + permissions = [] + for user_permission in self.user_permissions: + permissions.append(user_permission.permission) + + return permissions + + def to_dict(self): + dict_info = super(User, self).to_dict() + dict_info['permissions'] = [ + permission.to_dict() + for permission in self.permissions + ] + return dict_info + + +class SwitchMachine(BASE, HelperMixin, TimestampMixin): + """Switch Machine table.""" + __tablename__ = 'switch_machine' + switch_machine_id = Column( + 'id', Integer, primary_key=True + ) + switch_id = Column( + Integer, + ForeignKey('switch.id', onupdate='CASCADE', ondelete='CASCADE') + ) + machine_id = Column( + Integer, + ForeignKey('machine.id', onupdate='CASCADE', ondelete='CASCADE') + ) + owner_id = Column(Integer, ForeignKey('user.id')) + port = Column(String(80), nullable=True) + vlans = Column(JSONEncoded, default=[]) + __table_args__ = ( + UniqueConstraint('switch_id', 'machine_id', name='constraint'), + ) + + def __init__(self, switch_id, machine_id, **kwargs): + self.switch_id = switch_id + self.machine_id = machine_id + super(SwitchMachine, self).__init__(**kwargs) + + def __str__(self): + return 'SwitchMachine[%s port %s]' % ( + self.switch_machine_id, self.port + ) + + def validate(self): + # TODO(xicheng): some validation can be moved to column. + super(SwitchMachine, self).validate() + if not self.switch: + raise exception.InvalidParameter( + 'switch is not set in %s' % self.id + ) + if not self.machine: + raise exception.Invalidparameter( + 'machine is not set in %s' % self.id + ) + if not self.port: + raise exception.InvalidParameter( + 'port is not set in %s' % self.id + ) + + @hybrid_property + def mac(self): + return self.machine.mac + + @hybrid_property + def tag(self): + return self.machine.tag + + @property + def switch_ip(self): + return self.switch.ip + + @hybrid_property + def switch_ip_int(self): + return self.switch.ip_int + + @switch_ip_int.expression + def switch_ip_int(cls): + return Switch.ip_int + + @hybrid_property + def switch_vendor(self): + return self.switch.vendor + + @switch_vendor.expression + def switch_vendor(cls): + return Switch.vendor + + @property + def patched_vlans(self): + return self.vlans + + @patched_vlans.setter + def patched_vlans(self, value): + if not value: + return + vlans = list(self.vlans) + for item in value: + if item not in vlans: + vlans.append(item) + self.vlans = vlans + + @property + def filtered(self): + """Check if switch machine should be filtered. + + port should be composed with + For each filter in switch machine filters, + if filter_type is allow and port match the pattern, the switch + machine is allowed to be got by api. If filter_type is deny and + port match the pattern, the switch machine is not allowed to be got + by api. + If not filter is matched, if the last filter is allow, deny all + unmatched switch machines, if the last filter is deny, allow all + unmatched switch machines. + If no filter defined, allow all switch machines. + if ports defined in filter and 'all' in ports, the switch machine is + matched. if ports defined in filter and 'all' not in ports, + the switch machine with the port name in ports will be matched. + If the port pattern matches + < and port number is in the + range of [port_start, port_end], the switch machine is matched. + """ + filters = self.switch.machine_filters + port = self.port + unmatched_allowed = True + ports_pattern = re.compile(r'(\D*)(\d+)-(\d+)(\D*)') + port_pattern = re.compile(r'(\D*)(\d+)(\D*)') + port_match = port_pattern.match(port) + if port_match: + port_prefix = port_match.group(1) + port_number = int(port_match.group(2)) + port_suffix = port_match.group(3) + else: + port_prefix = '' + port_number = 0 + port_suffix = '' + for port_filter in filters: + filter_type = port_filter.get('filter_type', 'allow') + denied = filter_type != 'allow' + unmatched_allowed = denied + if 'ports' in port_filter: + if 'all' in port_filter['ports']: + return denied + if port in port_filter['ports']: + return denied + if port_match: + for port_or_ports in port_filter['ports']: + ports_match = ports_pattern.match(port_or_ports) + if ports_match: + filter_port_prefix = ports_match.group(1) + filter_port_start = int(ports_match.group(2)) + filter_port_end = int(ports_match.group(3)) + filter_port_suffix = ports_match.group(4) + if ( + filter_port_prefix == port_prefix and + filter_port_suffix == port_suffix and + filter_port_start <= port_number and + port_number <= filter_port_end + ): + return denied + else: + filter_port_prefix = port_filter.get('port_prefix', '') + filter_port_suffix = port_filter.get('port_suffix', '') + if ( + port_match and + port_prefix == filter_port_prefix and + port_suffix == filter_port_suffix + ): + if ( + 'port_start' not in port_filter or + port_number >= port_filter['port_start'] + ) and ( + 'port_end' not in port_filter or + port_number <= port_filter['port_end'] + ): + return denied + return not unmatched_allowed + + def to_dict(self): + dict_info = self.machine.to_dict() + dict_info.update(super(SwitchMachine, self).to_dict()) + dict_info['switch_ip'] = self.switch.ip + return dict_info + + +class Machine(BASE, HelperMixin, TimestampMixin): + """Machine table.""" + __tablename__ = 'machine' + id = Column(Integer, primary_key=True) + mac = Column(String(24), unique=True, nullable=False) + ipmi_credentials = Column(JSONEncoded, default={}) + tag = Column(JSONEncoded, default={}) + location = Column(JSONEncoded, default={}) + owner_id = Column(Integer, ForeignKey('user.id')) + machine_attributes = Column(JSONEncoded, default={}) + + switch_machines = relationship( + SwitchMachine, + passive_deletes=True, passive_updates=True, + cascade='all, delete-orphan', + backref=backref('machine') + ) + host = relationship( + Host, + uselist=False, + passive_deletes=True, passive_updates=True, + cascade='all, delete-orphan', + backref=backref('machine') + ) + + def __init__(self, mac, **kwargs): + self.mac = mac + super(Machine, self).__init__(**kwargs) + + def __str__(self): + return 'Machine[%s:%s]' % (self.id, self.mac) + + def validate(self): + # TODO(xicheng): some validation can be moved to column. + super(Machine, self).validate() + try: + netaddr.EUI(self.mac) + except Exception: + raise exception.InvalidParameter( + 'mac address %s format uncorrect' % self.mac + ) + + @property + def patched_ipmi_credentials(self): + return self.ipmi_credentials + + @patched_ipmi_credentials.setter + def patched_ipmi_credentials(self, value): + if not value: + return + ipmi_credentials = copy.deepcopy(self.ipmi_credentials) + self.ipmi_credentials = util.merge_dict(ipmi_credentials, value) + + @property + def patched_tag(self): + return self.tag + + @patched_tag.setter + def patched_tag(self, value): + if not value: + return + tag = copy.deepcopy(self.tag) + tag.update(value) + self.tag = value + + @property + def patched_location(self): + return self.location + + @patched_location.setter + def patched_location(self, value): + if not value: + return + location = copy.deepcopy(self.location) + location.update(value) + self.location = location + + def to_dict(self): + # TODO(xicheng): move the filling of switches + # to db/api. + dict_info = {} + dict_info['switches'] = [ + { + 'switch_ip': switch_machine.switch_ip, + 'port': switch_machine.port, + 'vlans': switch_machine.vlans + } + for switch_machine in self.switch_machines + if not switch_machine.filtered + ] + if dict_info['switches']: + dict_info.update(dict_info['switches'][0]) + dict_info.update(super(Machine, self).to_dict()) + return dict_info + + +class Switch(BASE, HelperMixin, TimestampMixin): + """Switch table.""" + __tablename__ = 'switch' + id = Column(Integer, primary_key=True) + ip_int = Column('ip', BigInteger, unique=True, nullable=False) + credentials = Column(JSONEncoded, default={}) + vendor = Column(String(256), nullable=True) + state = Column(Enum('initialized', 'unreachable', 'notsupported', + 'repolling', 'error', 'under_monitoring', + name='switch_state'), + ColumnDefault('initialized')) + # filters is json formatted list, each element has following format: + # keys: ['filter_type', 'ports', 'port_prefix', 'port_suffix', + # 'port_start', 'port_end']. + # each port name is divided into + # filter_type is one of ['allow', 'deny'], default is 'allow' + # ports is a list of port name. + # port_prefix is the prefix that filtered port should start with. + # port_suffix is the suffix that filtered posrt should end with. + # port_start is integer that the port number should start with. + # port_end is the integer that the port number should end with. + _filters = Column('filters', JSONEncoded, default=[]) + switch_machines = relationship( + SwitchMachine, + passive_deletes=True, passive_updates=True, + cascade='all, delete-orphan', + backref=backref('switch') + ) + + def __str__(self): + return 'Switch[%s:%s]' % (self.id, self.ip) + + @classmethod + def parse_filters(cls, filters): + """parse filters set from outside to standard format. + + api can set switch filters with the flexible format, this + function will parse the flexible format filters. + + Supported format: + as string: + allow ports ae10,ae20 + allow port_prefix ae port_start 30 port_end 40 + deny ports all + as python object: + [{ + 'filter_type': 'allow', + 'ports': ['ae10', 'ae20'] + },{ + 'filter_type': 'allow', + 'port_prefix': 'ae', + 'port_suffix': '', + 'port_start': 30, + 'port_end': 40 + },{ + 'filter_type': 'deny', + 'ports': ['all'] + }] + """ + if isinstance(filters, basestring): + filters = filters.replace('\r\n', '\n').replace('\n', ';') + filters = [ + machine_filter for machine_filter in filters.split(';') + if machine_filter + ] + if not isinstance(filters, list): + filters = [filters] + machine_filters = [] + for machine_filter in filters: + if not machine_filter: + continue + if isinstance(machine_filter, basestring): + filter_dict = {} + filter_items = [ + item for item in machine_filter.split() if item + ] + if filter_items[0] in ['allow', 'deny']: + filter_dict['filter_type'] = filter_items[0] + filter_items = filter_items[1:] + elif filter_items[0] not in [ + 'ports', 'port_prefix', 'port_suffix', + 'port_start', 'port_end' + ]: + raise exception.InvalidParameter( + 'unrecognized filter type %s' % filter_items[0] + ) + while filter_items: + if len(filter_items) >= 2: + filter_dict[filter_items[0]] = filter_items[1] + filter_items = filter_items[2:] + else: + filter_dict[filter_items[0]] = '' + filter_items = filter_items[1:] + machine_filter = filter_dict + if not isinstance(machine_filter, dict): + raise exception.InvalidParameter( + 'filter %s is not dict' % machine_filter + ) + if 'filter_type' in machine_filter: + if machine_filter['filter_type'] not in ['allow', 'deny']: + raise exception.InvalidParameter( + 'filter_type should be `allow` or `deny` in %s' % ( + machine_filter + ) + ) + if 'ports' in machine_filter: + if isinstance(machine_filter['ports'], basestring): + machine_filter['ports'] = [ + port_or_ports + for port_or_ports in machine_filter['ports'].split(',') + if port_or_ports + ] + if not isinstance(machine_filter['ports'], list): + raise exception.InvalidParameter( + '`ports` type is not list in filter %s' % ( + machine_filter + ) + ) + for port_or_ports in machine_filter['ports']: + if not isinstance(port_or_ports, basestring): + raise exception.InvalidParameter( + '%s type is not basestring in `ports` %s' % ( + port_or_ports, machine_filter['ports'] + ) + ) + for key in ['port_start', 'port_end']: + if key in machine_filter: + if isinstance(machine_filter[key], basestring): + if machine_filter[key].isdigit(): + machine_filter[key] = int(machine_filter[key]) + if not isinstance(machine_filter[key], (int, long)): + raise exception.InvalidParameter( + '`%s` type is not int in filer %s' % ( + key, machine_filter + ) + ) + machine_filters.append(machine_filter) + return machine_filters + + @classmethod + def format_filters(cls, filters): + """format json formatted filters to string.""" + filter_strs = [] + for machine_filter in filters: + filter_properties = [] + filter_properties.append( + machine_filter.get('filter_type', 'allow') + ) + if 'ports' in machine_filter: + filter_properties.append( + 'ports ' + ','.join(machine_filter['ports']) + ) + if 'port_prefix' in machine_filter: + filter_properties.append( + 'port_prefix ' + machine_filter['port_prefix'] + ) + if 'port_suffix' in machine_filter: + filter_properties.append( + 'port_suffix ' + machine_filter['port_suffix'] + ) + if 'port_start' in machine_filter: + filter_properties.append( + 'port_start ' + str(machine_filter['port_start']) + ) + if 'port_end' in machine_filter: + filter_properties.append( + 'port_end ' + str(machine_filter['port_end']) + ) + filter_strs.append(' '.join(filter_properties)) + return ';'.join(filter_strs) + + def __init__(self, ip_int, **kwargs): + self.ip_int = ip_int + super(Switch, self).__init__(**kwargs) + + @property + def ip(self): + return str(netaddr.IPAddress(self.ip_int)) + + @ip.setter + def ip(self, ipaddr): + self.ip_int = int(netaddr.IPAddress(ipaddr)) + + @property + def patched_credentials(self): + return self.credentials + + @patched_credentials.setter + def patched_credentials(self, value): + if not value: + return + credentials = copy.deepcopy(self.credentials) + self.credentials = util.merge_dict(credentials, value) + + @property + def machine_filters(self): + return self._filters + + @machine_filters.setter + def machine_filters(self, value): + if not value: + return + self._filters = self.parse_filters(value) + + @property + def put_machine_filters(self): + return self._filters + + @put_machine_filters.setter + def put_machine_filters(self, value): + if not value: + return + self._filters = self.parse_filters(value) + + @property + def patched_machine_filters(self): + return self._filters + + @patched_machine_filters.setter + def patched_machine_filters(self, value): + if not value: + return + filters = list(self.machine_filters) + self._filters = self.parse_filters(value) + filters + + def to_dict(self): + dict_info = super(Switch, self).to_dict() + dict_info['ip'] = self.ip + dict_info['filters'] = self.format_filters(self._filters) + return dict_info + + +class Subnet(BASE, TimestampMixin, HelperMixin): + """network table.""" + __tablename__ = 'subnet' + + id = Column(Integer, primary_key=True) + name = Column(String(80), unique=True, nullable=True) + subnet = Column(String(80), unique=True, nullable=False) + + host_networks = relationship( + HostNetwork, + passive_deletes=True, passive_updates=True, + cascade='all, delete-orphan', + backref=backref('subnet') + ) + + def __init__(self, subnet, **kwargs): + self.subnet = subnet + super(Subnet, self).__init__(**kwargs) + + def __str__(self): + return 'Subnet[%s:%s]' % (self.id, self.subnet) + + def to_dict(self): + dict_info = super(Subnet, self).to_dict() + if not self.name: + dict_info['name'] = self.subnet + return dict_info + + +# TODO(grace): move this global variable into HealthCheckReport. +HEALTH_REPORT_STATES = ('verifying', 'success', 'finished', 'error') + + +class HealthCheckReport(BASE, HelperMixin): + """Health check report table.""" + __tablename__ = 'health_check_report' + + cluster_id = Column( + Integer, + ForeignKey('cluster.id', onupdate='CASCADE', ondelete='CASCADE'), + primary_key=True + ) + name = Column(String(80), nullable=False, primary_key=True) + display_name = Column(String(100)) + report = Column(JSONEncoded, default={}) + category = Column(String(80), default='') + state = Column( + Enum(*HEALTH_REPORT_STATES, name='report_state'), + ColumnDefault('verifying'), + nullable=False + ) + error_message = Column(Text, default='') + + def __init__(self, cluster_id, name, **kwargs): + self.cluster_id = cluster_id + self.name = name + if 'state' in kwargs and kwargs['state'] not in HEALTH_REPORT_STATES: + err_msg = 'State value %s is not accepted.' % kwargs['state'] + raise exception.InvalidParameter(err_msg) + + super(HealthCheckReport, self).__init__(**kwargs) + + def __str__(self): + return 'HealthCheckReport[cluster_id: %s, name: %s]' % ( + self.cluster_id, self.name + ) diff --git a/compass-tasks-base/db/v1/model.py b/compass-tasks-base/db/v1/model.py new file mode 100644 index 0000000..d74e355 --- /dev/null +++ b/compass-tasks-base/db/v1/model.py @@ -0,0 +1,724 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""database model.""" +from datetime import datetime +from hashlib import md5 +import logging +import simplejson as json +import uuid + +from sqlalchemy import Column, ColumnDefault, Integer, String +from sqlalchemy import Float, Enum, DateTime, ForeignKey, Text, Boolean +from sqlalchemy import UniqueConstraint +from sqlalchemy.orm import relationship, backref +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.hybrid import hybrid_property + +from compass.utils import util + +from flask.ext.login import UserMixin +from itsdangerous import URLSafeTimedSerializer + +BASE = declarative_base() +# TODO(grace) SECRET_KEY should be generated when installing compass +# and save to a config file or DB +SECRET_KEY = "abcd" + +# This is used for generating a token by user's ID and +# decode the ID from this token +login_serializer = URLSafeTimedSerializer(SECRET_KEY) + + +class User(BASE, UserMixin): + """User table.""" + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + email = Column(String(80), unique=True) + password = Column(String(225), default='') + active = Column(Boolean, default=True) + + def __init__(self, email, password, **kwargs): + self.email = email + self.password = self._set_password(password) + + def __repr__(self): + return '' % self.email + + def _set_password(self, password): + return self._hash_password(password) + + def get_password(self): + return self.password + + def valid_password(self, password): + return self.password == self._hash_password(password) + + def get_auth_token(self): + return login_serializer.dumps(self.id) + + def is_active(self): + return self.active + + def _hash_password(self, password): + return md5(password).hexdigest() + + +class SwitchConfig(BASE): + """Swtich Config table. + + :param id: The unique identifier of the switch config. + :param ip: The IP address of the switch. + :param filter_port: The port of the switch which need to be filtered. + """ + __tablename__ = 'switch_config' + id = Column(Integer, primary_key=True) + ip = Column(String(80)) + filter_port = Column(String(16)) + __table_args__ = (UniqueConstraint('ip', 'filter_port', name='filter1'), ) + + def __init__(self, **kwargs): + super(SwitchConfig, self).__init__(**kwargs) + + +class Switch(BASE): + """Switch table. + + :param id: the unique identifier of the switch. int as primary key. + :param ip: the IP address of the switch. + :param vendor_info: the name of the vendor + :param credential_data: used for accessing and retrieving information + from the switch. Store json format as string. + :param state: Enum.'initialized/repolling': polling switch not complete to + learn all MAC addresses of devices connected to the switch; + 'unreachable': one of the final state, indicates that the + switch is unreachable at this time, no MAC address could be + retrieved from the switch. + 'notsupported': one of the final state, indicates that the + vendor found is not supported yet, no MAC address will be + retrieved from the switch. + 'error': one of the final state, indicates that something + wrong happend. + 'under_monitoring': one of the final state, indicates that + MAC addresses has been learned successfully from the switch. + :param err_msg: Error message when polling switch failed. + :param machines: refer to list of Machine connected to the switch. + """ + __tablename__ = 'switch' + + id = Column(Integer, primary_key=True) + ip = Column(String(80), unique=True) + credential_data = Column(Text) + vendor_info = Column(String(256), nullable=True) + state = Column(Enum('initialized', 'unreachable', 'notsupported', + 'repolling', 'error', 'under_monitoring', + name='switch_state'), + default='initialized') + err_msg = Column(Text) + + def __init__(self, **kwargs): + super(Switch, self).__init__(**kwargs) + + def __repr__(self): + return ''\ + % (self.ip, self.credential, self.vendor, self.state) + + @hybrid_property + def vendor(self): + """vendor property getter""" + return self.vendor_info + + @vendor.setter + def vendor(self, value): + """vendor property setter""" + self.vendor_info = value + + @property + def credential(self): + """credential data getter. + + :returns: python primitive dictionary object. + """ + if self.credential_data: + try: + credential = json.loads(self.credential_data) + return credential + except Exception as error: + logging.error('failed to load credential data %s: %s', + self.id, self.credential_data) + logging.exception(error) + raise error + else: + return {} + + @credential.setter + def credential(self, value): + """credential property setter + + :param value: dict of configuration data needed to update. + """ + if value: + try: + credential = {} + if self.credential_data: + credential = json.loads(self.credential_data) + + credential.update(value) + self.credential_data = json.dumps(credential) + + except Exception as error: + logging.error('failed to dump credential data %s: %s', + self.id, value) + logging.exception(error) + raise error + + else: + self.credential_data = json.dumps({}) + + logging.debug('switch now is %s', self) + + +class Machine(BASE): + """Machine table. + + .. note:: + currently, we are taking care of management plane. + Therefore, we assume one machine is connected to one switch. + + :param id: int, identity as primary key + :param mac: string, the MAC address of the machine. + :param switch_id: switch id that this machine connected on to. + :param port: nth port of the switch that this machine connected. + :param vlan: vlan id that this machine connected on to. + :param update_timestamp: last time this entry got updated. + :param switch: refer to the Switch the machine connects to. + """ + __tablename__ = 'machine' + + id = Column(Integer, primary_key=True) + mac = Column(String(24), default='') + port = Column(String(16), default='') + vlan = Column(Integer, default=0) + update_timestamp = Column(DateTime, default=datetime.now, + onupdate=datetime.now) + switch_id = Column(Integer, ForeignKey('switch.id', + onupdate='CASCADE', + ondelete='SET NULL')) + __table_args__ = (UniqueConstraint('mac', 'switch_id', + name='unique_machine'),) + switch = relationship('Switch', backref=backref('machines', + lazy='dynamic')) + + def __init__(self, **kwargs): + super(Machine, self).__init__(**kwargs) + + def __repr__(self): + return '' % ( + self.mac, self.port, self.vlan, self.switch) + + +class HostState(BASE): + """The state of the ClusterHost. + + :param id: int, identity as primary key. + :param state: Enum. 'UNINITIALIZED': the host is ready to setup. + 'INSTALLING': the host is not installing. + 'READY': the host is setup. + 'ERROR': the host has error. + :param progress: float, the installing progress from 0 to 1. + :param message: the latest installing message. + :param severity: Enum, the installing message severity. + ('INFO', 'WARNING', 'ERROR') + :param update_timestamp: the lastest timestamp the entry got updated. + :param host: refer to ClusterHost. + :param os_progress: float, the installing progress of OS from 0 to 1. + """ + __tablename__ = "host_state" + + id = Column(Integer, ForeignKey('cluster_host.id', + onupdate='CASCADE', + ondelete='CASCADE'), + primary_key=True) + state = Column(Enum('UNINITIALIZED', 'INSTALLING', 'READY', 'ERROR'), + ColumnDefault('UNINITIALIZED')) + progress = Column(Float, ColumnDefault(0.0)) + message = Column(Text) + severity = Column(Enum('INFO', 'WARNING', 'ERROR'), ColumnDefault('INFO')) + update_timestamp = Column(DateTime, default=datetime.now, + onupdate=datetime.now) + host = relationship('ClusterHost', backref=backref('state', + uselist=False)) + + os_progress = Column(Float, ColumnDefault(0.0)) + os_message = Column(Text) + os_severity = Column( + Enum('INFO', 'WARNING', 'ERROR'), + ColumnDefault('INFO') + ) + """ + this is added by Lei for separating os and package progress purposes + os_state = Column(Enum('UNINITIALIZED', 'INSTALLING', 'OS_READY', 'ERROR'), + ColumnDefault('UNINITIALIZED')) + """ + + def __init__(self, **kwargs): + super(HostState, self).__init__(**kwargs) + + @hybrid_property + def hostname(self): + """hostname getter""" + return self.host.hostname + + @hybrid_property + def fullname(self): + """fullname getter""" + return self.host.fullname + + def __repr__(self): + return ( + '' + ) % ( + self.hostname, self.state, self.progress, + self.message, self.severity, self.os_progress + ) + + +class ClusterState(BASE): + """The state of the Cluster. + + :param id: int, identity as primary key. + :param state: Enum, 'UNINITIALIZED': the cluster is ready to setup. + 'INSTALLING': the cluster is not installing. + 'READY': the cluster is setup. + 'ERROR': the cluster has error. + :param progress: float, the installing progress from 0 to 1. + :param message: the latest installing message. + :param severity: Enum, the installing message severity. + ('INFO', 'WARNING', 'ERROR'). + :param update_timestamp: the lastest timestamp the entry got updated. + :param cluster: refer to Cluster. + """ + __tablename__ = 'cluster_state' + id = Column(Integer, ForeignKey('cluster.id', + onupdate='CASCADE', + ondelete='CASCADE'), + primary_key=True) + state = Column(Enum('UNINITIALIZED', 'INSTALLING', 'READY', 'ERROR'), + ColumnDefault('UNINITIALIZED')) + progress = Column(Float, ColumnDefault(0.0)) + message = Column(Text) + severity = Column(Enum('INFO', 'WARNING', 'ERROR'), ColumnDefault('INFO')) + update_timestamp = Column(DateTime, default=datetime.now, + onupdate=datetime.now) + cluster = relationship('Cluster', backref=backref('state', + uselist=False)) + + def __init__(self, **kwargs): + super(ClusterState, self).__init__(**kwargs) + + @hybrid_property + def clustername(self): + """clustername getter""" + return self.cluster.name + + def __repr__(self): + return ( + '' + ) % ( + self.clustername, self.state, self.progress, + self.message, self.severity + ) + + +class Cluster(BASE): + """Cluster configuration information. + + :param id: int, identity as primary key. + :param name: str, cluster name. + :param mutable: bool, if the Cluster is mutable. + :param security_config: str stores json formatted security information. + :param networking_config: str stores json formatted networking information. + :param partition_config: string stores json formatted parition information. + :param adapter_id: the refer id in the Adapter table. + :param raw_config: str stores json formatted other cluster information. + :param adapter: refer to the Adapter. + :param state: refer to the ClusterState. + """ + __tablename__ = 'cluster' + + id = Column(Integer, primary_key=True) + name = Column(String(80), unique=True) + mutable = Column(Boolean, default=True) + security_config = Column(Text) + networking_config = Column(Text) + partition_config = Column(Text) + adapter_id = Column(Integer, ForeignKey('adapter.id', + onupdate='CASCADE', + ondelete='SET NULL'), + nullable=True) + raw_config = Column(Text) + adapter = relationship("Adapter", backref=backref('clusters', + lazy='dynamic')) + + def __init__(self, **kwargs): + if 'name' not in kwargs or not kwargs['name']: + kwargs['name'] = str(uuid.uuid4()) + + super(Cluster, self).__init__(**kwargs) + + def __repr__(self): + return '' % (self.name, self.config) + + @property + def partition(self): + """partition getter""" + if self.partition_config: + try: + return json.loads(self.partition_config) + except Exception as error: + logging.error('failed to load security config %s: %s', + self.id, self.partition_config) + logging.exception(error) + raise error + else: + return {} + + @partition.setter + def partition(self, value): + """partition setter""" + logging.debug('cluster %s set partition %s', self.id, value) + if value: + try: + self.partition_config = json.dumps(value) + except Exception as error: + logging.error('failed to dump partition config %s: %s', + self.id, value) + logging.exception(error) + raise error + else: + self.partition_config = None + + @property + def security(self): + """security getter""" + if self.security_config: + try: + return json.loads(self.security_config) + except Exception as error: + logging.error('failed to load security config %s: %s', + self.id, self.security_config) + logging.exception(error) + raise error + else: + return {} + + @security.setter + def security(self, value): + """security setter""" + logging.debug('cluster %s set security %s', self.id, value) + if value: + try: + self.security_config = json.dumps(value) + except Exception as error: + logging.error('failed to dump security config %s: %s', + self.id, value) + logging.exception(error) + raise error + else: + self.security_config = None + + @property + def networking(self): + """networking getter""" + if self.networking_config: + try: + return json.loads(self.networking_config) + except Exception as error: + logging.error('failed to load networking config %s: %s', + self.id, self.networking_config) + logging.exception(error) + raise error + else: + return {} + + @networking.setter + def networking(self, value): + """networking setter.""" + logging.debug('cluster %s set networking %s', self.id, value) + if value: + try: + self.networking_config = json.dumps(value) + except Exception as error: + logging.error('failed to dump networking config %s: %s', + self.id, value) + logging.exception(error) + raise error + else: + self.networking_config = None + + @hybrid_property + def config(self): + """get config from security, networking, partition.""" + config = {} + if self.raw_config: + try: + config = json.loads(self.raw_config) + except Exception as error: + logging.error('failed to load raw config %s: %s', + self.id, self.raw_config) + logging.exception(error) + raise error + + util.merge_dict(config, {'security': self.security}) + util.merge_dict(config, {'networking': self.networking}) + util.merge_dict(config, {'partition': self.partition}) + util.merge_dict(config, {'clusterid': self.id, + 'clustername': self.name}) + return config + + @config.setter + def config(self, value): + """set config to security, networking, partition.""" + logging.debug('cluster %s set config %s', self.id, value) + if not value: + self.security = None + self.networking = None + self.partition = None + self.raw_config = None + return + + self.security = value.get('security') + self.networking = value.get('networking') + self.partition = value.get('partition') + + try: + self.raw_config = json.dumps(value) + except Exception as error: + logging.error('failed to dump raw config %s: %s', + self.id, value) + logging.exception(error) + raise error + + +class ClusterHost(BASE): + """ClusterHost information. + + :param id: int, identity as primary key. + :param machine_id: int, the id of the Machine. + :param cluster_id: int, the id of the Cluster. + :param mutable: if the ClusterHost information is mutable. + :param hostname: str, host name. + :param config_data: string, json formatted config data. + :param cluster: refer to Cluster the host in. + :param machine: refer to the Machine the host on. + :param state: refer to HostState indicates the host state. + """ + __tablename__ = 'cluster_host' + + id = Column(Integer, primary_key=True) + + machine_id = Column(Integer, ForeignKey('machine.id', + onupdate='CASCADE', + ondelete='CASCADE'), + nullable=True, unique=True) + + cluster_id = Column(Integer, ForeignKey('cluster.id', + onupdate='CASCADE', + ondelete='SET NULL'), + nullable=True) + + hostname = Column(String(80)) + config_data = Column(Text) + mutable = Column(Boolean, default=True) + __table_args__ = (UniqueConstraint('cluster_id', 'hostname', + name='unique_host'),) + + cluster = relationship("Cluster", + backref=backref('hosts', lazy='dynamic')) + machine = relationship("Machine", + backref=backref('host', uselist=False)) + + def __init__(self, **kwargs): + if 'hostname' not in kwargs or not kwargs['hostname']: + kwargs['hostname'] = str(uuid.uuid4()) + + super(ClusterHost, self).__init__(**kwargs) + + def __repr__(self): + return '' % ( + self.hostname, self.cluster, self.machine) + + @hybrid_property + def fullname(self): + return '%s.%s' % (self.hostname, self.cluster.id) + + @property + def config(self): + """config getter.""" + config = {} + try: + if self.config_data: + config.update(json.loads(self.config_data)) + + config.update({ + 'hostid': self.id, + 'hostname': self.hostname, + }) + if self.cluster: + config.update({ + 'clusterid': self.cluster.id, + 'clustername': self.cluster.name, + 'fullname': self.fullname, + }) + + if self.machine: + util.merge_dict( + config, { + 'networking': { + 'interfaces': { + 'management': { + 'mac': self.machine.mac + } + } + }, + 'switch_port': self.machine.port, + 'vlan': self.machine.vlan, + }) + if self.machine.switch: + util.merge_dict( + config, {'switch_ip': self.machine.switch.ip}) + + except Exception as error: + logging.error('failed to load config %s: %s', + self.hostname, self.config_data) + logging.exception(error) + raise error + + return config + + @config.setter + def config(self, value): + """config setter""" + if not self.config_data: + config = { + } + self.config_data = json.dumps(config) + + if value: + try: + config = json.loads(self.config_data) + util.merge_dict(config, value) + + self.config_data = json.dumps(config) + except Exception as error: + logging.error('failed to dump config %s: %s', + self.hostname, value) + logging.exception(error) + raise error + + +class LogProgressingHistory(BASE): + """host installing log history for each file. + + :param id: int, identity as primary key. + :param pathname: str, the full path of the installing log file. unique. + :param position: int, the position of the log file it has processed. + :param partial_line: str, partial line of the log. + :param progressing: float, indicate the installing progress between 0 to 1. + :param message: str, str, the installing message. + :param severity: Enum, the installing message severity. + ('ERROR', 'WARNING', 'INFO') + :param line_matcher_name: str, the line matcher name of the log processor. + :param update_timestamp: datetime, the latest timestamp the entry updated. + """ + __tablename__ = 'log_progressing_history' + id = Column(Integer, primary_key=True) + pathname = Column(String(80), unique=True) + position = Column(Integer, ColumnDefault(0)) + partial_line = Column(Text) + progress = Column(Float, ColumnDefault(0.0)) + message = Column(Text) + severity = Column(Enum('ERROR', 'WARNING', 'INFO'), ColumnDefault('INFO')) + line_matcher_name = Column(String(80), ColumnDefault('start')) + update_timestamp = Column(DateTime, default=datetime.now, + onupdate=datetime.now) + + def __init__(self, **kwargs): + super(LogProgressingHistory, self).__init__(**kwargs) + + def __repr__(self): + return ( + 'LogProgressingHistory[%r: position %r,' + 'partial_line %r,progress %r,message %r,' + 'severity %r]' + ) % ( + self.pathname, self.position, + self.partial_line, + self.progress, + self.message, + self.severity + ) + + +class Adapter(BASE): + """Table stores ClusterHost installing Adapter information. + + :param id: int, identity as primary key. + :param name: string, adapter name, unique. + :param os: string, os name for installing the host. + :param target_system: string, target system to be installed on the host. + :param clusters: refer to the list of Cluster. + """ + __tablename__ = 'adapter' + id = Column(Integer, primary_key=True) + name = Column(String(80), unique=True) + os = Column(String(80)) + target_system = Column(String(80)) + __table_args__ = ( + UniqueConstraint('os', 'target_system', name='unique_adapter'),) + + def __init__(self, **kwargs): + super(Adapter, self).__init__(**kwargs) + + def __repr__(self): + return '' % ( + self.name, self.os, self.target_system + ) + + +class Role(BASE): + """The Role table stores avaiable roles of one target system. + + .. note:: + the host can be deployed to one or several roles in the cluster. + + :param id: int, identity as primary key. + :param name: role name. + :param target_system: str, the target_system. + :param description: str, the description of the role. + """ + __tablename__ = 'role' + id = Column(Integer, primary_key=True) + name = Column(String(80), unique=True) + target_system = Column(String(80)) + description = Column(Text) + + def __init__(self, **kwargs): + super(Role, self).__init__(**kwargs) + + def __repr__(self): + return '' % ( + self.name, self.target_system, self.description) diff --git a/compass-tasks-base/db/validator.py b/compass-tasks-base/db/validator.py new file mode 100644 index 0000000..730bb52 --- /dev/null +++ b/compass-tasks-base/db/validator.py @@ -0,0 +1,195 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Validator methods.""" +import logging +import netaddr +import re +import socket + +from compass.utils import setting_wrapper as setting +from compass.utils import util + + +def is_valid_ip(name, ip_addr, **kwargs): + """Valid the format of an IP address.""" + if isinstance(ip_addr, list): + return all([ + is_valid_ip(name, item, **kwargs) for item in ip_addr + ]) + try: + netaddr.IPAddress(ip_addr) + except Exception: + logging.debug('%s invalid ip addr %s', name, ip_addr) + return False + return True + + +def is_valid_network(name, ip_network, **kwargs): + """Valid the format of an Ip network.""" + if isinstance(ip_network, list): + return all([ + is_valid_network(name, item, **kwargs) for item in ip_network + ]) + try: + netaddr.IPNetwork(ip_network) + except Exception: + logging.debug('%s invalid network %s', name, ip_network) + return False + return True + + +def is_valid_netmask(name, ip_addr, **kwargs): + """Valid the format of a netmask.""" + if isinstance(ip_addr, list): + return all([ + is_valid_netmask(name, item, **kwargs) for item in ip_addr + ]) + if not is_valid_ip(ip_addr): + return False + ip = netaddr.IPAddress(ip_addr) + if ip.is_netmask(): + return True + logging.debug('%s invalid netmask %s', name, ip_addr) + return False + + +def is_valid_gateway(name, ip_addr, **kwargs): + """Valid the format of gateway.""" + if isinstance(ip_addr, list): + return all([ + is_valid_gateway(name, item, **kwargs) for item in ip_addr + ]) + if not is_valid_ip(ip_addr): + return False + ip = netaddr.IPAddress(ip_addr) + if ip.is_private() or ip.is_public(): + return True + logging.debug('%s invalid gateway %s', name, ip_addr) + return False + + +def is_valid_dns(name, dns, **kwargs): + """Valid the format of DNS.""" + if isinstance(dns, list): + return all([is_valid_dns(name, item, **kwargs) for item in dns]) + if is_valid_ip(dns): + return True + try: + socket.gethostbyname_ex(dns) + except Exception: + logging.debug('%s invalid dns name %s', name, dns) + return False + return True + + +def is_valid_url(name, url, **kwargs): + """Valid the format of url.""" + if isinstance(url, list): + return all([ + is_valid_url(name, item, **kwargs) for item in url + ]) + if re.match( + r'^(http|https|ftp)://([0-9A-Za-z_-]+)(\.[0-9a-zA-Z_-]+)*' + r'(:\d+)?(/[0-9a-zA-Z_-]+)*$', + url + ): + return True + logging.debug( + '%s invalid url %s', name, url + ) + return False + + +def is_valid_domain(name, domain, **kwargs): + """Validate the format of domain.""" + if isinstance(domain, list): + return all([ + is_valid_domain(name, item, **kwargs) for item in domain + ]) + if re.match( + r'^([0-9a-zA-Z_-]+)(\.[0-9a-zA-Z_-]+)*$', + domain + ): + return True + logging.debug( + '%s invalid domain %s', name, domain + ) + return False + + +def is_valid_username(name, username, **kwargs): + """Valid the format of username.""" + if bool(username): + return True + logging.debug( + '%s username is empty', name + ) + + +def is_valid_password(name, password, **kwargs): + """Valid the format of password.""" + if bool(password): + return True + logging.debug('%s password is empty', name) + return False + + +def is_valid_partition(name, partition, **kwargs): + """Valid the format of partition name.""" + if name != 'swap' and not name.startswith('/'): + logging.debug( + '%s is not started with / or swap', name + ) + return False + if 'size' not in partition and 'percentage' not in partition: + logging.debug( + '%s partition does not contain sie or percentage', + name + ) + return False + return True + + +def is_valid_percentage(name, percentage, **kwargs): + """Valid the percentage.""" + if 0 <= percentage <= 100: + return True + logging.debug('%s invalid percentage %s', name, percentage) + + +def is_valid_port(name, port, **kwargs): + """Valid the format of port.""" + if 0 < port < 65536: + return True + logging.debug('%s invalid port %s', name, port) + + +def is_valid_size(name, size, **kwargs): + if re.match(r'^(\d+)(K|M|G|T)$', size): + return True + logging.debug('%s invalid size %s', name, size) + return False + + +VALIDATOR_GLOBALS = globals() +VALIDATOR_LOCALS = locals() +VALIDATOR_CONFIGS = util.load_configs( + setting.VALIDATOR_DIR, + config_name_suffix='.py', + env_globals=VALIDATOR_GLOBALS, + env_locals=VALIDATOR_LOCALS +) +for validator_config in VALIDATOR_CONFIGS: + VALIDATOR_LOCALS.update(validator_config) diff --git a/compass-tasks-base/deployment/__init__.py b/compass-tasks-base/deployment/__init__.py new file mode 100644 index 0000000..cbd36e0 --- /dev/null +++ b/compass-tasks-base/deployment/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" diff --git a/compass-tasks-base/deployment/deploy_manager.py b/compass-tasks-base/deployment/deploy_manager.py new file mode 100644 index 0000000..baf7cd6 --- /dev/null +++ b/compass-tasks-base/deployment/deploy_manager.py @@ -0,0 +1,237 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" + +"""Module to get configs from provider and isntallers and update + them to provider and installers. +""" +from compass.deployment.installers.installer import OSInstaller +from compass.deployment.installers.installer import PKInstaller +from compass.deployment.utils import constants as const +from compass.utils import util + + +import logging + + +class DeployManager(object): + """Deploy manager module.""" + def __init__(self, adapter_info, cluster_info, hosts_info): + """Init deploy manager.""" + self.os_installer = None + self.pk_installer = None + + # Get OS installer + os_installer_name = adapter_info[const.OS_INSTALLER][const.NAME] + self.os_installer = DeployManager._get_installer(OSInstaller, + os_installer_name, + adapter_info, + cluster_info, + hosts_info) + + # Get package installer + pk_info = adapter_info.setdefault(const.PK_INSTALLER, {}) + if pk_info: + pk_installer_name = pk_info[const.NAME] + self.pk_installer = DeployManager._get_installer(PKInstaller, + pk_installer_name, + adapter_info, + cluster_info, + hosts_info) + + @staticmethod + def _get_installer(installer_type, name, adapter_info, cluster_info, + hosts_info): + """Get installer instance.""" + callback = getattr(installer_type, 'get_installer') + installer = callback(name, adapter_info, cluster_info, hosts_info) + + return installer + + def deploy(self): + """Deploy the cluster.""" + deployed_config = self.deploy_os() + package_deployed_config = self.deploy_target_system() + + util.merge_dict(deployed_config, package_deployed_config) + + return deployed_config + + def check_cluster_health(self, callback_url): + logging.info("DeployManager check_cluster_health...........") + self.pk_installer.check_cluster_health(callback_url) + + def clean_progress(self): + """Clean previous installation log and progress.""" + self.clean_os_installtion_progress() + self.clean_package_installation_progress() + + def clean_os_installtion_progress(self): + # OS installer cleans previous installing progress. + if self.os_installer: + self.os_installer.clean_progress() + + def clean_package_installation_progress(self): + # Package installer cleans previous installing progress. + if self.pk_installer: + self.pk_installer.clean_progress() + + def prepare_for_deploy(self): + self.clean_progress() + + def deploy_os(self): + """Deploy OS to hosts which need to in the cluster. + + Return OS deployed config. + """ + if not self.os_installer: + return {} + + pk_installer_config = {} + if self.pk_installer: + # generate target system config which will be installed by OS + # installer right after OS installation is completed. + pk_installer_config = self.pk_installer.generate_installer_config() + logging.debug('[DeployManager]package installer config is %s', + pk_installer_config) + + # Send package installer config info to OS installer. + self.os_installer.set_package_installer_config(pk_installer_config) + + # start to deploy OS + return self.os_installer.deploy() + + def deploy_target_system(self): + """Deploy target system to all hosts in the cluster. + + Return package deployed config. + """ + if not self.pk_installer: + return {} + + return self.pk_installer.deploy() + + def redeploy_os(self): + """Redeploy OS for this cluster without changing configurations.""" + if not self.os_installer: + logging.info("Redeploy_os: No OS installer found!") + return + + self.os_installer.redeploy() + logging.info("Start to redeploy OS for cluster.") + + def redeploy_target_system(self): + """Redeploy target system for the cluster without changing config.""" + if not self.pk_installer: + logging.info("Redeploy_target_system: No installer found!") + return + + self.pk_installer.deploy() + logging.info("Start to redeploy target system.") + + def redeploy(self): + """Redeploy the cluster without changing configurations.""" + self.redeploy_os() + self.redeploy_target_system() + + def remove_hosts(self, package_only=False, delete_cluster=False): + """Remove hosts from both OS and/or package installlers server side.""" + if self.os_installer and not package_only: + self.os_installer.delete_hosts() + + if self.pk_installer: + self.pk_installer.delete_hosts(delete_cluster=delete_cluster) + + def os_installed(self): + if self.os_installer: + self.os_installer.ready() + if self.pk_installer: + self.pk_installer.os_ready() + + def cluster_os_installed(self): + if self.os_installer: + self.os_installer.cluster_ready() + if self.pk_installer: + self.pk_installer.cluster_os_ready() + + def package_installed(self): + if self.pk_installer: + self.pk_installer.ready() + + def cluster_installed(self): + if self.pk_installer: + self.pk_installer.cluster_ready() + + +class Patcher(DeployManager): + """Patcher Module.""" + def __init__(self, adapter_info, cluster_info, hosts_info, cluster_hosts): + self.pk_installer = None + self.cluster_info = cluster_info + registered_roles = cluster_info['flavor']['roles'] + + pk_info = adapter_info.setdefault(const.PK_INSTALLER, {}) + if pk_info: + pk_installer_name = pk_info[const.NAME] + self.pk_installer = Patcher._get_installer(PKInstaller, + pk_installer_name, + adapter_info, + cluster_info, + hosts_info) + + patched_role_mapping = {} + for role in registered_roles: + patched_role_mapping[role] = [] + for host in cluster_hosts: + if len(host['patched_roles']) == 0: + continue + for role in host['patched_roles']: + patched_role_mapping[role['name']].append(host) + self.patched_role_mapping = patched_role_mapping + + def patch(self): + patched_config = self.pk_installer.patch(self.patched_role_mapping) + + return patched_config + + +class PowerManager(object): + """Manage host to power on, power off, and reset.""" + + def __init__(self, adapter_info, cluster_info, hosts_info): + os_installer_name = adapter_info[const.OS_INSTALLER][const.NAME] + self.os_installer = DeployManager._get_installer(OSInstaller, + os_installer_name, + adapter_info, + cluster_info, + hosts_info) + + def poweron(self): + if not self.os_installer: + logging.info("No OS installer found, cannot power on machine!") + return + self.os_installer.poweron() + + def poweroff(self): + if not self.os_installer: + logging.info("No OS installer found, cannot power on machine!") + return + self.os_installer.poweroff() + + def reset(self): + if not self.os_installer: + logging.info("No OS installer found, cannot power on machine!") + return + self.os_installer.reset() diff --git a/compass-tasks-base/deployment/installers/__init__.py b/compass-tasks-base/deployment/installers/__init__.py new file mode 100644 index 0000000..0296be5 --- /dev/null +++ b/compass-tasks-base/deployment/installers/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" + + +"""modules to read/write cluster/host config from installers. + + .. moduleauthor:: Grace Yu +""" diff --git a/compass-tasks-base/deployment/installers/config_manager.py b/compass-tasks-base/deployment/installers/config_manager.py new file mode 100644 index 0000000..597c3a6 --- /dev/null +++ b/compass-tasks-base/deployment/installers/config_manager.py @@ -0,0 +1,527 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "baigk baiguoku@huawei.com)" + +from collections import defaultdict +from copy import deepcopy +import json +import logging +import netaddr + +from compass.deployment.utils import constants as const + +ip_generator_map = {} + + +def get_ip_addr(ip_ranges): + def _get_ip_addr(): + for ip_range in ip_ranges: + for ip in netaddr.iter_iprange(*ip_range): + yield str(ip) + + s = json.dumps(ip_ranges) + if s not in ip_generator_map: + ip_generator_map[s] = _get_ip_addr() + return ip_generator_map[s] + else: + return ip_generator_map[s] + + +class AdapterInfo(object): + def __init__(self, adapter_info): + self.adapter_info = adapter_info + self.name = self.adapter_info.get(const.NAME) + self.dist_system_name = self.name + self.health_check_cmd = self.adapter_info.get(const.HEALTH_CHECK_CMD) + + self.os_installer = self.adapter_info.setdefault( + const.OS_INSTALLER, {} + ) + self.os_installer.setdefault(const.INSTALLER_SETTINGS, {}) + + self.package_installer = self.adapter_info.setdefault( + const.PK_INSTALLER, {} + ) + self.package_installer.setdefault(const.INSTALLER_SETTINGS, {}) + + self.metadata = self.adapter_info.setdefault(const.METADATA, {}) + self.os_metadata = self.metadata.setdefault(const.OS_CONFIG, {}) + self.package_metadata = self.metadata.setdefault(const.PK_CONFIG, {}) + + self.flavors = dict([(f[const.FLAVOR_NAME], f) + for f in self.adapter_info.get(const.FLAVOR, [])]) + + @property + def flavor_list(self): + return self.flavors.values() + + def get_flavor(self, flavor_name): + return self.flavors.get(flavor_name) + + +class ClusterInfo(object): + def __init__(self, cluster_info): + self.cluster_info = cluster_info + self.id = self.cluster_info.get(const.ID) + self.name = self.cluster_info.get(const.NAME) + self.os_version = self.cluster_info.get(const.OS_VERSION) + self.flavor = self.cluster_info.setdefault( + const.FLAVOR, {} + ) + self.os_config = self.cluster_info.setdefault( + const.OS_CONFIG, {} + ) + self.package_config = self.cluster_info.setdefault( + const.PK_CONFIG, {} + ) + self.deployed_os_config = self.cluster_info.setdefault( + const.DEPLOYED_OS_CONFIG, {} + ) + self.deployed_package_config = self.cluster_info.setdefault( + const.DEPLOYED_PK_CONFIG, {} + ) + self.network_mapping = self.package_config.setdefault( + const.NETWORK_MAPPING, {} + ) + + os_config_general = self.os_config.setdefault( + const.OS_CONFIG_GENERAL, {} + ) + self.domain = os_config_general.setdefault(const.DOMAIN, None) + self.hosts = [] + + def add_host(self, host): + self.hosts.append(host) + + @property + def roles_mapping(self): + deploy_config = self.deployed_package_config + return deploy_config.setdefault( + const.ROLES_MAPPING, self._get_cluster_roles_mapping() + ) + + def _get_cluster_roles_mapping(self): + """The ouput format will be as below, for example: + + { + "controller": [{ + "hostname": "xxx", + "management": { + "interface": "eth0", + "ip": "192.168.1.10", + "netmask": "255.255.255.0", + "subnet": "192.168.1.0/24", + "is_mgmt": True, + "is_promiscuous": False + }, + ... + }], + ... + } + """ + mapping = defaultdict(list) + for host in self.hosts: + for role, value in host.roles_mapping.iteritems(): + mapping[role].append(value) + + return dict(mapping) + + def _get_cluster_patched_roles_mapping(self): + mapping = defaultdict(list) + for host in self.hosts: + for role, value in host.patched_roles_mapping.iteritems(): + mapping[role].append(value) + + return dict(mapping) + + @property + def base_info(self): + return { + const.ID: self.id, + const.NAME: self.name, + const.OS_VERSION: self.os_version + } + + +class HostInfo(object): + def __init__(self, host_info, cluster_info): + self.host_info = host_info + self.cluster_info = cluster_info + self.id = self.host_info.get(const.ID) + self.name = self.host_info.get(const.NAME) + self.mac = self.host_info.get(const.MAC_ADDR) + self.hostname = self.host_info.get(const.HOSTNAME) + self.networks = self.host_info.setdefault(const.NETWORKS, {}) + self.os_config = self.host_info.setdefault(const.OS_CONFIG, {}) + + self.package_config = self.host_info.setdefault(const.PK_CONFIG, {}) + self.roles = self.host_info.setdefault(const.ROLES, []) + self.patched_roles = self.host_info.setdefault(const.PATCHED_ROLES, []) + self.ipmi = deepcopy(self.host_info.setdefault(const.IPMI, {})) + self.reinstall_os_flag = self.host_info.get(const.REINSTALL_OS_FLAG) + self.deployed_os_config = self.host_info.setdefault( + const.DEPLOYED_OS_CONFIG, {} + ) + self.deployed_package_config = self.host_info.setdefault( + const.DEPLOYED_PK_CONFIG, {} + ) + + os_general_config = self.os_config.setdefault( + const.OS_CONFIG_GENERAL, {} + ) + domain = os_general_config.setdefault(const.DOMAIN, None) + if domain is None: + self.domain = self.cluster_info.domain + else: + self.domain = domain + + if const.DNS in host_info: + self.dns = host_info[const.DNS] + else: + self.dns = '.'.join((self.hostname, self.domain)) + + if const.NETWORK_MAPPING not in self.package_config: + self.network_mapping = self.cluster_info.network_mapping + else: + self.network_mapping = self.package_config[const.NETWORK_MAPPING] + + if const.ROLES_MAPPING not in self.deployed_package_config: + self.roles_mapping = self._get_host_roles_mapping() + self.deployed_package_config[ + const.ROLES_MAPPING + ] = self.roles_mapping + else: + self.roles_mapping = \ + self.deployed_package_config[const.ROLES_MAPPING] + + self.patched_roles_mapping = self._get_host_patched_roles_mapping() + + self.cluster_info.add_host(self) + + def valid_interface(self, interface): + if interface not in self.networks: + raise RuntimeError("interface %s is invalid" % interface) + + def get_interface(self, interface): + self.valid_interface(interface) + return self.networks[interface] + + def get_interface_ip(self, interface): + return self.get_interface(interface).get(const.IP_ADDR) + + def get_interface_netmask(self, interface): + return self.get_interface(interface).get(const.NETMASK) + + def get_interface_subnet(self, interface): + return self.get_interface(interface).get(const.SUBNET) + + def is_interface_promiscuous(self, interface): + return self.get_interface(interface).get(const.PROMISCUOUS_FLAG) + + def is_interface_mgmt(self, interface): + return self.get_interface(interface).get(const.MGMT_NIC_FLAG) + + def _get_host_roles_mapping(self): + if not self.network_mapping: + return {} + + net_info = {const.HOSTNAME: self.hostname} + for k, v in self.network_mapping.items(): + try: + net_info[k] = self.networks[v[const.NIC]] + net_info[k][const.NIC] = v[const.NIC] + except Exception: + pass + + mapping = {} + for role in self.roles: + role = role.replace("-", "_") + mapping[role] = net_info + + return mapping + + def _get_host_patched_roles_mapping(self): + if not self.network_mapping: + return {} + + net_info = {const.HOSTNAME: self.hostname} + for k, v in self.network_mapping.items(): + try: + net_info[k] = self.networks[v[const.NIC]] + net_info[k][const.NIC] = v[const.NIC] + except Exception: + pass + + mapping = {} + for role in self.patched_roles: + role = role['name'].replace("-", "_") + mapping[role] = net_info + + return mapping + + @property + def baseinfo(self): + return { + const.REINSTALL_OS_FLAG: self.reinstall_os_flag, + const.MAC_ADDR: self.mac, + const.NAME: self.name, + const.HOSTNAME: self.hostname, + const.DNS: self.dns, + const.NETWORKS: deepcopy(self.networks) + } + + +class BaseConfigManager(object): + def __init__(self, adapter_info={}, cluster_info={}, hosts_info={}): + assert(adapter_info and isinstance(adapter_info, dict)) + assert(cluster_info and isinstance(cluster_info, dict)) + assert(hosts_info and isinstance(hosts_info, dict)) + + self.adapter_info = AdapterInfo(adapter_info) + self.cluster_info = ClusterInfo(cluster_info) + self.hosts_info = dict([(k, HostInfo(v, self.cluster_info)) + for k, v in hosts_info.iteritems()]) + + def get_adapter_name(self): + return self.adapter_info.name + + def get_dist_system_name(self): + return self.adapter_info.dist_system_name + + def get_adapter_health_check_cmd(self): + return self.adapter_info.health_check_cmd + + def get_os_installer_settings(self): + return self.adapter_info.os_installer[const.INSTALLER_SETTINGS] + + def get_pk_installer_settings(self): + return self.adapter_info.package_installer[const.INSTALLER_SETTINGS] + + def get_os_config_metadata(self): + return self.adapter_info.metadata[const.OS_CONFIG] + + def get_pk_config_meatadata(self): + return self.adapter_info.metadata[const.PK_CONFIG] + + def get_adapter_all_flavors(self): + return self.adapter_info.flavor_list + + def get_adapter_flavor(self, flavor_name): + return self.adapter_info.get_flavor(flavor_name) + + def get_cluster_id(self): + return self.cluster_info.id + + def get_clustername(self): + return self.cluster_info.name + + def get_os_version(self): + return self.cluster_info.os_version + + def get_cluster_os_config(self): + return self.cluster_info.os_config + + def get_cluster_baseinfo(self): + return self.cluster_info.base_info + + def get_cluster_flavor_name(self): + return self.cluster_info.flavor.get(const.FLAVOR_NAME) + + def get_cluster_flavor_roles(self): + return self.cluster_info.flavor.get(const.ROLES, []) + + def get_cluster_flavor_template(self): + return self.cluster_info.flavor.get(const.TMPL) + + def get_cluster_package_config(self): + return self.cluster_info.package_config + + def get_cluster_network_mapping(self): + mapping = self.cluster_info.network_mapping + logging.info("Network mapping in the config is '%s'!", mapping) + return mapping + + def get_cluster_deployed_os_config(self): + return self.cluster_info.deployed_os_config + + def get_cluster_deployed_package_config(self): + return self.cluster_info.deployed_package_config + + def get_cluster_roles_mapping(self): + return self.cluster_info.roles_mapping + + def get_cluster_patched_roles_mapping(self): + return self.cluster_info._get_cluster_patched_roles_mapping() + + def validate_host(self, host_id): + if host_id not in self.hosts_info: + raise RuntimeError("host_id %s is invalid" % host_id) + + def get_host_id_list(self): + return self.hosts_info.keys() + + def get_hosts_id_list_for_os_installation(self): + """Get info of hosts which need to install/reinstall OS.""" + return [ + id for id, info in self.hosts_info.items() + if info.reinstall_os_flag + ] + + def get_server_credentials(self): + cluster_os_config = self.get_cluster_os_config() + if not cluster_os_config: + logging.info("cluster os_config is None!") + return () + + username = cluster_os_config[const.SERVER_CREDS][const.USERNAME] + password = cluster_os_config[const.SERVER_CREDS][const.PASSWORD] + return (username, password) + + def _get_host_info(self, host_id): + self.validate_host(host_id) + return self.hosts_info[host_id] + + def get_host_baseinfo(self, host_id): + self.validate_host(host_id) + host_info = self.hosts_info[host_id] + return host_info.baseinfo + + def get_host_fullname(self, host_id): + self.validate_host(host_id) + return self.hosts_info[host_id].name + + def get_host_dns(self, host_id): + self.validate_host(host_id) + return self.hosts_info[host_id].dns + + def get_host_mac_address(self, host_id): + self.validate_host(host_id) + return self.hosts_info[host_id].mac + + def get_hostname(self, host_id): + self.validate_host(host_id) + return self.hosts_info[host_id].hostname + + def get_host_networks(self, host_id): + self.validate_host(host_id) + return self.hosts_info[host_id].networks + + def get_host_interfaces(self, host_id): + # get interface names + return self.get_host_networks(host_id).keys() + + def get_host_interface_ip(self, host_id, interface): + self.validate_host(host_id) + return self.hosts_info[host_id].get_interface_ip(interface) + + def get_host_interface_netmask(self, host_id, interface): + self.validate_host(host_id) + return self.hosts_info[host_id].get_interface_netmask(interface) + + def get_host_interface_subnet(self, host_id, interface): + self.validate_host(host_id) + return self.hosts_info[host_id].get_interface_subnet(interface) + + def is_interface_promiscuous(self, host_id, interface): + self.validate_host(host_id) + return self.hosts_info[host_id].is_interface_promiscuous(interface) + + def is_interface_mgmt(self, host_id, interface): + self.validate_host(host_id) + return self.hosts_info[host_id].is_interface_mgmt(interface) + + def get_host_os_config(self, host_id): + self.validate_host(host_id) + return self.hosts_info[host_id].os_config + + def get_host_domain(self, host_id): + self.validate_host(host_id) + return self.hosts_info[host_id].domain + + def get_host_network_mapping(self, host_id): + self.validate_host(host_id) + return self.hosts_info[host_id].network_mapping + + def get_host_package_config(self, host_id): + self.validate_host(host_id) + return self.hosts_info[host_id].package_config + + def get_host_deployed_os_config(self, host_id): + self.validate_host(host_id) + return self.hosts_info[host_id].deployed_os_config + + def get_host_deployed_package_config(self, host_id): + self.validate_host(host_id) + return self.hosts_info[host_id].deployed_package_config + + def get_host_roles(self, host_id): + self.validate_host(host_id) + return self.hosts_info[host_id].roles + + def get_all_hosts_roles(self, hosts_id_list=None): + roles = [] + for host_id, host_info in self.hosts_info.iteritems(): + roles.extend(host_info.roles) + + return list(set(roles)) + + def get_hosts_ip_settings(self, ip_settings, sys_intf_mappings): + logging.info( + "get_hosts_ip_settings:ip_settings=%s, sys_intf_mappings=%s" % + (ip_settings, sys_intf_mappings) + ) + + intf_alias = {} + for m in sys_intf_mappings: + if "vlan_tag" in m: + intf_alias[m["name"]] = m["name"] + else: + intf_alias[m["name"]] = m["interface"] + + mappings = {} + hosts_id_list = self.get_host_id_list() + for host_id in hosts_id_list: + hostname = self.get_hostname(host_id) + mappings[hostname] = [] + for ip_info in ip_settings: + logging.info("ip_info=%s" % ip_info) + new_ip_info = deepcopy(ip_info) + del new_ip_info["ip_ranges"] + + ip_ranges = ip_info["ip_ranges"] + new_ip_info["netmask"] = netaddr.IPNetwork( + ip_info["cidr"] + ).netmask.bin.count("1") + new_ip_info["ip"] = get_ip_addr(ip_ranges).next() + new_ip_info["alias"] = intf_alias[ip_info["name"]] + mappings[hostname].append(new_ip_info) + + return {"ip_settings": mappings} + + def get_host_roles_mapping(self, host_id): + self.validate_host(host_id) + return self.hosts_info[host_id].roles_mapping + + def get_host_ipmi_info(self, host_id): + self.validate_host(host_id) + if self.hosts_info[host_id].ipmi: + return ( + self.hosts_info[host_id].ipmi[const.IP_ADDR], + self.hosts_info[host_id].ipmi + [const.IPMI_CREDS][const.USERNAME], + self.hosts_info[host_id].ipmi + [const.IPMI_CREDS][const.USERNAME]) + else: + return (None, None, None) diff --git a/compass-tasks-base/deployment/installers/installer.py b/compass-tasks-base/deployment/installers/installer.py new file mode 100644 index 0000000..cfeb9e8 --- /dev/null +++ b/compass-tasks-base/deployment/installers/installer.py @@ -0,0 +1,291 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" + + +"""Module to provider installer interface. +""" +from Cheetah.Template import Template +from copy import deepcopy +import imp +import logging +import os +import simplejson as json + +from compass.deployment.installers.config_manager import BaseConfigManager +from compass.utils import setting_wrapper as compass_setting +from compass.utils import util + + +CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) + + +class BaseInstaller(object): + """Interface for installer.""" + NAME = 'installer' + + def __repr__(self): + return '%r[%r]' % (self.__class__.__name__, self.NAME) + + def deploy(self, **kwargs): + """virtual method to start installing process.""" + raise NotImplementedError + + def clean_progress(self, **kwargs): + raise NotImplementedError + + def delete_hosts(self, **kwargs): + """Delete hosts from installer server.""" + raise NotImplementedError + + def redeploy(self, **kwargs): + raise NotImplementedError + + def ready(self, **kwargs): + pass + + def cluster_ready(self, **kwargs): + pass + + def get_tmpl_vars_from_metadata(self, metadata, config): + """Get variables dictionary for rendering templates from metadata. + + :param dict metadata: The metadata dictionary. + :param dict config: The + """ + template_vars = {} + self._get_tmpl_vars_helper(metadata, config, template_vars) + + return template_vars + + def _get_key_mapping(self, metadata, key, is_regular_key): + """Get the keyword which the input key maps to. + + This keyword will be added to dictionary used to render templates. + + If the key in metadata has a mapping to another keyword which is + used for templates, then return this keyword. If the key is started + with '$', which is a variable in metadata, return the key itself as + the mapping keyword. If the key has no mapping, return None. + + :param dict metadata: metadata/submetadata dictionary. + :param str key: The keyword defined in metadata. + :param bool is_regular_key: False when the key defined in metadata + is a variable(starting with '$'). + """ + mapping_to = key + if is_regular_key: + try: + mapping_to = metadata['_self']['mapping_to'] + except Exception: + mapping_to = None + + return mapping_to + + def _get_submeta_by_key(self, metadata, key): + """Get submetadata dictionary. + + Based on current metadata key. And + determines the input key is a regular string keyword or a variable + keyword defined in metadata, which starts with '$'. + + :param dict metadata: The metadata dictionary. + :param str key: The keyword defined in the metadata. + """ + if key in metadata: + return (True, metadata[key]) + + temp = deepcopy(metadata) + if '_self' in temp: + del temp['_self'] + meta_key = temp.keys()[0] + if meta_key.startswith("$"): + return (False, metadata[meta_key]) + + raise KeyError("'%s' is invalid in metadata '%s'!" % (key, metadata)) + + def _get_tmpl_vars_helper(self, metadata, config, output): + for key, config_value in sorted(config.iteritems()): + is_regular_key, sub_meta = self._get_submeta_by_key(metadata, key) + mapping_to = self._get_key_mapping(sub_meta, key, is_regular_key) + + if isinstance(config_value, dict): + if mapping_to: + new_output = output[mapping_to] = {} + else: + new_output = output + + self._get_tmpl_vars_helper(sub_meta, config_value, new_output) + + elif mapping_to: + output[mapping_to] = config_value + + def get_config_from_template(self, tmpl_path, vars_dict): + logging.debug("template path is %s", tmpl_path) + logging.debug("vars_dict is %s", vars_dict) + + if not os.path.exists(tmpl_path) or not vars_dict: + logging.info("Template dir or vars_dict is None!") + return {} + + searchList = [] + copy_vars_dict = deepcopy(vars_dict) + for key, value in vars_dict.iteritems(): + if isinstance(value, dict): + temp = copy_vars_dict[key] + del copy_vars_dict[key] + searchList.append(temp) + searchList.append(copy_vars_dict) + + # Load base template first if it exists + base_config = {} + base_tmpl_path = os.path.join(os.path.dirname(tmpl_path), 'base.tmpl') + if os.path.isfile(base_tmpl_path) and base_tmpl_path != tmpl_path: + base_tmpl = Template(file=base_tmpl_path, searchList=searchList) + base_config = json.loads(base_tmpl.respond(), encoding='utf-8') + base_config = json.loads(json.dumps(base_config), encoding='utf-8') + + # Load specific template for current adapter + tmpl = Template(file=open(tmpl_path, "r"), searchList=searchList) + config = json.loads(tmpl.respond(), encoding='utf-8') + config = json.loads(json.dumps(config), encoding='utf-8') + + # Merge the two outputs + config = util.merge_dict(base_config, config) + + logging.debug("get_config_from_template resulting %s", config) + return config + + @classmethod + def get_installer(cls, name, path, adapter_info, cluster_info, hosts_info): + try: + mod_file, path, descr = imp.find_module(name, [path]) + if mod_file: + mod = imp.load_module(name, mod_file, path, descr) + config_manager = BaseConfigManager(adapter_info, cluster_info, + hosts_info) + return getattr(mod, mod.NAME)(config_manager) + + except ImportError as exc: + logging.error('No such module found: %s', name) + logging.exception(exc) + + return None + + +class OSInstaller(BaseInstaller): + """Interface for os installer.""" + NAME = 'OSInstaller' + INSTALLER_BASE_DIR = os.path.join(CURRENT_DIR, 'os_installers') + + def get_oses(self): + """virtual method to get supported oses. + + :returns: list of str, each is the supported os version. + """ + return [] + + @classmethod + def get_installer(cls, name, adapter_info, cluster_info, hosts_info): + if name is None: + logging.info("Installer name is None! No OS installer loaded!") + return None + + path = os.path.join(cls.INSTALLER_BASE_DIR, name) + installer = super(OSInstaller, cls).get_installer(name, path, + adapter_info, + cluster_info, + hosts_info) + + if not isinstance(installer, OSInstaller): + logging.info("Installer '%s' is not an OS installer!" % name) + return None + + return installer + + def poweron(self, host_id): + pass + + def poweroff(self, host_id): + pass + + def reset(self, host_id): + pass + + +class PKInstaller(BaseInstaller): + """Interface for package installer.""" + NAME = 'PKInstaller' + INSTALLER_BASE_DIR = os.path.join(CURRENT_DIR, 'pk_installers') + + def generate_installer_config(self): + raise NotImplementedError( + 'generate_installer_config is not defined in %s', + self.__class__.__name__ + ) + + def get_target_systems(self): + """virtual method to get available target_systems for each os. + + :param oses: supported os versions. + :type oses: list of st + + :returns: dict of os_version to target systems as list of str. + """ + return {} + + def get_roles(self, target_system): + """virtual method to get all roles of given target system. + + :param target_system: target distributed system such as OpenStack. + :type target_system: str + + :returns: dict of role to role description as str. + """ + return {} + + def os_ready(self, **kwargs): + pass + + def cluster_os_ready(self, **kwargs): + pass + + def serialize_config(self, config, destination): + with open(destination, "w") as f: + f.write(config) + + @classmethod + def get_installer(cls, name, adapter_info, cluster_info, hosts_info): + if name is None: + logging.info("Install name is None. No package installer loaded!") + return None + + path = os.path.join(cls.INSTALLER_BASE_DIR, name) + if not os.path.exists(path): + path = os.path.join(os.path.join(os.path.join( + compass_setting.PLUGINS_DIR, name), "implementation"), name) + if not os.path.exists(path): + logging.info("Installer '%s' does not exist!" % name) + return None + installer = super(PKInstaller, cls).get_installer(name, path, + adapter_info, + cluster_info, + hosts_info) + + if not isinstance(installer, PKInstaller): + logging.info("Installer '%s' is not a package installer!" % name) + return None + + return installer diff --git a/compass-tasks-base/deployment/installers/os_installers/__init__.py b/compass-tasks-base/deployment/installers/os_installers/__init__.py new file mode 100644 index 0000000..5e42ae9 --- /dev/null +++ b/compass-tasks-base/deployment/installers/os_installers/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/deployment/installers/os_installers/cobbler/__init__.py b/compass-tasks-base/deployment/installers/os_installers/cobbler/__init__.py new file mode 100644 index 0000000..5e42ae9 --- /dev/null +++ b/compass-tasks-base/deployment/installers/os_installers/cobbler/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/deployment/installers/os_installers/cobbler/cobbler.py b/compass-tasks-base/deployment/installers/os_installers/cobbler/cobbler.py new file mode 100644 index 0000000..9c2a935 --- /dev/null +++ b/compass-tasks-base/deployment/installers/os_installers/cobbler/cobbler.py @@ -0,0 +1,449 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""os installer cobbler plugin. +""" +import logging +import os +import shutil +import xmlrpclib + +from compass.deployment.installers.installer import OSInstaller +from compass.deployment.utils import constants as const +from compass.utils import setting_wrapper as compass_setting +from compass.utils import util +from copy import deepcopy + + +NAME = 'CobblerInstaller' + + +class CobblerInstaller(OSInstaller): + """cobbler installer""" + CREDENTIALS = "credentials" + USERNAME = 'username' + PASSWORD = 'password' + + INSTALLER_URL = "cobbler_url" + TMPL_DIR = 'tmpl_dir' + SYS_TMPL = 'system.tmpl' + SYS_TMPL_NAME = 'system.tmpl' + SYS_PROFILE_NAME = 'profile.tmpl' + PROFILE = 'profile' + + POWER_TYPE = 'power_type' + POWER_ADDR = 'power_address' + POWER_USER = 'power_user' + POWER_PASS = 'power_pass' + + def __init__(self, config_manager): + super(CobblerInstaller, self).__init__() + + self.config_manager = config_manager + installer_settings = self.config_manager.get_os_installer_settings() + try: + username = installer_settings[self.CREDENTIALS][self.USERNAME] + password = installer_settings[self.CREDENTIALS][self.PASSWORD] + cobbler_url = installer_settings[self.INSTALLER_URL] + self.tmpl_dir = CobblerInstaller.get_tmpl_path() + + except KeyError as ex: + raise KeyError(ex.message) + + # The connection is created when cobbler installer is initialized. + self.remote = self._get_cobbler_server(cobbler_url) + self.token = self._get_token(username, password) + self.pk_installer_config = None + + logging.debug('%s instance created', 'CobblerInstaller') + + @classmethod + def get_tmpl_path(cls): + return os.path.join(compass_setting.TMPL_DIR, 'cobbler') + + def __repr__(self): + return '%r[remote=%r,token=%r' % ( + self.__class__.__name__, self.remote, self.token) + + def _get_cobbler_server(self, cobbler_url): + if not cobbler_url: + logging.error("Cobbler URL is None!") + raise Exception("Cobbler URL cannot be None!") + + return xmlrpclib.Server(cobbler_url) + + def _get_token(self, username, password): + if self.remote is None: + raise Exception("Cobbler remote instance is None!") + return self.remote.login(username, password) + + def get_supported_oses(self): + """get supported os versions. + + note:: + In cobbler, we treat profile name as the indicator + of os version. It is just a simple indicator + and not accurate. + """ + profiles = self.remote.get_profiles() + oses = [] + for profile in profiles: + oses.append(profile['name']) + return oses + + def deploy(self): + """Sync cobbler to catch up the latest update config and start to + + install OS. Return both cluster and hosts deploy configs. The return + format: + { + "cluster": { + "id": 1, + "deployed_os_config": {}, + }, + "hosts": { + 1($clusterhost_id): { + "deployed_os_config": {...}, + }, + .... + } + } + """ + host_ids = self.config_manager.get_hosts_id_list_for_os_installation() + if not host_ids: + # No hosts need to install OS + logging.info("Cobbler: No host needs to install OS.") + return {} + + os_version = self.config_manager.get_os_version() + profile = self._get_profile_from_server(os_version) + + global_vars_dict = self._get_cluster_tmpl_vars_dict() + + self.update_profile_config_to_cobbler(profile, global_vars_dict) + + hosts_deploy_config = {} + + for host_id in host_ids: + hostname = self.config_manager.get_hostname(host_id) + vars_dict = self._get_host_tmpl_vars_dict(host_id, + global_vars_dict, + hostname=hostname, + profile=profile) + + self.update_host_config_to_cobbler(host_id, hostname, vars_dict) + + # set host deploy config + host_config = {} + host_config[const.DEPLOYED_OS_CONFIG] = vars_dict[const.OS_CONFIG] + hosts_deploy_config[host_id] = host_config + + # sync to cobbler and trigger installtion. + self._sync() + + cluster_config = global_vars_dict.setdefault(const.OS_CONFIG, {}) + + return { + const.CLUSTER: { + const.ID: self.config_manager.get_cluster_id(), + const.DEPLOYED_OS_CONFIG: cluster_config + }, + const.HOSTS: hosts_deploy_config + } + + def clean_progress(self): + """clean log files and config for hosts which to deploy.""" + clusterhost_list = self.config_manager.get_host_id_list() + log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[NAME] + + for host_id in clusterhost_list: + hostname = self.config_manager.get_hostname(host_id) + self._clean_log(log_dir_prefix, hostname) + + def redeploy(self): + """redeploy hosts.""" + host_ids = self.config_manager.get_host_id_list() + if not host_ids: + logging.info("Cobbler: hostlist is None, no host is redeployed") + return + for host_id in host_ids: + hostname = self.config_manager.get_hostname(host_id) + sys_id = self._get_create_system(hostname) + if sys_id: + # enable netboot for this host + self._netboot_enabled(sys_id) + + self._sync() + + def set_package_installer_config(self, package_configs): + """Cobbler can install and configure package installer right after + + OS installation compelets by setting package_config info provided + by package installer. + + :param dict package_configs: The dict of config generated by package + installer for each clusterhost. The IDs + of clusterhosts are the keys of + package_configs. + """ + self.pk_installer_config = package_configs + + def _sync(self): + """Sync the updated config to cobbler and trigger installation.""" + try: + self.remote.sync(self.token) + os.system('sudo service rsyslog restart') + except Exception as ex: + logging.debug("Failed to sync cobbler server! Error: %s" % ex) + raise ex + + def dump_system_info(self, host_id): + + hostname = self.config_manager.get_hostname(host_id) + if self.remote is None or not hostname: + logging.info("[dump_system_info]Remote or hostname is None.") + return {} + + return self.remote.get_system_as_rendered(hostname) + + def _generate_system_config(self, host_id, host_vars_dict): + """Generate updated system config from the template. + + :param host_vars_dict: dict of variables for the system template to + generate system config dict for each host. + """ + os_version = self.config_manager.get_os_version() + + tmpl_path = os.path.join( + os.path.join(self.tmpl_dir, os_version), self.SYS_TMPL_NAME + ) + if not os.path.exists(tmpl_path): + err_msg = "Template '%s' does not exists!" % tmpl_path + logging.error(err_msg) + raise Exception(err_msg) + host_vars_dict[const.BASEINFO]['host_id'] = host_id + system_config = self.get_config_from_template(tmpl_path, + host_vars_dict) + + # update package config info to cobbler ksmeta + if self.pk_installer_config and host_id in self.pk_installer_config: + pk_config = self.pk_installer_config[host_id] + ksmeta = system_config.setdefault("ksmeta", {}) + util.merge_dict(ksmeta, pk_config) + system_config["ksmeta"] = ksmeta + + return system_config + + def _generate_profile_config(self, cluster_vars_dict): + os_version = self.config_manager.get_os_version() + tmpl_path = os.path.join( + os.path.join(self.tmpl_dir, os_version), self.SYS_PROFILE_NAME + ) + + return self.get_config_from_template(tmpl_path, cluster_vars_dict) + + def _get_profile_from_server(self, os_version): + """Get profile from cobbler server.""" + result = self.remote.find_profile({'name': os_version}) + if not result: + raise Exception("Cannot find profile for '%s'", os_version) + + profile = result[0] + return profile + + def _get_create_system(self, hostname): + """get system reference id for the host.""" + sys_name = hostname + sys_id = None + system_info = self.remote.find_system({"name": hostname}) + + if not system_info: + # Create a new system + sys_id = self.remote.new_system(self.token) + self.remote.modify_system(sys_id, "name", hostname, self.token) + logging.debug('create new system %s for %s', sys_id, sys_name) + else: + sys_id = self.remote.get_system_handle(sys_name, self.token) + + return sys_id + + def _get_profile_id(self, profilename): + """get profile reference id for the cluster.""" + return self.remote.get_profile_handle(profilename, self.token) + + def _clean_system(self, hostname): + """clean system.""" + sys_name = hostname + try: + self.remote.remove_system(sys_name, self.token) + logging.debug('system %s is removed', sys_name) + except Exception: + logging.debug('no system %s found to remove', sys_name) + + def _update_system_config(self, sys_id, system_config): + """update modify system.""" + for key, value in system_config.iteritems(): + self.remote.modify_system(sys_id, str(key), value, self.token) + + self.remote.save_system(sys_id, self.token) + + def _update_profile_config(self, profile_id, profile_config): + for key, value in profile_config.iteritems(): + self.remote.modify_profile(profile_id, str(key), value, self.token) + + self.remote.save_profile(profile_id, self.token) + + def _netboot_enabled(self, sys_id): + """enable netboot.""" + self.remote.modify_system(sys_id, 'netboot_enabled', True, self.token) + self.remote.save_system(sys_id, self.token) + + def _clean_log(self, log_dir_prefix, system_name): + """clean log.""" + log_dir = os.path.join(log_dir_prefix, system_name) + shutil.rmtree(log_dir, True) + + def update_host_config_to_cobbler(self, host_id, hostname, host_vars_dict): + """update host config and upload to cobbler server.""" + sys_id = self._get_create_system(hostname) + + system_config = self._generate_system_config(host_id, host_vars_dict) + logging.debug('%s system config to update: %s', host_id, system_config) + + self._update_system_config(sys_id, system_config) + self._netboot_enabled(sys_id) + + def update_profile_config_to_cobbler(self, profilename, cluster_vars_dict): + """update profile config and upload to cobbler server.""" + + profile_id = self._get_profile_id(profilename) + + profile_config = self._generate_profile_config(cluster_vars_dict) + logging.debug( + '%s profile config to update: %s', profilename, profile_config + ) + + self._update_profile_config(profile_id, profile_config) + + def delete_hosts(self): + hosts_id_list = self.config_manager.get_host_id_list() + logging.debug('delete hosts %s', hosts_id_list) + for host_id in hosts_id_list: + self.delete_single_host(host_id) + self._sync() + + def delete_single_host(self, host_id): + """Delete the host from cobbler server and clean up the installation + + progress. + """ + hostname = self.config_manager.get_hostname(host_id) + try: + log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[NAME] + self._clean_system(hostname) + self._clean_log(log_dir_prefix, hostname) + except Exception as ex: + logging.error("Deleting host got exception: %s", ex) + logging.exception(ex) + + def _get_host_tmpl_vars_dict(self, host_id, global_vars_dict, **kwargs): + """Generate template variables dictionary.""" + vars_dict = {} + if global_vars_dict: + # Set cluster template vars_dict from cluster os_config. + vars_dict = deepcopy(global_vars_dict) + + # Set hostname, MAC address and hostname, networks, dns and so on. + host_baseinfo = self.config_manager.get_host_baseinfo(host_id) + vars_dict[const.BASEINFO] = host_baseinfo + + # Set profile + if self.PROFILE in kwargs: + profile = kwargs[self.PROFILE] + else: + os_version = self.config_manager.get_os_version() + profile = self._get_profile_from_server(os_version) + + vars_dict[const.BASEINFO][self.PROFILE] = profile + + metadata = self.config_manager.get_os_config_metadata() + os_config = self.config_manager.get_host_os_config(host_id) + + # Get template variables values from host os_config + host_vars_dict = self.get_tmpl_vars_from_metadata(metadata, os_config) + util.merge_dict( + vars_dict.setdefault(const.OS_CONFIG, {}), host_vars_dict + ) + return vars_dict + + def _get_cluster_tmpl_vars_dict(self): + metadata = self.config_manager.get_os_config_metadata() + os_config = self.config_manager.get_cluster_os_config() + + cluster_vas_dict = {} + cluster_vas_dict[const.OS_CONFIG] = \ + self.get_tmpl_vars_from_metadata(metadata, os_config) + + return cluster_vas_dict + + def _check_and_set_system_impi(self, host_id, sys_id): + if not sys_id: + logging.info("System is None!") + return False + + system = self.dump_system_info(host_id) + if system[self.POWER_TYPE] != 'ipmilan' or not system[self.POWER_USER]: + # Set sytem power type to ipmilan if needs and set IPMI info + ipmi_info = self.config_manager.get_host_ipmi_info(host_id) + if not ipmi_info: + logging.info('No IPMI information found! Failed power on.') + return False + + ipmi_ip, ipmi_user, ipmi_pass = ipmi_info + power_opts = {} + power_opts[self.POWER_TYPE] = 'ipmilan' + power_opts[self.POWER_ADDR] = ipmi_ip + power_opts[self.POWER_USER] = ipmi_user + power_opts[self.POWER_PASS] = ipmi_pass + + self._update_system_config(sys_id, power_opts) + + return True + + def poweron(self, host_id): + hostname = self.config_manager.get_hostname(host_id) + sys_id = self._get_create_system(hostname) + if not self._check_and_set_system_impi(sys_id): + return + + self.remote.power_system(sys_id, self.token, power='on') + logging.info("Host with ID=%d starts to power on!" % host_id) + + def poweroff(self, host_id): + hostname = self.config_manager.get_hostname(host_id) + sys_id = self._get_create_system(hostname) + if not self._check_and_set_system_impi(sys_id): + return + + self.remote.power_system(sys_id, self.token, power='off') + logging.info("Host with ID=%d starts to power off!" % host_id) + + def reset(self, host_id): + hostname = self.config_manager.get_hostname(host_id) + sys_id = self._get_create_system(hostname) + if not self._check_and_set_system_impi(sys_id): + return + + self.remote.power_system(sys_id, self.token, power='reboot') + logging.info("Host with ID=%d starts to reboot!" % host_id) diff --git a/compass-tasks-base/deployment/installers/pk_installers/__init__.py b/compass-tasks-base/deployment/installers/pk_installers/__init__.py new file mode 100644 index 0000000..5e42ae9 --- /dev/null +++ b/compass-tasks-base/deployment/installers/pk_installers/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/deployment/installers/pk_installers/ansible_installer/__init__.py b/compass-tasks-base/deployment/installers/pk_installers/ansible_installer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/compass-tasks-base/deployment/installers/pk_installers/ansible_installer/ansible_installer.py b/compass-tasks-base/deployment/installers/pk_installers/ansible_installer/ansible_installer.py new file mode 100644 index 0000000..0a86be4 --- /dev/null +++ b/compass-tasks-base/deployment/installers/pk_installers/ansible_installer/ansible_installer.py @@ -0,0 +1,441 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__auther__ = "Compass Dev Team (dev-team@syscompass.org)" + +"""package installer: ansible plugin.""" + +from Cheetah.Template import Template +from copy import deepcopy +import json +import logging +import os +import re +import shutil +import subprocess + +from compass.deployment.installers.installer import PKInstaller +from compass.deployment.utils import constants as const +from compass.utils import setting_wrapper as compass_setting +from compass.utils import util + +NAME = "AnsibleInstaller" + + +def byteify(input): + if isinstance(input, dict): + return dict([(byteify(key), byteify(value)) + for key, value in input.iteritems()]) + elif isinstance(input, list): + return [byteify(element) for element in input] + elif isinstance(input, unicode): + return input.encode('utf-8') + else: + return input + + +class AnsibleInstaller(PKInstaller): + INVENTORY_TMPL_DIR = 'inventories' + GROUPVARS_TMPL_DIR = 'vars' + INVENTORY_PATCH_TEMPALTE_DIR = 'inventories' + + # keywords in package installer settings + ANSIBLE_DIR = 'ansible_dir' + ANSIBLE_RUN_DIR = 'ansible_run_dir' + LOG_FILE = 'ansible_log_file' + ANSIBLE_CONFIG = 'ansible_config' + INVENTORY = 'inventory_file' + INVENTORY_JSON = 'inventory_json_file' + INVENTORY_GROUP = 'inventory_group' + GROUP_VARIABLE = 'group_variable' + HOSTS_PATH = 'etc_hosts_path' + RUNNER_DIRS = 'runner_dirs' + + def __init__(self, config_manager): + super(AnsibleInstaller, self).__init__() + + self.config_manager = config_manager + self.tmpl_name = self.config_manager.get_cluster_flavor_template() + self.installer_settings = ( + self.config_manager.get_pk_installer_settings() + ) + settings = self.installer_settings + self.ansible_dir = settings.setdefault(self.ANSIBLE_DIR, None) + self.ansible_run_dir = ( + settings.setdefault(self.ANSIBLE_RUN_DIR, None) + ) + self.log_file = settings.setdefault(self.LOG_FILE, None) + self.ansible_config = ( + settings.setdefault(self.ANSIBLE_CONFIG, None) + ) + self.inventory = settings.setdefault(self.INVENTORY, None) + self.inventory_json = settings.setdefault(self.INVENTORY_JSON, None) + self.inventory_group = settings.setdefault(self.INVENTORY_GROUP, None) + self.group_variable = ( + settings.setdefault(self.GROUP_VARIABLE, None) + ) + self.hosts_path = ( + settings.setdefault(self.HOSTS_PATH, None) + ) + self.runner_dirs = ( + settings.setdefault(self.RUNNER_DIRS, None) + ) + self.playbook = self.tmpl_name.replace('tmpl', 'yml') + self.runner_files = [self.playbook] + + adapter_name = self.config_manager.get_dist_system_name() + self.tmpl_dir = AnsibleInstaller.get_tmpl_path(adapter_name) + self.adapter_dir = os.path.join(self.ansible_dir, adapter_name) + logging.debug('%s instance created', self) + + @classmethod + def get_tmpl_path(cls, adapter_name): + tmpl_path = os.path.join( + os.path.join(compass_setting.TMPL_DIR, 'ansible_installer'), + adapter_name + ) + return tmpl_path + + def __repr__(self): + return '%s[name=%s,installer_url=%s]' % ( + self.__class__.__name__, self.NAME, self.installer_url) + + def dump_inventory(self, data, inventory): + with open(inventory, "w") as f: + json.dump(data, f, indent=4) + + def _generate_inventory_data(self, global_vars_dict): + vars_dict = global_vars_dict['roles_mapping'] + inventory_data = {} + inventory_data['_meta'] = {'hostvars': {}} + for item in self.inventory_group: + if item in vars_dict: + inventory_data[item] = {'hosts': []} + for host in vars_dict[item]: + hostname = host['hostname'] + if hostname not in inventory_data['_meta']['hostvars']: + host_dict = {} + host_dict['ansible_ssh_host'] = host['install']['ip'] + host_dict['ansible_ssh_user'] = 'root' + host_dict['ansible_ssh_pass'] = 'root' + inventory_data['_meta']['hostvars'].update( + {hostname: host_dict}) + inventory_data[item]['hosts'].append(hostname) + + inventory_data['ceph'] = {'children': + ['ceph_adm', 'ceph_mon', 'ceph_osd']} + return inventory_data + + def generate_installer_config(self): + """Render ansible config file by OS installing. + + The output format: + { + '1'($host_id/clusterhost_id):{ + 'tool': 'ansible', + }, + ..... + } + """ + host_ids = self.config_manager.get_host_id_list() + os_installer_configs = {} + for host_id in host_ids: + temp = { + "tool": "ansible", + } + os_installer_configs[host_id] = temp + + return os_installer_configs + + def get_env_name(self, dist_sys_name, cluster_name): + return "-".join((dist_sys_name, cluster_name)) + + def _get_cluster_tmpl_vars(self): + """Generate template variables dict + + Generates based on cluster level config. + The vars_dict will be: + { + "baseinfo": { + "id":1, + "name": "cluster01", + ... + }, + "package_config": { + .... //mapped from original package config based on metadata + }, + "role_mapping": { + .... + } + } + """ + cluster_vars_dict = {} + # set cluster basic information to vars_dict + cluster_baseinfo = self.config_manager.get_cluster_baseinfo() + cluster_vars_dict[const.BASEINFO] = cluster_baseinfo + + # get and set template variables from cluster package config. + pk_metadata = self.config_manager.get_pk_config_meatadata() + pk_config = self.config_manager.get_cluster_package_config() + + # get os config as ansible needs them + os_metadata = self.config_manager.get_os_config_metadata() + os_config = self.config_manager.get_cluster_os_config() + + pk_meta_dict = self.get_tmpl_vars_from_metadata(pk_metadata, pk_config) + os_meta_dict = self.get_tmpl_vars_from_metadata(os_metadata, os_config) + util.merge_dict(pk_meta_dict, os_meta_dict) + + cluster_vars_dict[const.PK_CONFIG] = pk_meta_dict + + # get and set roles_mapping to vars_dict + mapping = self.config_manager.get_cluster_roles_mapping() + logging.info("cluster role mapping is %s", mapping) + cluster_vars_dict[const.ROLES_MAPPING] = mapping + + # get ip settings to vars_dict + hosts_ip_settings = self.config_manager.get_hosts_ip_settings( + pk_meta_dict["network_cfg"]["ip_settings"], + pk_meta_dict["network_cfg"]["sys_intf_mappings"] + ) + logging.info("hosts_ip_settings is %s", hosts_ip_settings) + cluster_vars_dict["ip_settings"] = hosts_ip_settings + + return byteify(cluster_vars_dict) + + def _generate_inventory_attributes(self, global_vars_dict): + inventory_tmpl_path = os.path.join( + os.path.join(self.tmpl_dir, self.INVENTORY_TMPL_DIR), + self.tmpl_name + ) + if not os.path.exists(inventory_tmpl_path): + logging.error( + "Inventory template '%s' does not exist", self.tmpl_name + ) + raise Exception("Template '%s' does not exist!" % self.tmpl_name) + inventory_dir = os.path.join(global_vars_dict['run_dir'], 'inventories') + inventory_json = os.path.join(inventory_dir, self.inventory_json) + vars_dict = {'inventory_json': inventory_json} + return self.get_config_from_template( + inventory_tmpl_path, vars_dict + ) + + def _generate_group_vars_attributes(self, global_vars_dict): + logging.info("global vars dict is %s", global_vars_dict) + group_vars_tmpl_path = os.path.join( + os.path.join(self.tmpl_dir, self.GROUPVARS_TMPL_DIR), + self.tmpl_name + ) + if not os.path.exists(group_vars_tmpl_path): + logging.error("Vars template '%s' does not exist", + self.tmpl_name) + raise Exception("Template '%s' does not exist!" % self.tmpl_name) + + return self.get_config_from_template( + group_vars_tmpl_path, global_vars_dict + ) + + def _generate_hosts_attributes(self, global_vars_dict): + hosts_tmpl_path = os.path.join( + os.path.join(self.tmpl_dir, 'hosts'), self.tmpl_name + ) + if not os.path.exists(hosts_tmpl_path): + logging.error("Hosts template '%s' does not exist", self.tmpl_name) + raise Exception("Template '%s' does not exist!" % self.tmpl_name) + + return self.get_config_from_template(hosts_tmpl_path, global_vars_dict) + + def _generate_ansible_cfg_attributes(self, global_vars_dict): + ansible_cfg_tmpl_path = os.path.join( + os.path.join(self.tmpl_dir, 'ansible_cfg'), self.tmpl_name + ) + if not os.path.exists(ansible_cfg_tmpl_path): + logging.error("cfg template '%s' does not exist", self.tmpl_name) + raise Exception("Template '%s' does not exist!" % self.tmpl_name) + + return self.get_config_from_template( + ansible_cfg_tmpl_path, + global_vars_dict + ) + + def get_config_from_template(self, tmpl_path, vars_dict): + logging.debug("vars_dict is %s", vars_dict) + + if not os.path.exists(tmpl_path) or not vars_dict: + logging.info("Template dir or vars_dict is None!") + return {} + + searchList = [] + copy_vars_dict = deepcopy(vars_dict) + for key, value in vars_dict.iteritems(): + if isinstance(value, dict): + temp = copy_vars_dict[key] + del copy_vars_dict[key] + searchList.append(temp) + searchList.append(copy_vars_dict) + + # Load specific template for current adapter + tmpl = Template(file=open(tmpl_path, "r"), searchList=searchList) + return tmpl.respond() + + def _create_ansible_run_env(self, env_name, ansible_run_destination): + if os.path.exists(ansible_run_destination): + shutil.rmtree(ansible_run_destination, True) + + os.mkdir(ansible_run_destination) + + # copy roles to run env + dirs = self.runner_dirs + files = self.runner_files + for dir in dirs: + if not os.path.exists(os.path.join(self.ansible_dir, dir)): + continue + os.system( + "cp -rf %s %s" % ( + os.path.join(self.ansible_dir, dir), + ansible_run_destination + ) + ) + for file in files: + logging.info('file is %s', file) + shutil.copy( + os.path.join(self.adapter_dir, file), + os.path.join( + ansible_run_destination, + file + ) + ) + + def prepare_ansible(self, env_name, global_vars_dict): + ansible_run_destination = os.path.join(self.ansible_run_dir, env_name) + if os.path.exists(ansible_run_destination): + ansible_run_destination += "-expansion" + self._create_ansible_run_env(env_name, ansible_run_destination) + global_vars_dict.update({'run_dir': ansible_run_destination}) + + inv_config = self._generate_inventory_attributes(global_vars_dict) + inventory_dir = os.path.join(ansible_run_destination, 'inventories') + + vars_config = self._generate_group_vars_attributes(global_vars_dict) + vars_dir = os.path.join(ansible_run_destination, 'group_vars') + + hosts_config = self._generate_hosts_attributes(global_vars_dict) + hosts_destination = os.path.join( + ansible_run_destination, self.hosts_path + ) + + cfg_config = self._generate_ansible_cfg_attributes(global_vars_dict) + cfg_destination = os.path.join( + ansible_run_destination, + self.ansible_config + ) + + inventory_data = self._generate_inventory_data(global_vars_dict) + inventory_json_destination = os.path.join(inventory_dir, + self.inventory_json) + + os.mkdir(inventory_dir) + os.mkdir(vars_dir) + + inventory_destination = os.path.join(inventory_dir, self.inventory) + group_vars_destination = os.path.join(vars_dir, self.group_variable) + self.dump_inventory(inventory_data, inventory_json_destination) + self.serialize_config(inv_config, inventory_destination) + self.serialize_config(vars_config, group_vars_destination) + self.serialize_config(hosts_config, hosts_destination) + self.serialize_config(cfg_config, cfg_destination) + + def deploy(self): + """Start to deploy a distributed system. + + Return both cluster and hosts deployed configs. + The return format: + { + "cluster": { + "id": 1, + "deployed_package_config": { + "roles_mapping": {...}, + "service_credentials": {...}, + .... + } + }, + "hosts": { + 1($clusterhost_id): { + "deployed_package_config": {...} + }, + .... + } + } + """ + host_list = self.config_manager.get_host_id_list() + if not host_list: + return {} + + adapter_name = self.config_manager.get_adapter_name() + cluster_name = self.config_manager.get_clustername() + env_name = self.get_env_name(adapter_name, cluster_name) + + global_vars_dict = self._get_cluster_tmpl_vars() + logging.info( + '%s var dict: %s', self.__class__.__name__, global_vars_dict + ) + # Create ansible related files + self.prepare_ansible(env_name, global_vars_dict) + + def patch(self, patched_role_mapping): + adapter_name = self.config_manager.get_adapter_name() + cluster_name = self.config_manager.get_clustername() + env_name = self.get_env_name(adapter_name, cluster_name) + ansible_run_destination = os.path.join(self.ansible_run_dir, env_name) + inventory_dir = os.path.join(ansible_run_destination, 'inventories') + patched_global_vars_dict = self._get_cluster_tmpl_vars() + mapping = self.config_manager.get_cluster_patched_roles_mapping() + patched_global_vars_dict['roles_mapping'] = mapping + patched_inv = self._generate_inventory_attributes( + patched_global_vars_dict) + inv_file = os.path.join(inventory_dir, 'patched_inventory.yml') + self.serialize_config(patched_inv, inv_file) + config_file = os.path.join( + ansible_run_destination, self.ansible_config + ) + playbook_file = os.path.join(ansible_run_destination, self.playbook) + log_file = os.path.join(ansible_run_destination, 'patch.log') + cmd = "ANSIBLE_CONFIG=%s ansible-playbook -i %s %s" % (config_file, + inv_file, + playbook_file) + with open(log_file, 'w') as logfile: + subprocess.Popen(cmd, shell=True, stdout=logfile, stderr=logfile) + return patched_role_mapping + + def cluster_os_ready(self): + adapter_name = self.config_manager.get_adapter_name() + cluster_name = self.config_manager.get_clustername() + env_name = self.get_env_name(adapter_name, cluster_name) + ansible_run_destination = os.path.join(self.ansible_run_dir, env_name) + expansion_dir = ansible_run_destination + "-expansion" + if os.path.exists(expansion_dir): + ansible_run_destination = expansion_dir + inventory_dir = os.path.join(ansible_run_destination, 'inventories') + inventory_file = os.path.join(inventory_dir, self.inventory) + playbook_file = os.path.join(ansible_run_destination, self.playbook) + log_file = os.path.join(ansible_run_destination, 'run.log') + config_file = os.path.join( + ansible_run_destination, self.ansible_config + ) + os.system("chmod +x %s" % inventory_file) + cmd = "ANSIBLE_CONFIG=%s ansible-playbook -i %s %s" % (config_file, + inventory_file, + playbook_file) + with open(log_file, 'w') as logfile: + subprocess.Popen(cmd, shell=True, stdout=logfile, stderr=logfile) diff --git a/compass-tasks-base/deployment/utils/__init__.py b/compass-tasks-base/deployment/utils/__init__.py new file mode 100644 index 0000000..cbd36e0 --- /dev/null +++ b/compass-tasks-base/deployment/utils/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" diff --git a/compass-tasks-base/deployment/utils/constants.py b/compass-tasks-base/deployment/utils/constants.py new file mode 100644 index 0000000..e90b1b2 --- /dev/null +++ b/compass-tasks-base/deployment/utils/constants.py @@ -0,0 +1,84 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" + + +"""All keywords variables in deployment are defined in this module.""" + + +# General keywords +BASEINFO = 'baseinfo' +CLUSTER = 'cluster' +HOST = 'host' +HOSTS = 'hosts' +ID = 'id' +NAME = 'name' +PASSWORD = 'password' +USERNAME = 'username' + + +# Adapter info related keywords +FLAVOR = 'flavor' +FLAVORS = 'flavors' +PLAYBOOK = 'playbook' +FLAVOR_NAME = 'flavor_name' +HEALTH_CHECK_CMD = 'health_check_cmd' +TMPL = 'template' +INSTALLER_SETTINGS = 'settings' +METADATA = 'metadata' +OS_INSTALLER = 'os_installer' +PK_INSTALLER = 'package_installer' +SUPPORT_OSES = 'supported_oses' + + +# Cluster info related keywords +ADAPTER_ID = 'adapter_id' +OS_VERSION = 'os_name' + + +# Host info related keywords +DNS = 'dns' +DOMAIN = 'domain' +HOST_ID = 'host_id' +HOSTNAME = 'hostname' +IP_ADDR = 'ip' +IPMI = 'ipmi' +IPMI_CREDS = 'ipmi_credentials' +MAC_ADDR = 'mac' +MGMT_NIC_FLAG = 'is_mgmt' +NETMASK = 'netmask' +NETWORKS = 'networks' +NIC = 'interface' +CLUSTER_ID = 'cluster_id' +ORIGIN_CLUSTER_ID = 'origin_cluster_id' +PROMISCUOUS_FLAG = 'is_promiscuous' +REINSTALL_OS_FLAG = 'reinstall_os' +SUBNET = 'subnet' + + +# Cluster/host config related keywords +COMPLETED_PK_CONFIG = 'completed_package_config' +COMPLETED_OS_CONFIG = 'completed_os_config' +DEPLOYED_OS_CONFIG = 'deployed_os_config' +DEPLOYED_PK_CONFIG = 'deployed_package_config' +NETWORK_MAPPING = 'network_mapping' +OS_CONFIG = 'os_config' +OS_CONFIG_GENERAL = 'general' +PK_CONFIG = 'package_config' +ROLES = 'roles' +PATCHED_ROLES = 'patched_roles' +ROLES_MAPPING = 'roles_mapping' +SERVER_CREDS = 'server_credentials' +TMPL_VARS_DICT = 'vars_dict' diff --git a/compass-tasks-base/hdsdiscovery/SNMP_CONFIG.md b/compass-tasks-base/hdsdiscovery/SNMP_CONFIG.md new file mode 100644 index 0000000..337b542 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/SNMP_CONFIG.md @@ -0,0 +1,33 @@ +Install & Config Prerequisite Packages: + +1. Net-Snmp: + a. #apt-get install -y snmpd snmp libsnmp-python + b. #apt-get install -y snmp-mibs-downloader + For Centos: + # yum install net-snmp net-snmp-utils + + c. create vendor's mibs directory(for example): + - #mkdir -p /root/.snmp/mibs/huawei + - #vim /etc/snmp/snmp.conf (if not exists, create snmp.conf file) + * add vendor;s mibs directory: + mibdirs +/root/.snmp/mibs/huawei + * comment the line: + #mibs: + d. copy vendor's mibs to that directory + e. #vim /etc/default/snmpd + * modify the directive from + TRAPDRUN=no --> TRAPDRUN=yes + For Centos: + # vim /etc/sysconfig/snmpd + * modify into or add the directive + TRAPDRUN=yes + + f. #vim /etc/snmp/snmpd.conf + * add the following line, where $ip is the ip address of manager machine: + com2sec mynetwork $ip/24 public + g. #service snmpd restart + + Note: net-snmp-config is used to see default configuration + +2. paramiko: + #apt-get install python-paramiko diff --git a/compass-tasks-base/hdsdiscovery/__init__.py b/compass-tasks-base/hdsdiscovery/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/hdsdiscovery/base.py b/compass-tasks-base/hdsdiscovery/base.py new file mode 100644 index 0000000..77b3b0b --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/base.py @@ -0,0 +1,185 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Base class extended by specific vendor in vendors directory. +A vendor needs to implement abstract methods of base class. +""" +import logging +import re + +from abc import ABCMeta + +from compass.hdsdiscovery.error import TimeoutError +from compass.hdsdiscovery import utils + + +class BaseVendor(object): + """Basic Vendor object.""" + __metaclass__ = ABCMeta + + def is_this_vendor(self, sys_info, **kwargs): + """Determine if the host is associated with this vendor. + + This function must be implemented by vendor itself + """ + raise NotImplementedError + + +class BaseSnmpVendor(BaseVendor): + """Base SNMP-based vendor plugin. + + .. note:: + It uses MIB-II sysDescr value to determine the vendor of the switch. + """ + + def __init__(self, matched_names): + super(BaseSnmpVendor, self).__init__() + self._matched_names = matched_names + + def is_this_vendor(self, sys_info, **kwargs): + """Determine if the switch belongs to this vendor. + + Matching the system information retrieved from the switch. + :param str sys_info: the system information retrieved from a switch + Return True + """ + if sys_info: + for name in self._matched_names: + if re.search(r"\b" + re.escape(name) + r"\b", sys_info, + re.IGNORECASE): + return True + return False + + +class BasePlugin(object): + """Extended by vendor's plugin. + + This plugin processes request and retrieve info directly from the switch. + """ + __metaclass__ = ABCMeta + + def process_data(self, oper='SCAN', **kwargs): + """Each vendors will have some plugins to do some operations. + + Plugin will process request data and return expected result. + + :param oper: operation function name. + :param kwargs: key-value pairs of arguments + """ + raise NotImplementedError + + # At least one of these three functions below must be implemented. + def scan(self, **kwargs): + """Get multiple records at once.""" + pass + + def set(self, **kwargs): + """Set value to desired variable.""" + pass + + def get(self, **kwargs): + """Get one record from a host.""" + pass + + +class BaseSnmpMacPlugin(BasePlugin): + """Base snmp plugin.""" + + def __init__(self, host, credential, oid='BRIDGE-MIB::dot1dTpFdbPort', + vlan_oid='Q-BRIDGE-MIB::dot1qPvid'): + super(BaseSnmpMacPlugin, self).__init__() + self.host = host + self.credential = credential + self.oid = oid + self.port_oid = 'ifName' + self.vlan_oid = vlan_oid + + def process_data(self, oper='SCAN', **kwargs): + """progress data.""" + func_name = oper.lower() + return getattr(self, func_name)(**kwargs) + + def scan(self, **kwargs): + """scan.""" + results = None + try: + results = utils.snmpwalk_by_cl(self.host, self.credential, + self.oid) + except TimeoutError as error: + logging.debug("PluginMac:scan snmpwalk_by_cl failed: %s", + error.message) + return None + + mac_list = [] + for entity in results: + if_index = entity['value'] + if entity and int(if_index): + tmp = {} + mac_numbers = entity['iid'].split('.') + tmp['mac'] = self.get_mac_address(mac_numbers) + tmp['port'] = self.get_port(if_index) + tmp['vlan'] = self.get_vlan_id(if_index) + mac_list.append(tmp) + + return mac_list + + def get_vlan_id(self, port): + """Get vlan Id.""" + if not port: + return None + + oid = '.'.join((self.vlan_oid, port)) + vlan_id = None + result = None + try: + result = utils.snmpget_by_cl(self.host, self.credential, oid) + except TimeoutError as error: + logging.debug("[PluginMac:get_vlan_id snmpget_by_cl failed: %s]", + error.message) + return None + + vlan_id = result.split()[-1] + return vlan_id + + def get_port(self, if_index): + """Get port number.""" + + if_name = '.'.join((self.port_oid, if_index)) + result = None + try: + result = utils.snmpget_by_cl(self.host, self.credential, if_name) + except TimeoutError as error: + logging.debug("[PluginMac:get_port snmpget_by_cl failed: %s]", + error.message) + return None + + # A result may be like "Value: FasterEthernet1/2/34 + port = result.split()[-1].split('/')[-1] + return port + + def convert_to_hex(self, value): + """Convert the integer from decimal to hex.""" + + return "%0.2x" % int(value) + + def get_mac_address(self, mac_numbers): + """Assemble mac address from the list.""" + if len(mac_numbers) != 6: + logging.error("[PluginMac:get_mac_address] MAC address must be " + "6 digitals") + return None + + mac_in_hex = [self.convert_to_hex(num) for num in mac_numbers] + return ":".join(mac_in_hex) diff --git a/compass-tasks-base/hdsdiscovery/error.py b/compass-tasks-base/hdsdiscovery/error.py new file mode 100644 index 0000000..5bcf1a2 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/error.py @@ -0,0 +1,26 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""hdsdiscovery module errors.""" + + +class TimeoutError(Exception): + """Timeout error.""" + + def __init__(self, message): + super(TimeoutError, self).__init__(message) + self.message = message + + def __str__(self): + return repr(self.message) diff --git a/compass-tasks-base/hdsdiscovery/hdmanager.py b/compass-tasks-base/hdsdiscovery/hdmanager.py new file mode 100644 index 0000000..028d444 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/hdmanager.py @@ -0,0 +1,171 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Manage hdsdiscovery functionalities.""" +import logging +import os +import re + +from compass.hdsdiscovery.error import TimeoutError +from compass.hdsdiscovery import utils +from compass.utils import setting_wrapper as setting +from compass.utils import util + + +UNREACHABLE = 'unreachable' +NOTSUPPORTED = 'notsupported' +ERROR = 'error' +REPOLLING = 'repolling' + + +class HDManager(object): + """Process a request.""" + + def __init__(self): + base_dir = os.path.dirname(os.path.realpath(__file__)) + self.vendors_dir = os.path.join(base_dir, 'vendors') + self.vendor_plugins_dir = os.path.join(self.vendors_dir, '?/plugins') + self.snmp_sysdescr = 'sysDescr.0' + + def learn(self, host, credential, vendor, req_obj, oper="SCAN", **kwargs): + """Insert/update record of switch_info. + + Get expected results from switch according to sepcific operation. + + :param req_obj: the object of a machine + :param host: switch IP address + :param credientials: credientials to access switch + :param oper: operations of the plugin (SCAN, GETONE, SET) + :param kwargs(optional): key-value pairs + """ + plugin_dir = self.vendor_plugins_dir.replace('?', vendor) + if not os.path.exists(plugin_dir): + logging.error('No such directory: %s', plugin_dir) + return None + + plugin = utils.load_module(req_obj, plugin_dir, host, credential) + if not plugin: + # No plugin found! + # TODO(Grace): add more code to catch excpetion or unexpected state + logging.error('no plugin %s to load from %s', req_obj, plugin_dir) + return None + + return plugin.process_data(oper, **kwargs) + + def is_valid_vendor(self, host, credential, vendor): + """Check if vendor is associated with this host and credential + + :param host: switch ip + :param credential: credential to access switch + :param vendor: the vendor of switch + """ + vendor_dir = os.path.join(self.vendors_dir, vendor) + if not os.path.exists(vendor_dir): + logging.error('no such directory: %s', vendor_dir) + return False + + sys_info, err = self.get_sys_info(host, credential) + if not sys_info: + logging.debug("[hdsdiscovery][hdmanager][is_valid_vendor]" + "failded to get sys information: %s", err) + return False + + instance = utils.load_module(vendor, vendor_dir) + if not instance: + logging.debug("[hdsdiscovery][hdmanager][is_valid_vendor]" + "No such vendor found!") + return False + + if instance.is_this_vendor(sys_info): + logging.info("[hdsdiscovery][hdmanager][is_valid_vendor]" + "vendor %s is correct!", vendor) + return True + + return False + + def get_vendor(self, host, credential): + """Check and get vendor of the switch. + + :param host: switch ip: + :param credential: credential to access switch + :return a tuple (vendor, switch_state, error) + """ + + switch_lists = util.load_configs(setting.MACHINE_LIST_DIR) + switch_list = [] + for items in switch_lists: + for item in items['MACHINE_LIST']: + for k, v in item.items(): + switch_list.append(k) + if host in switch_list: + return ("appliance", "Found", "") + + # TODO(grace): Why do we need to have valid IP? + # a hostname should also work. + if not utils.valid_ip_format(host): + logging.error("host '%s' is not valid IP address!", host) + return (None, ERROR, "Invalid IP address %s!" % host) + + if not utils.is_valid_snmp_v2_credential(credential): + logging.debug("******The credential %s of host %s cannot " + "be used for either SNMP v2 or SSH*****", + credential, host) + return (None, ERROR, "Invalid credential") + + sys_info, err = self.get_sys_info(host, credential) + if not sys_info: + return (None, UNREACHABLE, err) + + # List all vendors in vendors directory -- a directory but hidden + # under ../vendors + all_vendors = [o for o in os.listdir(self.vendors_dir) + if os.path.isdir(os.path.join(self.vendors_dir, o)) + and re.match(r'^[^\.]', o)] + + logging.debug("[get_vendor][available vendors]: %s ", all_vendors) + logging.debug("[get_vendor] System Information is [%s]", sys_info) + + # TODO(grace): should not conver to lower. The vendor impl can choose + # to do case-insensitive match + # sys_info = sys_info.lower() + vendor = None + for vname in all_vendors: + vpath = os.path.join(self.vendors_dir, vname) + instance = utils.load_module(vname, vpath) + if not instance: + logging.error('no instance %s load from %s', vname, vpath) + continue + + if instance.is_this_vendor(sys_info): + logging.info("[get_vendor]****Found vendor '%s'****", vname) + vendor = vname + break + + if not vendor: + logging.debug("[get_vendor] No vendor found! <==================") + return (None, NOTSUPPORTED, "Not supported switch vendor!") + + return (vendor, REPOLLING, "") + + def get_sys_info(self, host, credential): + """get sys info.""" + sys_info = None + try: + sys_info = utils.snmpget_by_cl(host, + credential, + self.snmp_sysdescr) + except TimeoutError as error: + return (None, error.message) + + return (sys_info, "") diff --git a/compass-tasks-base/hdsdiscovery/utils.py b/compass-tasks-base/hdsdiscovery/utils.py new file mode 100644 index 0000000..72adb0a --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/utils.py @@ -0,0 +1,289 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility functions + Including functions of get/getbulk/walk/set of snmp for three versions +""" +import imp +import logging +import re +import subprocess + +from compass.hdsdiscovery.error import TimeoutError + + +def load_module(mod_name, path, host=None, credential=None): + """Load a module instance. + + :param str mod_name: module name + :param str path: directory of the module + :param str host: switch ip address + :param str credential: credential used to access switch + """ + try: + mod_file, path, descr = imp.find_module(mod_name, [path]) + if mod_file: + mod = imp.load_module(mod_name, mod_file, path, descr) + if host and credential: + instance = getattr(mod, mod.CLASS_NAME)(host, credential) + else: + instance = getattr(mod, mod.CLASS_NAME)() + + return instance + except ImportError as exc: + logging.error('No such module found: %s', mod_name) + logging.exception(exc) + return None + + +def ssh_remote_execute(host, username, password, cmd): + """SSH to execute script on remote machine + + :param host: ip of the remote machine + :param username: username to access the remote machine + :param password: password to access the remote machine + :param cmd: command to execute + """ + try: + import paramiko + if not cmd: + logging.error("[hdsdiscovery][utils][ssh_remote_execute] command" + "is None! Failed!") + return None + + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect(host, username=username, password=password, timeout=15) + stdin, stdout, stderr = client.exec_command(cmd) + result = stdout.readlines() + return result + + except ImportError as exc: + logging.error("[hdsdiscovery][utils][ssh_remote_execute] failed to" + "load module 'paramiko', donnot exist!") + logging.exception(exc) + return None + + except Exception as exc: + logging.error("[hdsdiscovery][utils][ssh_remote_execute] failed: %s", + cmd) + logging.exception(exc) + return None + + finally: + stdin.close() + stdout.close() + stderr.close() + client.close() + + +def valid_ip_format(ip_address): + """Valid the format of an Ip address.""" + + if not re.match(r'^((([0-2]?\d{0,2}\.){3}([0-2]?\d{0,2}))' + r'|(([\da-fA-F]{1,4}:){7}([\da-fA-F]{1,4})))$', + ip_address): + # check IP's format is match ipv4 or ipv6 by regex + return False + + return True + +################################################################# +# Implement snmpwalk and snmpget funtionality +# The structure of returned dictionary will by tag/iid/value/type +################################################################# +AUTH_VERSIONS = { + '1': 1, + '2c': 2, + '3': 3 +} + + +def snmp_walk(host, credential, *args, **kwargs): + """Impelmentation of snmpwalk functionality + + :param host: switch ip + :param credential: credential to access switch + :param args: OIDs + :param kwargs: key-value pairs + """ + try: + import netsnmp + + except ImportError: + logging.error("Module 'netsnmp' do not exist! Please install it first") + return None + + if 'version' not in credential or 'community' not in credential: + logging.error("[utils] missing 'version' and 'community' in %s", + credential) + return None + + version = None + if credential['version'] in AUTH_VERSIONS: + version = AUTH_VERSIONS[credential['version']] + + varbind_list = [] + for arg in args: + varbind = netsnmp.Varbind(arg) + varbind_list.append(varbind) + + var_list = netsnmp.VarList(*varbind_list) + + netsnmp.snmpwalk(var_list, + DestHost=host, + Version=version, + Community=credential['community'], + **kwargs) + + result = [] + if not var_list: + logging.error("[hsdiscovery][utils][snmp_walk] retrived no record!") + return result + + for var in var_list: + response = {} + response['elem_name'] = var.tag + response['iid'] = var.iid + response['value'] = var.val + response['type'] = var.type + result.append(response) + + return result + + +def snmp_get(host, credential, object_type, **kwargs): + """Impelmentation of snmp get functionality + + :param object_type: mib object + :param host: switch ip + :param credential: the dict of credential to access switch + """ + try: + import netsnmp + + except ImportError: + logging.error("Module 'netsnmp' do not exist! Please install it first") + return None + + if 'version' not in credential or 'community' not in credential: + logging.error('[uitls][snmp_get] missing keywords in %s for %s', + credential, host) + return None + + version = None + if credential['version'] in AUTH_VERSIONS: + version = AUTH_VERSIONS[credential['version']] + + varbind = netsnmp.Varbind(object_type) + res = netsnmp.snmpget(varbind, + DestHost=host, + Version=version, + Community=credential['community'], + **kwargs) + if res and res[0]: + return res[0] + + logging.info('no result found for %s %s', host, credential) + return None + + +SSH_CREDENTIALS = {"username": "", "password": ""} +SNMP_V2_CREDENTIALS = {"version": "", "community": ""} + + +def is_valid_snmp_v2_credential(credential): + """check if credential is valid snmp v2 credential.""" + if credential.keys() != SNMP_V2_CREDENTIALS.keys(): + return False + if credential['version'] != '2c': + logging.error("The value of version in credential is not '2c'!") + return False + return True + + +def is_valid_ssh_credential(credential): + """check if credential is valid ssh credential.""" + if credential.keys() != SSH_CREDENTIALS.keys(): + return False + return True + + +def snmpget_by_cl(host, credential, oid, timeout=8, retries=3): + """snmpget by credential.""" + if not is_valid_snmp_v2_credential(credential): + logging.error("[utils][snmpget_by_cl] Credential %s cannot be used " + "for SNMP request!", credential) + return None + + version = credential['version'] + community = credential['community'] + cmd = "snmpget -v %s -c %s -Ob -r %s -t %s %s %s" % ( + version, community, retries, timeout, host, oid) + + returncode, output, err = exec_command(cmd) + + if returncode and err: + logging.error("[snmpget_by_cl] %s", err) + raise TimeoutError(err.strip('\n')) + + return output.strip('\n') + + +def snmpwalk_by_cl(host, credential, oid, timeout=5, retries=3): + """snmpwalk by credential.""" + if not is_valid_snmp_v2_credential(credential): + logging.error("[utils][snmpwalk_by_cl] Credential %s cannot be used " + "for SNMP request!", credential) + return None + + version = credential['version'] + community = credential['community'] + cmd = "snmpwalk -v %s -c %s -Cc -r %s -t %s -Ob %s %s" % ( + version, community, retries, timeout, host, oid) + + returncode, output, err = exec_command(cmd) + + if returncode and err: + logging.debug("[snmpwalk_by_cl] %s ", err) + raise TimeoutError(err) + + result = [] + if not output: + return result + + output = output.split('\n') + for line in output: + if not line: + continue + temp = {} + arr = line.split(" ") + temp['iid'] = arr[0].split('.', 1)[-1] + temp['value'] = arr[-1] + result.append(temp) + + return result + + +def exec_command(command): + """Execute command. + + Return a tuple: returncode, output and error message(None if no error). + """ + sub_p = subprocess.Popen(command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, err_msg = sub_p.communicate() + return (sub_p.returncode, output, err_msg) diff --git a/compass-tasks-base/hdsdiscovery/vendors/__init__.py b/compass-tasks-base/hdsdiscovery/vendors/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/hdsdiscovery/vendors/appliance/__init__.py b/compass-tasks-base/hdsdiscovery/vendors/appliance/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/compass-tasks-base/hdsdiscovery/vendors/appliance/appliance.py b/compass-tasks-base/hdsdiscovery/vendors/appliance/appliance.py new file mode 100644 index 0000000..3d66f4e --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/appliance/appliance.py @@ -0,0 +1,34 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Vendor: Compass Appliance""" +from compass.hdsdiscovery import base + + +# Vendor_loader will load vendor instance by CLASS_NAME +CLASS_NAME = 'Appliance' + + +class Appliance(base.BaseSnmpVendor): + """Fake SNMP object for compass appliance.""" + + def __init__(self): + base.BaseSnmpVendor.__init__(self, ['appliance']) + self.__name = 'appliance' + + @property + def name(self): + """Get 'name' proptery.""" + return self.__name diff --git a/compass-tasks-base/hdsdiscovery/vendors/appliance/plugins/__init__.py b/compass-tasks-base/hdsdiscovery/vendors/appliance/plugins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/compass-tasks-base/hdsdiscovery/vendors/appliance/plugins/mac.py b/compass-tasks-base/hdsdiscovery/vendors/appliance/plugins/mac.py new file mode 100644 index 0000000..cc14881 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/appliance/plugins/mac.py @@ -0,0 +1,48 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compass Appliance Mac module.""" +from compass.hdsdiscovery import base +from compass.utils import setting_wrapper as setting +from compass.utils import util + +import logging + + +CLASS_NAME = "Mac" + + +class Mac(base.BaseSnmpMacPlugin): + """Processes MAC address.""" + + def __init__(self, host, credential): + self.host = host + # self.credential = credential + # return + + def scan(self): + """Implemnets the scan method in BasePlugin class. + + .. note:: + Dummy scan function for compass appliance. + Returns fixed mac addresses. + """ + mac_list = None + machine_lists = util.load_configs(setting.MACHINE_LIST_DIR) + for items in machine_lists: + for item in items['MACHINE_LIST']: + for k, v in item.items(): + if k == self.host: + mac_list = v + return mac_list diff --git a/compass-tasks-base/hdsdiscovery/vendors/arista/__init__.py b/compass-tasks-base/hdsdiscovery/vendors/arista/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/arista/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/hdsdiscovery/vendors/arista/arista.py b/compass-tasks-base/hdsdiscovery/vendors/arista/arista.py new file mode 100644 index 0000000..5eacea1 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/arista/arista.py @@ -0,0 +1,33 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Vendor: Arista.""" +from compass.hdsdiscovery import base + + +# Vendor_loader will load vendor instance by CLASS_NAME +CLASS_NAME = 'Arista' + + +class Arista(base.BaseSnmpVendor): + """Arista Network switch object.""" + + def __init__(self): + base.BaseSnmpVendor.__init__(self, ['arista']) + self._name = 'arista' + + @property + def name(self): + """Get 'name' proptery.""" + return self._name diff --git a/compass-tasks-base/hdsdiscovery/vendors/arista/plugins/__init__.py b/compass-tasks-base/hdsdiscovery/vendors/arista/plugins/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/arista/plugins/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/hdsdiscovery/vendors/arista/plugins/mac.py b/compass-tasks-base/hdsdiscovery/vendors/arista/plugins/mac.py new file mode 100644 index 0000000..ed2f331 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/arista/plugins/mac.py @@ -0,0 +1,24 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pica8 Switch Mac module.""" +from compass.hdsdiscovery.base import BaseSnmpMacPlugin + + +CLASS_NAME = 'Mac' + + +class Mac(BaseSnmpMacPlugin): + """Process MAC address by Arista switch.""" + pass diff --git a/compass-tasks-base/hdsdiscovery/vendors/hp/__init__.py b/compass-tasks-base/hdsdiscovery/vendors/hp/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/hp/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/hdsdiscovery/vendors/hp/hp.py b/compass-tasks-base/hdsdiscovery/vendors/hp/hp.py new file mode 100644 index 0000000..7ddc5e9 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/hp/hp.py @@ -0,0 +1,33 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Vendor: HP.""" +from compass.hdsdiscovery import base + + +# Vendor_loader will load vendor instance by CLASS_NAME +CLASS_NAME = 'Hp' + + +class Hp(base.BaseSnmpVendor): + """Hp switch object.""" + + def __init__(self): + base.BaseSnmpVendor.__init__(self, ['hp', 'procurve']) + self.names = ['hp', 'procurve'] + + @property + def name(self): + """Get 'name' proptery.""" + return self.names[0] diff --git a/compass-tasks-base/hdsdiscovery/vendors/hp/plugins/__init__.py b/compass-tasks-base/hdsdiscovery/vendors/hp/plugins/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/hp/plugins/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/hdsdiscovery/vendors/hp/plugins/mac.py b/compass-tasks-base/hdsdiscovery/vendors/hp/plugins/mac.py new file mode 100644 index 0000000..3bc81f4 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/hp/plugins/mac.py @@ -0,0 +1,23 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""HP Switch Mac module.""" +from compass.hdsdiscovery.base import BaseSnmpMacPlugin + +CLASS_NAME = 'Mac' + + +class Mac(BaseSnmpMacPlugin): + """Process MAC address by HP switch.""" + pass diff --git a/compass-tasks-base/hdsdiscovery/vendors/huawei/__init__.py b/compass-tasks-base/hdsdiscovery/vendors/huawei/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/huawei/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/hdsdiscovery/vendors/huawei/huawei.py b/compass-tasks-base/hdsdiscovery/vendors/huawei/huawei.py new file mode 100644 index 0000000..19fd043 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/huawei/huawei.py @@ -0,0 +1,33 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Huawei Switch.""" +from compass.hdsdiscovery import base + + +# Vendor_loader will load vendor instance by CLASS_NAME +CLASS_NAME = "Huawei" + + +class Huawei(base.BaseSnmpVendor): + """Huawei switch.""" + + def __init__(self): + base.BaseSnmpVendor.__init__(self, ["huawei"]) + self.__name = "huawei" + + @property + def name(self): + """Return switch name.""" + return self.__name diff --git a/compass-tasks-base/hdsdiscovery/vendors/huawei/plugins/__init__.py b/compass-tasks-base/hdsdiscovery/vendors/huawei/plugins/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/huawei/plugins/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/hdsdiscovery/vendors/huawei/plugins/mac.py b/compass-tasks-base/hdsdiscovery/vendors/huawei/plugins/mac.py new file mode 100644 index 0000000..49d3863 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/huawei/plugins/mac.py @@ -0,0 +1,63 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Huawei Switch Mac module.""" +import logging + +from compass.hdsdiscovery.base import BaseSnmpMacPlugin +from compass.hdsdiscovery import utils + + +CLASS_NAME = "Mac" + + +class Mac(BaseSnmpMacPlugin): + """Processes MAC address.""" + + def __init__(self, host, credential): + super(Mac, self).__init__( + host, credential, + 'HUAWEI-L2MAM-MIB::hwDynFdbPort') + + def scan(self): + """Implemnets the scan method in BasePlugin class. + + .. note:: + In this mac module, mac addesses were retrieved by + snmpwalk commandline. + """ + results = utils.snmpwalk_by_cl(self.host, self.credential, self.oid) + + if not results: + logging.info("[Huawei][mac] No results returned from SNMP walk!") + return None + + mac_list = [] + + for entity in results: + # The format of 'iid' is like '248.192.1.214.34.15.31.1.48' + # The first 6 numbers will be the MAC address + # The 7th number is its vlan ID + numbers = entity['iid'].split('.') + mac = self.get_mac_address(numbers[:6]) + vlan = numbers[6] + port = self.get_port(entity['value']) + + tmp = {} + tmp['port'] = port + tmp['mac'] = mac + tmp['vlan'] = vlan + mac_list.append(tmp) + + return mac_list diff --git a/compass-tasks-base/hdsdiscovery/vendors/ovswitch/__init__.py b/compass-tasks-base/hdsdiscovery/vendors/ovswitch/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/ovswitch/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/hdsdiscovery/vendors/ovswitch/ovswitch.py b/compass-tasks-base/hdsdiscovery/vendors/ovswitch/ovswitch.py new file mode 100644 index 0000000..4d03328 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/ovswitch/ovswitch.py @@ -0,0 +1,76 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Open Vswitch module.""" +import logging +import re + +from compass.hdsdiscovery import base +from compass.hdsdiscovery import utils + + +# Vendor_loader will load vendor instance by CLASS_NAME +CLASS_NAME = "OVSwitch" + + +class OVSwitch(base.BaseVendor): + """Open Vswitch.""" + def __init__(self): + self.__name = "Open vSwitch" + + def is_this_vendor(self, sys_info, host=None, credential=None, **kwargs): + """Determine if the hostname is accociated witH this vendor. + + :param host: swtich's IP address + :param credential: credential to access switch + """ + result = sys_info + if host and credential: + if utils.is_valid_ssh_credential(credential): + user = credential['username'] + pwd = credential['password'] + + else: + msg = ("[OVSwitch]The format of credential %r is not for SSH " + "or incorrect Keywords! " % credential) + logging.info(msg) + return False + + cmd = "ovs-vsctl -V" + result = None + try: + result = utils.ssh_remote_execute(host, user, pwd, cmd) + logging.debug('%s result for %s is %s', cmd, host, result) + if not result: + return False + except Exception as exc: + logging.error("No vendor or connection failed to run %s", cmd) + logging.exception(exc) + return False + + if isinstance(result, str): + result = [result] + + for line in result: + if not line: + continue + if re.search(r"\b" + re.escape(self.__name) + r"\b", line): + return True + + return False + + @property + def name(self): + """Open Vswitch name.""" + return self.__name diff --git a/compass-tasks-base/hdsdiscovery/vendors/ovswitch/plugins/__init__.py b/compass-tasks-base/hdsdiscovery/vendors/ovswitch/plugins/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/ovswitch/plugins/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/hdsdiscovery/vendors/ovswitch/plugins/mac.py b/compass-tasks-base/hdsdiscovery/vendors/ovswitch/plugins/mac.py new file mode 100644 index 0000000..5f497a0 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/ovswitch/plugins/mac.py @@ -0,0 +1,87 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Open Vswitch Mac address module.""" +import logging + +from compass.hdsdiscovery import base +from compass.hdsdiscovery import utils + + +CLASS_NAME = "Mac" + + +class Mac(base.BasePlugin): + """Open Vswitch MAC address module.""" + def __init__(self, host, credential): + self.host = host + self.credential = credential + + def process_data(self, oper="SCAN", **kwargs): + """Dynamically call the function according 'oper' + + :param oper: operation of data processing + """ + func_name = oper.lower() + return getattr(self, func_name)(**kwargs) + + def scan(self, **kwargs): + """Implemnets the scan method in BasePlugin class. + + .. note:: + In this module, mac addesses were retrieved by ssh. + """ + try: + user = self.credential['username'] + pwd = self.credential['password'] + except KeyError: + logging.error("Cannot find username and password in credential") + return None + + cmd = ("BRIDGES=$(ovs-vsctl show |grep Bridge |cut -f 2 -d '\"');" + "for br in $BRIDGES; do" + "PORTS=$(ovs-ofctl show $br |grep addr |cut -f 1 -d ':' " + "|egrep -v 'eth|wlan|LOCAL'|awk -F '(' '{print $1}');" + "for port in $PORTS; do" + "RESULT=$(ovs-appctl fdb/show $br |" + "awk '$1 == '$port' {print $1" "$2" "$3}');" + "echo '$RESULT'" + "done;" + "done;") + output = None + try: + output = utils.ssh_remote_execute(self.host, user, pwd, cmd) + except Exception as error: + logging.exception(error) + return None + + logging.debug("[scan][output] output is %s", output) + if not output: + return None + + fields_arr = ['port', 'vlan', 'mac'] + + result = [] + for line in output: + if not line or line == '\n': + continue + + values_arr = line.split() + temp = {} + for field, value in zip(fields_arr, values_arr): + temp[field] = value + + result.append(temp.copy()) + + return result diff --git a/compass-tasks-base/hdsdiscovery/vendors/pica8/__init__.py b/compass-tasks-base/hdsdiscovery/vendors/pica8/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/pica8/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/hdsdiscovery/vendors/pica8/pica8.py b/compass-tasks-base/hdsdiscovery/vendors/pica8/pica8.py new file mode 100644 index 0000000..856817e --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/pica8/pica8.py @@ -0,0 +1,33 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Vendor: Pica8.""" +from compass.hdsdiscovery import base + + +# Vendor_loader will load vendor instance by CLASS_NAME +CLASS_NAME = 'Pica8' + + +class Pica8(base.BaseSnmpVendor): + """Pica8 switch object.""" + + def __init__(self): + base.BaseSnmpVendor.__init__(self, ['pica8']) + self._name = 'pica8' + + @property + def name(self): + """Get 'name' proptery.""" + return self._name diff --git a/compass-tasks-base/hdsdiscovery/vendors/pica8/plugins/__init__.py b/compass-tasks-base/hdsdiscovery/vendors/pica8/plugins/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/pica8/plugins/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/hdsdiscovery/vendors/pica8/plugins/mac.py b/compass-tasks-base/hdsdiscovery/vendors/pica8/plugins/mac.py new file mode 100644 index 0000000..d5ccfc0 --- /dev/null +++ b/compass-tasks-base/hdsdiscovery/vendors/pica8/plugins/mac.py @@ -0,0 +1,24 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pica8 Switch Mac module.""" +from compass.hdsdiscovery.base import BaseSnmpMacPlugin + + +CLASS_NAME = 'Mac' + + +class Mac(BaseSnmpMacPlugin): + """Process MAC address by Pica8 switch.""" + pass diff --git a/compass-tasks-base/log_analyzor/__init__.py b/compass-tasks-base/log_analyzor/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/log_analyzor/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/log_analyzor/adapter_matcher.py b/compass-tasks-base/log_analyzor/adapter_matcher.py new file mode 100644 index 0000000..8630e01 --- /dev/null +++ b/compass-tasks-base/log_analyzor/adapter_matcher.py @@ -0,0 +1,126 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to provider installing progress calculation for the adapter. + + .. moduleauthor:: Xiaodong Wang +""" +import logging +import re + + +class AdapterItemMatcher(object): + """Progress matcher for the os installing or package installing.""" + + def __init__(self, file_matchers): + self.file_matchers_ = file_matchers + + def __repr__(self): + return '%r[file_matchers: %r]' % ( + self.__class__.__name__, self.file_matchers_ + ) + + def update_progress( + self, file_reader_factory, name, state, log_history_mapping + ): + """Update progress. + + :param name: the fullname of the installing host. + :type name: str + :param progress: Progress instance to update. + """ + for file_matcher in self.file_matchers_: + filename = file_matcher.filename_ + if filename not in log_history_mapping: + log_history_mapping[filename] = { + 'filename': filename, + 'partial_line': '', + 'position': 0, + 'line_matcher_name': 'start', + 'percentage': 0.0, + 'message': '', + 'severity': 'INFO' + } + log_history = log_history_mapping[filename] + file_matcher.update_progress( + file_reader_factory, name, state, log_history + ) + + +class OSMatcher(object): + """Progress matcher for os installer.""" + + def __init__( + self, os_installer_name, + os_pattern, item_matcher, + file_reader_factory + ): + self.name_ = re.compile(os_installer_name) + self.os_regex_ = re.compile(os_pattern) + self.matcher_ = item_matcher + self.file_reader_factory_ = file_reader_factory + + def __repr__(self): + return '%r[name:%r, os_pattern:%r, matcher:%r]' % ( + self.__class__.__name__, self.name_.pattern, + self.os_regex_.pattern, self.matcher_) + + def match(self, os_installer_name, os_name): + """Check if the os matcher is acceptable.""" + if os_name is None: + return False + else: + return all([ + self.name_.match(os_installer_name), + self.os_regex_.match(os_name) + ]) + + def update_progress(self, name, state, log_history_mapping): + """Update progress.""" + self.matcher_.update_progress( + self.file_reader_factory_, name, state, log_history_mapping) + + +class PackageMatcher(object): + """Progress matcher for package installer.""" + + def __init__( + self, package_installer_name, adapter_pattern, + item_matcher, file_reader_factory + ): + self.name_ = re.compile(package_installer_name) + self.adapter_regex_ = re.compile(adapter_pattern) + self.matcher_ = item_matcher + self.file_reader_factory_ = file_reader_factory + + def __repr__(self): + return '%s[name:%s, adapter_pattern:%s, matcher:%s]' % ( + self.__class__.__name__, self.name_.pattern, + self.adapter_regex_.pattern, self.matcher_) + + def match(self, package_installer_name, adapter_name): + """Check if the package matcher is acceptable.""" + if package_installer_name is None: + return False + else: + return all([ + self.name_.match(package_installer_name), + self.adapter_regex_.match(adapter_name) + ]) + + def update_progress(self, name, state, log_history_mapping): + """Update progress.""" + self.matcher_.update_progress( + self.file_reader_factory_, name, state, log_history_mapping + ) diff --git a/compass-tasks-base/log_analyzor/environment.py b/compass-tasks-base/log_analyzor/environment.py new file mode 100644 index 0000000..80ff738 --- /dev/null +++ b/compass-tasks-base/log_analyzor/environment.py @@ -0,0 +1,29 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""module to provide environment to load progress calculator configurations. + + .. moduleauthor:: Xiaodong Wang +""" +from compass.log_analyzor.adapter_matcher import AdapterItemMatcher +from compass.log_analyzor.file_matcher import FileMatcher +from compass.log_analyzor.file_matcher import FileReaderFactory +from compass.log_analyzor.line_matcher import IncrementalProgress +from compass.log_analyzor.line_matcher import LineMatcher +from compass.utils import setting_wrapper as setting +from compass.utils import util + + +ENV_GLOBALS = globals() +ENV_LOCALS = locals() diff --git a/compass-tasks-base/log_analyzor/file_matcher.py b/compass-tasks-base/log_analyzor/file_matcher.py new file mode 100644 index 0000000..be3143b --- /dev/null +++ b/compass-tasks-base/log_analyzor/file_matcher.py @@ -0,0 +1,252 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to update intalling progress by processing log file. + + .. moduleauthor:: Xiaodong Wang +""" +import logging +import os.path + +from compass.utils import setting_wrapper as setting + + +class FileFilter(object): + """base class to filter log file.""" + def __repr__(self): + return self.__class__.__name__ + + def filter(self, pathname): + """Filter log file. + + :param pathname: the absolute path name to the log file. + """ + raise NotImplementedError(str(self)) + + +class CompositeFileFilter(FileFilter): + """filter log file based on the list of filters.""" + def __init__(self, filters): + self.filters_ = filters + + def __str__(self): + return 'CompositeFileFilter[%s]' % self.filters_ + + def append_filter(self, file_filter): + """append filter.""" + self.filters_.append(file_filter) + + def filter(self, pathname): + """filter log file.""" + for file_filter in self.filters_: + if not file_filter.filter(pathname): + return False + + return True + + +class FilterFileExist(FileFilter): + """filter log file if not exists.""" + def filter(self, pathname): + """filter log file.""" + file_exist = os.path.isfile(pathname) + if not file_exist: + logging.debug("%s is not exist", pathname) + + return file_exist + + +def get_file_filter(): + """get file filter""" + composite_filter = CompositeFileFilter([FilterFileExist()]) + return composite_filter + + +class FileReader(object): + """Class to read log file. + + The class provide support to read log file from the position + it has read last time. and update the position when it finish + reading the log. + """ + def __init__(self, pathname, log_history): + self.pathname_ = pathname + self.log_history_ = log_history + + def __repr__(self): + return ( + '%s[pathname:%s, log_history:%s]' % ( + self.__class__.__name__, self.pathname_, + self.log_history_ + ) + ) + + def readline(self): + """Generate each line of the log file.""" + old_position = self.log_history_['position'] + position = self.log_history_['position'] + partial_line = self.log_history_['partial_line'] + try: + with open(self.pathname_) as logfile: + logfile.seek(position) + while True: + line = logfile.readline() + partial_line += line + position = logfile.tell() + if position > self.log_history_['position']: + self.log_history_['position'] = position + + if partial_line.endswith('\n'): + self.log_history_['partial_line'] = '' + yield partial_line + partial_line = self.log_history_['partial_line'] + else: + self.log_history_['partial_line'] = partial_line + break + if partial_line: + yield partial_line + + except Exception as error: + logging.error('failed to processing file %s', self.pathname_) + raise error + + logging.debug( + 'processing file %s log %s bytes to position %s', + self.pathname_, position - old_position, position + ) + + +class FileReaderFactory(object): + """factory class to create FileReader instance.""" + + def __init__(self, logdir): + self.logdir_ = logdir + self.filefilter_ = get_file_filter() + + def __str__(self): + return '%s[logdir: %s filefilter: %s]' % ( + self.__class__.__name__, self.logdir_, self.filefilter_) + + def get_file_reader(self, hostname, filename, log_history): + """Get FileReader instance. + + :param fullname: fullname of installing host. + :param filename: the filename of the log file. + + :returns: :class:`FileReader` instance if it is not filtered. + """ + pathname = os.path.join(self.logdir_, hostname, filename) + logging.debug('get FileReader from %s', pathname) + if not self.filefilter_.filter(pathname): + logging.debug('%s is filtered', pathname) + return None + + return FileReader(pathname, log_history) + + +class FileMatcher(object): + """File matcher to get the installing progress from the log file.""" + def __init__(self, line_matchers, min_progress, max_progress, filename): + if not 0.0 <= min_progress <= max_progress <= 1.0: + raise IndexError( + '%s restriction is not mat: 0.0 <= min_progress' + '(%s) <= max_progress(%s) <= 1.0' % ( + self.__class__.__name__, + min_progress, + max_progress)) + if 'start' not in line_matchers: + raise KeyError( + 'key `start` does not in line matchers %s' % line_matchers + ) + self.line_matchers_ = line_matchers + self.min_progress_ = min_progress + self.max_progress_ = max_progress + self.progress_diff_ = max_progress - min_progress + self.filename_ = filename + + def __repr__(self): + return ( + '%r[filename: %r, progress:[%r:%r], ' + 'line_matchers: %r]' % ( + self.__class__.__name__, self.filename_, + self.min_progress_, + self.max_progress_, self.line_matchers_) + ) + + def update_progress_from_log_history(self, state, log_history): + file_percentage = log_history['percentage'] + percentage = max( + self.min_progress_, + min( + self.max_progress_, + self.min_progress_ + file_percentage * self.progress_diff_ + ) + ) + if ( + percentage > state['percentage'] or + ( + percentage == state['percentage'] and + log_history['message'] != state['message'] + ) + ): + state['percentage'] = percentage + state['message'] = log_history['message'] + state['severity'] = log_history['severity'] + else: + logging.debug( + 'ingore update state %s from log history %s ' + 'since the updated progress %s lag behind', + state, log_history, percentage + ) + + def update_progress(self, file_reader_factory, name, state, log_history): + """update progress from file. + + :param fullname: the fullname of the installing host. + :type fullname: str + :param total_progress: Progress instance to update. + + the function update installing progress by reading the log file. + It contains a list of line matcher, when one log line matches + with current line matcher, the installing progress is updated. + and the current line matcher got updated. + Notes: some line may be processed multi times. The case is the + last line of log file is processed in one run, while in the other + run, it will be reprocessed at the beginning because there is + no line end indicator for the last line of the file. + """ + file_reader = file_reader_factory.get_file_reader( + name, self.filename_, log_history) + if not file_reader: + return + + line_matcher_name = log_history['line_matcher_name'] + for line in file_reader.readline(): + if line_matcher_name not in self.line_matchers_: + logging.debug('early exit at\n%s\nbecause %s is not in %s', + line, line_matcher_name, self.line_matchers_) + break + + same_line_matcher_name = line_matcher_name + while same_line_matcher_name in self.line_matchers_: + line_matcher = self.line_matchers_[same_line_matcher_name] + same_line_matcher_name, line_matcher_name = ( + line_matcher.update_progress(line, log_history) + ) + log_history['line_matcher_name'] = line_matcher_name + logging.debug( + 'updated log history %s after processing %s', + log_history, self + ) + self.update_progress_from_log_history(state, log_history) diff --git a/compass-tasks-base/log_analyzor/line_matcher.py b/compass-tasks-base/log_analyzor/line_matcher.py new file mode 100644 index 0000000..ada9ed6 --- /dev/null +++ b/compass-tasks-base/log_analyzor/line_matcher.py @@ -0,0 +1,206 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to get the progress when found match with a line of the log.""" +import logging +import re + +from abc import ABCMeta + +from compass.utils import util + + +class ProgressCalculator(object): + """base class to generate progress.""" + + __metaclass__ = ABCMeta + + @classmethod + def update_progress( + cls, progress_data, message, + severity, log_history + ): + """Update progress with the given progress_data, message and severity. + + :param progress_data: installing progress. + :type progress_data: float between 0 to 1. + :param message: installing progress message. + :param severity: installing message severity. + :param progress: :class:`Progress` instance to update + """ + # the progress is only updated when the new progress + # is greater than the stored progress or the progress + # to update is the same but the message is different. + if ( + progress_data > log_history['percentage'] or ( + progress_data == log_history['percentage'] and + message != log_history['message'] + ) + ): + log_history['percentage'] = progress_data + if message: + log_history['message'] = message + if severity: + log_history['severity'] = severity + logging.debug('update progress to %s', log_history) + else: + logging.debug('ignore update progress %s to %s', + progress_data, log_history) + + def update(self, message, severity, log_history): + """vritual method to update progress by message and severity. + + :param message: installing message. + :param severity: installing severity. + """ + raise NotImplementedError(str(self)) + + def __repr__(self): + return self.__class__.__name__ + + +class IncrementalProgress(ProgressCalculator): + """Class to increment the progress.""" + + def __init__(self, min_progress, + max_progress, incremental_ratio): + super(IncrementalProgress, self).__init__() + if not 0.0 <= min_progress <= max_progress <= 1.0: + raise IndexError( + '%s restriction is not mat: 0.0 <= min_progress(%s)' + ' <= max_progress(%s) <= 1.0' % ( + self.__class__.__name__, min_progress, max_progress)) + + if not 0.0 <= incremental_ratio <= 1.0: + raise IndexError( + '%s restriction is not mat: ' + '0.0 <= incremental_ratio(%s) <= 1.0' % ( + self.__class__.__name__, incremental_ratio)) + + self.min_progress_ = min_progress + self.max_progress_ = max_progress + self.incremental_progress_ = ( + incremental_ratio * (max_progress - min_progress)) + + def __str__(self): + return '%s[%s:%s:%s]' % ( + self.__class__.__name__, + self.min_progress_, + self.max_progress_, + self.incremental_progress_ + ) + + def update(self, message, severity, log_history): + """update progress from message and severity.""" + progress_data = max( + self.min_progress_, + min( + self.max_progress_, + log_history['percentage'] + self.incremental_progress_ + ) + ) + self.update_progress(progress_data, + message, severity, log_history) + + +class RelativeProgress(ProgressCalculator): + """class to update progress to the given relative progress.""" + + def __init__(self, progress): + super(RelativeProgress, self).__init__() + if not 0.0 <= progress <= 1.0: + raise IndexError( + '%s restriction is not mat: 0.0 <= progress(%s) <= 1.0' % ( + self.__class__.__name__, progress)) + + self.progress_ = progress + + def __str__(self): + return '%s[%s]' % (self.__class__.__name__, self.progress_) + + def update(self, message, severity, log_history): + """update progress from message and severity.""" + self.update_progress( + self.progress_, message, severity, log_history) + + +class SameProgress(ProgressCalculator): + """class to update message and severity for progress.""" + + def update(self, message, severity, log_history): + """update progress from the message and severity.""" + self.update_progress(log_history['percentage'], message, + severity, log_history) + + +class LineMatcher(object): + """Progress matcher for each line.""" + + def __init__(self, pattern, progress=None, + message_template='', severity=None, + unmatch_sameline_next_matcher_name='', + unmatch_nextline_next_matcher_name='', + match_sameline_next_matcher_name='', + match_nextline_next_matcher_name=''): + self.regex_ = re.compile(pattern) + if not progress: + self.progress_ = SameProgress() + elif isinstance(progress, ProgressCalculator): + self.progress_ = progress + elif isinstance(progress, (int, long, float)): + self.progress_ = RelativeProgress(progress) + else: + raise TypeError( + 'progress unsupport type %s: %s' % ( + type(progress), progress)) + + self.message_template_ = message_template + self.severity_ = severity + self.unmatch_sameline_ = unmatch_sameline_next_matcher_name + self.unmatch_nextline_ = unmatch_nextline_next_matcher_name + self.match_sameline_ = match_sameline_next_matcher_name + self.match_nextline_ = match_nextline_next_matcher_name + + def __repr__(self): + return '%r[pattern:%r, message_template:%r, severity:%r]' % ( + self.__class__.__name__, self.regex_.pattern, + self.message_template_, self.severity_) + + def update_progress(self, line, log_history): + """Update progress by the line. + + :param line: one line in log file to indicate the installing progress. + .. note:: + The line may be partial if the latest line of the log file is + not the whole line. But the whole line may be resent + in the next run. + :param progress: the :class:`Progress` instance to update. + """ + mat = self.regex_.search(line) + if not mat: + return ( + self.unmatch_sameline_, + self.unmatch_nextline_) + + try: + message = self.message_template_ % mat.groupdict() + except Exception as error: + logging.error('failed to get message %s %% %s in line matcher %s', + self.message_template_, mat.groupdict(), self) + raise error + + self.progress_.update(message, self.severity_, log_history) + return ( + self.match_sameline_, + self.match_nextline_) diff --git a/compass-tasks-base/log_analyzor/progress_calculator.py b/compass-tasks-base/log_analyzor/progress_calculator.py new file mode 100644 index 0000000..b0f35f2 --- /dev/null +++ b/compass-tasks-base/log_analyzor/progress_calculator.py @@ -0,0 +1,208 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""module to provide updating installing process function. + + .. moduleauthor:: Xiaodong Wang +""" +import logging + +from compass.log_analyzor.adapter_matcher import OSMatcher +from compass.log_analyzor.adapter_matcher import PackageMatcher +from compass.log_analyzor.environment import ENV_GLOBALS +from compass.log_analyzor.environment import ENV_LOCALS +from compass.log_analyzor.file_matcher import FileReaderFactory + +from compass.utils import setting_wrapper as setting +from compass.utils import util + +OS_ADAPTER_CONFIGURATIONS = None +PACKAGE_ADAPTER_CONFIGURATIONS = None +PROGRESS_CALCULATOR_CONFIGURATIONS = None + + +def _load_calculator_configurations(force=False): + global PROGRESS_CALCULATOR_CONFIGURATIONS + if force or PROGRESS_CALCULATOR_CONFIGURATIONS is None: + env_locals = {} + env_locals.update(ENV_GLOBALS) + env_locals.update(ENV_LOCALS) + PROGRESS_CALCULATOR_CONFIGURATIONS = util.load_configs( + setting.PROGRESS_CALCULATOR_DIR, + env_locals=env_locals + ) + if not PROGRESS_CALCULATOR_CONFIGURATIONS: + logging.info('No configuration found for progress calculator.') + + global OS_ADAPTER_CONFIGURATIONS + if force or OS_ADAPTER_CONFIGURATIONS is None: + OS_ADAPTER_CONFIGURATIONS = [] + for progress_calculator_configuration in ( + PROGRESS_CALCULATOR_CONFIGURATIONS + ): + if 'OS_LOG_CONFIGURATIONS' in ( + progress_calculator_configuration + ): + os_installer_configurations = ( + progress_calculator_configuration['OS_LOG_CONFIGURATIONS'] + ) + for os_installer_configuration in os_installer_configurations: + OS_ADAPTER_CONFIGURATIONS.append(OSMatcher( + os_installer_name=( + os_installer_configuration['os_installer_name'] + ), + os_pattern=os_installer_configuration['os_pattern'], + item_matcher=( + os_installer_configuration['item_matcher'] + ), + file_reader_factory=FileReaderFactory( + os_installer_configuration['logdir'] + ) + )) + if not OS_ADAPTER_CONFIGURATIONS: + logging.info( + 'no OS_LOG_CONFIGURATIONS section found ' + 'in progress calculator.' + ) + else: + logging.debug( + 'OS_ADAPTER_CONFIGURATIONS is\n%s', + OS_ADAPTER_CONFIGURATIONS + ) + + global PACKAGE_ADAPTER_CONFIGURATIONS + if force or PACKAGE_ADAPTER_CONFIGURATIONS is None: + PACKAGE_ADAPTER_CONFIGURATIONS = [] + for progress_calculator_configuration in ( + PROGRESS_CALCULATOR_CONFIGURATIONS + ): + if 'ADAPTER_LOG_CONFIGURATIONS' in ( + progress_calculator_configuration + ): + package_installer_configurations = ( + progress_calculator_configuration[ + 'ADAPTER_LOG_CONFIGURATIONS' + ] + ) + for package_installer_configuration in ( + package_installer_configurations + ): + PACKAGE_ADAPTER_CONFIGURATIONS.append(PackageMatcher( + package_installer_name=( + package_installer_configuration[ + 'package_installer_name' + ] + ), + adapter_pattern=( + package_installer_configuration['adapter_pattern'] + ), + item_matcher=( + package_installer_configuration['item_matcher'] + ), + file_reader_factory=FileReaderFactory( + package_installer_configuration['logdir'] + ) + )) + if not PACKAGE_ADAPTER_CONFIGURATIONS: + logging.info( + 'no PACKAGE_LOG_CONFIGURATIONS section found ' + 'in progress calculator.' + ) + else: + logging.debug( + 'PACKAGE_ADAPTER_CONFIGURATIONS is\n%s', + PACKAGE_ADAPTER_CONFIGURATIONS + ) + + +def load_calculator_configurations(force_reload=False): + _load_calculator_configurations(force=force_reload) + + +def _get_os_matcher(os_installer_name, os_name): + """Get OS adapter matcher by os name and installer name.""" + _load_calculator_configurations() + for configuration in OS_ADAPTER_CONFIGURATIONS: + if configuration.match(os_installer_name, os_name): + return configuration + else: + logging.debug('configuration %s does not match %s and %s', + configuration, os_name, os_installer_name) + logging.error('No configuration found for os installer %s os %s', + os_installer_name, os_name) + return None + + +def _get_package_matcher( + package_installer_name, adapter_name +): + """Get package adapter matcher by adapter name and installer name.""" + _load_calculator_configurations() + for configuration in PACKAGE_ADAPTER_CONFIGURATIONS: + if configuration.match( + package_installer_name, + adapter_name + ): + return configuration + else: + logging.debug('configuration %s does not match %s and %s', + configuration, adapter_name, + package_installer_name) + logging.error('No configuration found for package installer %s adapter %s', + package_installer_name, adapter_name) + return None + + +def update_host_progress(host_mappping): + for host_id, (host, host_state, host_log_history_mapping) in ( + host_mappping.items() + ): + os_name = host['os_name'] + os_installer_name = host['os_installer']['name'] + os_matcher = _get_os_matcher( + os_installer_name, os_name + ) + if not os_matcher: + continue + name = host[setting.HOST_INSTALLATION_LOGDIR_NAME] + os_matcher.update_progress( + name, host_state, host_log_history_mapping + ) + + +def update_clusterhost_progress(clusterhost_mapping): + for ( + clusterhost_id, + (clusterhost, clusterhost_state, clusterhost_log_history_mapping) + ) in ( + clusterhost_mapping.items() + ): + adapter_name = clusterhost['adapter_name'] + package_installer_name = clusterhost['package_installer']['name'] + package_matcher = _get_package_matcher( + package_installer_name, + adapter_name + ) + if not package_matcher: + continue + name = clusterhost[setting.CLUSTERHOST_INATALLATION_LOGDIR_NAME] + package_matcher.update_progress( + name, clusterhost_state, + clusterhost_log_history_mapping + ) + + +def update_cluster_progress(cluster_mapping): + for cluster_id, (cluster, cluster_state) in cluster_mapping.items(): + pass diff --git a/compass-tasks-base/misc/Dockerfile b/compass-tasks-base/misc/Dockerfile new file mode 100644 index 0000000..ed47aea --- /dev/null +++ b/compass-tasks-base/misc/Dockerfile @@ -0,0 +1,53 @@ +from centos:latest + +# repos +COPY misc/compass_install.repo /etc/yum.repos.d/compass_install.repo + +RUN rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \ + sed -i 's/^mirrorlist=https/mirrorlist=http/g' /etc/yum.repos.d/epel.repo && \ + yum update -y + +# packages +RUN yum --enablerepo=compass_install --nogpgcheck install -y python python-devel git amqp python-pip libffi-devel openssl-devel gcc python-setuptools MySQL-python supervisor redis sshpass python-keyczar vim ansible-2.2.1.0 + +# code +RUN mkdir -p /root/compass-tasks +COPY . /root/compass-tasks +RUN mkdir -p /root/compass-tasks/compass && \ + touch /root/compass-tasks/compass/__init__.py +RUN mv /root/compass-tasks/actions /root/compass-tasks/compass/ && \ + mv /root/compass-tasks/apiclient /root/compass-tasks/compass/ && \ + mv /root/compass-tasks/tasks /root/compass-tasks/compass/ && \ + mv /root/compass-tasks/utils /root/compass-tasks/compass/ && \ + mv /root/compass-tasks/deployment /root/compass-tasks/compass/ && \ + mv /root/compass-tasks/db /root/compass-tasks/compass/ && \ + mv /root/compass-tasks/hdsdiscovery /root/compass-tasks/compass/ && \ + mv /root/compass-tasks/log_analyzor /root/compass-tasks/compass/ + +# pip +RUN easy_install --upgrade pip && \ + pip install --upgrade pip && \ + pip install --upgrade setuptools && \ + pip install --upgrade Flask + +# conf +RUN mkdir -p /etc/compass/ && \ + mkdir -p /etc/compass/machine_list && \ + mkdir -p /etc/compass/switch_list && \ + #cp -rf /root/compass-tasks/conf/* /etc/compass/ && \ + cd /root/compass-tasks && \ + python setup.py install + +RUN mkdir -p /root/.ssh; \ + echo "UserKnownHostsFile /dev/null" >> /root/.ssh/config; \ + echo "StrictHostKeyChecking no" >> /root/.ssh/config + +COPY supervisord.conf /etc/supervisord.conf +COPY start.sh /usr/local/bin/start.sh +RUN mkdir -p /var/log/compass +RUN mkdir -p /opt/ansible_callbacks +#RUN git clone https://github.com/openstack-ansible/openstack-ansible-modules /opt/openstack-ansible-modules +EXPOSE 6379 +VOLUME ["/var/ansible", "/etc/compass/machine_list", "/etc/compass/switch_list"] +ENTRYPOINT ["/bin/bash", "-c"] +CMD ["/usr/local/bin/start.sh"] diff --git a/compass-tasks-base/misc/compass_install.repo b/compass-tasks-base/misc/compass_install.repo new file mode 100644 index 0000000..6b97ed0 --- /dev/null +++ b/compass-tasks-base/misc/compass_install.repo @@ -0,0 +1,5 @@ +[compass_install] +name=compass_repo +baseurl=http://192.168.104.2:9999/download/compass_install/centos7/ +gpgcheck=0 +enabled=1 diff --git a/compass-tasks-base/requirements.txt b/compass-tasks-base/requirements.txt new file mode 100644 index 0000000..5520a0b --- /dev/null +++ b/compass-tasks-base/requirements.txt @@ -0,0 +1,22 @@ +amqplib +argparse +celery +Markdown<2.5 +Cheetah<=2.4.1 +Flask +Flask-Login<=0.3.2 +Flask-RESTful +Flask-Script +Flask-SQLAlchemy +Flask-WTF +itsdangerous +importlib +lazypy +lockfile +netaddr +# MySQL-python +paramiko +SQLAlchemy>=0.9.0 +simplejson +requests +redis diff --git a/compass-tasks-base/setup.py b/compass-tasks-base/setup.py new file mode 100644 index 0000000..5ee2e12 --- /dev/null +++ b/compass-tasks-base/setup.py @@ -0,0 +1,97 @@ +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""setup script.""" +try: + from setuptools import find_packages +except ImportError: + from ez_setup import use_setuptools + use_setuptools() + + +from setuptools.command.test import test as TestCommand +from setuptools import setup + + +import os +import sys + + +# This helps python setup.py test command to utilize tox +# See the instruction at https://testrun.org/tox/latest/example/basic.html\ +# #integration-with-setuptools-distribute-test-commands + +class Tox(TestCommand): + """Tox to do the setup.""" + + def finalize_options(self): + TestCommand.finalize_options(self) + self.test_args = [] + self.test_suite = True + + def run_tests(self): + import tox + errno = tox.cmdline(self.test_args) + sys.exit(errno) + + +INSTALL_REQUIRES_FILE = os.path.join( + os.path.dirname(__file__), 'requirements.txt') +with open(INSTALL_REQUIRES_FILE, 'r') as requires_file: + REQUIREMENTS = [line.strip() for line in requires_file if line != '\n'] + +DATA_FILES_DIR = os.path.join( + os.path.dirname(__file__), 'conf') +DATA_FILES = [] +for parent_dir, sub_dirs, files in os.walk(DATA_FILES_DIR): + if files == []: + pass + for file in files: + DATA_FILES.append((parent_dir, [os.path.join(parent_dir, file)])) + +setup( + name='compass-tasks', + version='0.1.0', + + # general info + description="""compass-tasks: Compass tasks module""", + author='Compass Development Group', + author_email='dev@syscompass.org', + url='https://github.com/openstack/compass-core', + download_url='', + + # dependency + install_requires=REQUIREMENTS, + packages=find_packages(exclude=['compass.tests']), + include_package_data=True, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'Intended Audience :: Information Technology', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + ], + # data + # data_files=DATA_FILES, + # test, + tests_require=['tox'], + cmdclass={'test': Tox}, +) diff --git a/compass-tasks-base/start.sh b/compass-tasks-base/start.sh new file mode 100755 index 0000000..4d3956d --- /dev/null +++ b/compass-tasks-base/start.sh @@ -0,0 +1,4 @@ +#!/bin/bash +/sbin/init +/usr/bin/supervisord +tail -f /dev/null diff --git a/compass-tasks-base/supervisord.conf b/compass-tasks-base/supervisord.conf new file mode 100644 index 0000000..9d993d3 --- /dev/null +++ b/compass-tasks-base/supervisord.conf @@ -0,0 +1,135 @@ +; Sample supervisor config file. + +[unix_http_server] +file=/var/run/supervisor/supervisor.sock ; (the path to the socket file) +;chmod=0700 ; sockef file mode (default 0700) +;chown=nobody:nogroup ; socket file uid:gid owner +;username=user ; (default is no username (open server)) +;password=123 ; (default is no password (open server)) + +;[inet_http_server] ; inet (TCP) server disabled by default +;port=127.0.0.1:9001 ; (ip_address:port specifier, *:port for all iface) +;username=user ; (default is no username (open server)) +;password=123 ; (default is no password (open server)) + +[supervisord] +logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log) +logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) +logfile_backups=10 ; (num of main logfile rotation backups;default 10) +loglevel=info ; (log level;default info; others: debug,warn,trace) +pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) +nodaemon=false ; (start in foreground if true;default false) +minfds=1024 ; (min. avail startup file descriptors;default 1024) +minprocs=200 ; (min. avail process descriptors;default 200) +;umask=022 ; (process file creation umask;default 022) +;user=chrism ; (default is current user, required if root) +;identifier=supervisor ; (supervisord identifier, default is 'supervisor') +;directory=/tmp ; (default is not to cd during start) +;nocleanup=true ; (don't clean up tempfiles at start;default false) +;childlogdir=/tmp ; ('AUTO' child log dir, default $TEMP) +;environment=KEY=value ; (key value pairs to add to environment) +;strip_ansi=false ; (strip ansi escape codes in logs; def. false) + +; the below section must remain in the config file for RPC +; (supervisorctl/web interface) to work, additional interfaces may be +; added by defining them in separate rpcinterface: sections +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisor/supervisor.sock ; use a unix:// URL for a unix socket +;serverurl=http://127.0.0.1:9001 ; use an http:// url to specify an inet socket +;username=chris ; should be same as http_username if set +;password=123 ; should be same as http_password if set +;prompt=mysupervisor ; cmd line prompt (default "supervisor") +;history_file=~/.sc_history ; use readline history if available + +; The below sample program section shows all possible program subsection values, +; create one or more 'real' program: sections to be able to control them under +; supervisor. + +;[program:theprogramname] +;command=/bin/cat ; the program (relative uses PATH, can take args) +;process_name=%(program_name)s ; process_name expr (default %(program_name)s) +;numprocs=1 ; number of processes copies to start (def 1) +;directory=/tmp ; directory to cwd to before exec (def no cwd) +;umask=022 ; umask for process (default None) +;priority=999 ; the relative start priority (default 999) +;autostart=true ; start at supervisord start (default: true) +;autorestart=true ; retstart at unexpected quit (default: true) +;startsecs=10 ; number of secs prog must stay running (def. 1) +;startretries=3 ; max # of serial start failures (default 3) +;exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) +;stopsignal=QUIT ; signal used to kill process (default TERM) +;stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) +;user=chrism ; setuid to this UNIX account to run the program +;redirect_stderr=true ; redirect proc stderr to stdout (default false) +;stdout_logfile=/a/path ; stdout log path, NONE for none; default AUTO +;stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) +;stdout_logfile_backups=10 ; # of stdout logfile backups (default 10) +;stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) +;stdout_events_enabled=false ; emit events on stdout writes (default false) +;stderr_logfile=/a/path ; stderr log path, NONE for none; default AUTO +;stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) +;stderr_logfile_backups=10 ; # of stderr logfile backups (default 10) +;stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) +;stderr_events_enabled=false ; emit events on stderr writes (default false) +;environment=A=1,B=2 ; process environment additions (def no adds) +;serverurl=AUTO ; override serverurl computation (childutils) + +; The below sample eventlistener section shows all possible +; eventlistener subsection values, create one or more 'real' +; eventlistener: sections to be able to handle event notifications +; sent by supervisor. + +;[eventlistener:theeventlistenername] +;command=/bin/eventlistener ; the program (relative uses PATH, can take args) +;process_name=%(program_name)s ; process_name expr (default %(program_name)s) +;numprocs=1 ; number of processes copies to start (def 1) +;events=EVENT ; event notif. types to subscribe to (req'd) +;buffer_size=10 ; event buffer queue size (default 10) +;directory=/tmp ; directory to cwd to before exec (def no cwd) +;umask=022 ; umask for process (default None) +;priority=-1 ; the relative start priority (default -1) +;autostart=true ; start at supervisord start (default: true) +;autorestart=unexpected ; restart at unexpected quit (default: unexpected) +;startsecs=10 ; number of secs prog must stay running (def. 1) +;startretries=3 ; max # of serial start failures (default 3) +;exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) +;stopsignal=QUIT ; signal used to kill process (default TERM) +;stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) +;user=chrism ; setuid to this UNIX account to run the program +;redirect_stderr=true ; redirect proc stderr to stdout (default false) +;stdout_logfile=/a/path ; stdout log path, NONE for none; default AUTO +;stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) +;stdout_logfile_backups=10 ; # of stdout logfile backups (default 10) +;stdout_events_enabled=false ; emit events on stdout writes (default false) +;stderr_logfile=/a/path ; stderr log path, NONE for none; default AUTO +;stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) +;stderr_logfile_backups ; # of stderr logfile backups (default 10) +;stderr_events_enabled=false ; emit events on stderr writes (default false) +;environment=A=1,B=2 ; process environment additions +;serverurl=AUTO ; override serverurl computation (childutils) + +; The below sample group section shows all possible group values, +; create one or more 'real' group: sections to create "heterogeneous" +; process groups. + +;[group:thegroupname] +;programs=progname1,progname2 ; each refers to 'x' in [program:x] definitions +;priority=999 ; the relative start priority (default 999) + +; The [include] section can just contain the "files" setting. This +; setting can list multiple files (separated by whitespace or +; newlines). It can also contain wildcards. The filenames are +; interpreted as relative to this file. Included files *cannot* +; include files themselves. + +[program:celeryd] +command=nohup bash -c "C_FORCE_ROOT=1 CELERY_CONFIG_MODULE=compass.utils.celeryconfig_wrapper /usr/bin/celery worker &> /tmp/celery-worker.log" & + +[program:orca-heartbeatd] +command=nohup bash -c "/usr/bin/redis-server" & +[include] +files = supervisord.d/*.ini + diff --git a/compass-tasks-base/tasks/__init__.py b/compass-tasks-base/tasks/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/tasks/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/tasks/client.py b/compass-tasks-base/tasks/client.py new file mode 100644 index 0000000..ca7ad14 --- /dev/null +++ b/compass-tasks-base/tasks/client.py @@ -0,0 +1,33 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to setup celery client. + + .. moduleauthor:: Xiaodong Wang + + .. note:: + If CELERY_CONFIG_MODULE is set in environment, load celery config from + the filename declared in CELERY_CONFIG_MODULE. +""" +import os + +from celery import Celery + + +celery = Celery(__name__) +if 'CELERY_CONFIG_MODULE' in os.environ: + celery.config_from_envvar('CELERY_CONFIG_MODULE') +else: + from compass.utils import celeryconfig_wrapper as celeryconfig + celery.config_from_object(celeryconfig) diff --git a/compass-tasks-base/tasks/tasks.py b/compass-tasks-base/tasks/tasks.py new file mode 100644 index 0000000..f649afd --- /dev/null +++ b/compass-tasks-base/tasks/tasks.py @@ -0,0 +1,326 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to define celery tasks. + + .. moduleauthor:: Xiaodong Wang +""" +import logging + +from celery.signals import celeryd_init +from celery.signals import setup_logging + +from compass.actions import clean +from compass.actions import delete +from compass.actions import deploy +from compass.actions import install_callback +from compass.actions import patch +from compass.actions import poll_switch +from compass.actions import update_progress +from compass.db.api import adapter_holder as adapter_api +from compass.db.api import database +from compass.db.api import metadata_holder as metadata_api +from compass.log_analyzor import progress_calculator + +from compass.tasks.client import celery +from compass.utils import flags +from compass.utils import logsetting +from compass.utils import setting_wrapper as setting + + +@celeryd_init.connect() +def global_celery_init(**_): + """Initialization code.""" + flags.init() + flags.OPTIONS.logfile = setting.CELERY_LOGFILE + logsetting.init() + database.init() + adapter_api.load_adapters() + metadata_api.load_metadatas() + adapter_api.load_flavors() + progress_calculator.load_calculator_configurations() + + +@setup_logging.connect() +def tasks_setup_logging(**_): + """Setup logging options from compass setting.""" + flags.init() + flags.OPTIONS.logfile = setting.CELERY_LOGFILE + logsetting.init() + + +@celery.task(name='compass.tasks.pollswitch') +def pollswitch( + poller_email, ip_addr, credentials, + req_obj='mac', oper='SCAN' +): + """Query switch and return expected result. + + :param ip_addr: switch ip address. + :type ip_addr: str + :param credentials: switch credentials + :type credentials: dict + :param reqObj: the object requested to query from switch. + :type reqObj: str + :param oper: the operation to query the switch (SCAN, GET, SET). + :type oper: str + """ + try: + poll_switch.poll_switch( + poller_email, ip_addr, credentials, + req_obj=req_obj, oper=oper + ) + except Exception as error: + logging.exception(error) + + +@celery.task(name='compass.tasks.cluster_health') +def health_check(cluster_id, send_report_url, useremail): + """Verify the deployed cluster functionally works. + + :param cluster_id: ID of the cluster + :param send_report_url: The URL which reports should send back + """ + try: + deploy.health_check(cluster_id, send_report_url, useremail) + except Exception as error: + logging.exception(error) + + +@celery.task(name='compass.tasks.deploy_cluster') +def deploy_cluster(deployer_email, cluster_id, clusterhost_ids): + """Deploy the given cluster. + + :param cluster_id: id of the cluster + :type cluster_id: int + :param clusterhost_ids: the id of the hosts in the cluster + :type clusterhost_ids: list of int + """ + try: + deploy.deploy(cluster_id, clusterhost_ids, deployer_email) + except Exception as error: + logging.exception(error) + + +@celery.task(name='compass.tasks.redeploy_cluster') +def redeploy_cluster(deployer_email, cluster_id): + """Redeploy the given cluster. + + :param cluster_id: id of the cluster + :type cluster_id: int + """ + try: + deploy.redeploy(cluster_id, deployer_email) + except Exception as error: + logging.exception(error) + + +@celery.task(name='compass.tasks.patch_cluster') +def patch_cluster(patcher_email, cluster_id): + """Patch the existing cluster. + + :param cluster_id: id of the cluster + :type cluster_id: int + """ + try: + patch.patch(cluster_id, patcher_email) + except Exception as error: + logging.exception(error) + + +@celery.task(name='compass.tasks.reinstall_cluster') +def reinstall_cluster(installer_email, cluster_id, clusterhost_ids): + """reinstall the given cluster. + + :param cluster_id: id of the cluster + :type cluster_id: int + :param clusterhost_ids: the id of the hosts in the cluster + :type clusterhost_ids: list of int + """ + try: + deploy.redeploy(cluster_id, clusterhost_ids, installer_email) + except Exception as error: + logging.exception(error) + + +@celery.task(name='compass.tasks.delete_cluster') +def delete_cluster( + deleter_email, cluster_id, clusterhost_ids, + delete_underlying_host=False +): + """Delete the given cluster. + + :param cluster_id: id of the cluster + :type cluster_id: int + :param clusterhost_ids: the id of the hosts in the cluster + :type clusterhost_ids: list of int + """ + try: + delete.delete_cluster( + cluster_id, clusterhost_ids, deleter_email, + delete_underlying_host=delete_underlying_host + ) + except Exception as error: + logging.exception(error) + + +@celery.task(name='compass.tasks.delete_cluster_host') +def delete_cluster_host( + deleter_email, cluster_id, host_id, + delete_underlying_host=False +): + """Delte the given cluster host. + + :param cluster_id: id of the cluster + :type cluster_id: int + :param host_id: id of the host + :type host_id: int + """ + try: + delete.delete_cluster_host( + cluster_id, host_id, deleter_email, + delete_underlying_host=delete_underlying_host + ) + except Exception as error: + logging.exception(error) + + +@celery.task(name='compass.tasks.delete_host') +def delete_host(deleter_email, host_id, cluster_ids): + """Delete the given host. + + :param host_id: id of the host + :type host_id: int + :param cluster_ids: list of cluster id + :type cluster_ids: list of int + """ + try: + delete.delete_host( + host_id, cluster_ids, deleter_email + ) + except Exception as error: + logging.exception(error) + + +@celery.task(name='compass.tasks.clean_os_installer') +def clean_os_installer( + os_installer_name, os_installer_settings +): + """Clean os installer.""" + try: + clean.clean_os_installer( + os_installer_name, os_installer_settings + ) + except Exception as error: + logging.excception(error) + + +@celery.task(name='compass.tasks.clean_package_installer') +def clean_package_installer( + package_installer_name, package_installer_settings +): + """Clean package installer.""" + try: + clean.clean_package_installer( + package_installer_name, package_installer_settings + ) + except Exception as error: + logging.excception(error) + + +@celery.task(name='compass.tasks.poweron_host') +def poweron_host(host_id): + """Deploy the given cluster.""" + pass + + +@celery.task(name='compass.tasks.poweroff_host') +def poweroff_host(host_id): + """Deploy the given cluster.""" + pass + + +@celery.task(name='compass.tasks.reset_host') +def reset_host(host_id): + """Deploy the given cluster.""" + pass + + +@celery.task(name='compass.tasks.poweron_machine') +def poweron_machine(machine_id): + """Deploy the given cluster.""" + pass + + +@celery.task(name='compass.tasks.poweroff_machine') +def poweroff_machine(machine_id): + """Deploy the given cluster.""" + pass + + +@celery.task(name='compass.tasks.reset_machine') +def reset_machine(machine_id): + """Deploy the given cluster.""" + pass + + +@celery.task(name='compass.tasks.os_installed') +def os_installed( + host_id, clusterhosts_ready, + clusters_os_ready +): + """callback when os is installed.""" + try: + install_callback.os_installed( + host_id, clusterhosts_ready, + clusters_os_ready + ) + except Exception as error: + logging.exception(error) + + +@celery.task(name='compass.tasks.package_installed') +def package_installed( + cluster_id, host_id, cluster_ready, host_ready +): + """callback when package is installed.""" + try: + install_callback.package_installed( + cluster_id, host_id, cluster_ready, host_ready + ) + except Exception as error: + logging.exception(error) + + +@celery.task(name='compass.tasks.cluster_installed') +def cluster_installed( + cluster_id, clusterhosts_ready +): + """callback when package is installed.""" + try: + install_callback.cluster_installed( + cluster_id, clusterhosts_ready + ) + except Exception as error: + logging.exception(error) + + +@celery.task(name='compass.tasks.update_progress') +def update_clusters_progress(): + """Calculate the installing progress of the given cluster.""" + logging.info('update_clusters_progress') + try: + update_progress.update_progress() + except Exception as error: + logging.exception(error) diff --git a/compass-tasks-base/utils/__init__.py b/compass-tasks-base/utils/__init__.py new file mode 100644 index 0000000..4ee55a4 --- /dev/null +++ b/compass-tasks-base/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass-tasks-base/utils/celeryconfig_wrapper.py b/compass-tasks-base/utils/celeryconfig_wrapper.py new file mode 100644 index 0000000..b6644ba --- /dev/null +++ b/compass-tasks-base/utils/celeryconfig_wrapper.py @@ -0,0 +1,44 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""celeryconfig wrapper. + + .. moduleauthor:: Xiaodong Wang +""" +import logging +import os.path +import urllib + +from compass.utils import setting_wrapper as setting + + +# CELERY_RESULT_BACKEND = 'amqp://' + +# BROKER_URL = 'amqp://guest:guest@localhost:5672//' + + +CELERY_IMPORTS = ('compass.tasks.tasks',) + + +if setting.CELERYCONFIG_FILE: + CELERY_CONFIG = os.path.join( + str(setting.CELERYCONFIG_DIR), + str(setting.CELERYCONFIG_FILE)) + + try: + logging.info('load celery config from %s', CELERY_CONFIG) + execfile(CELERY_CONFIG, globals(), locals()) + except Exception as error: + logging.exception(error) + raise error diff --git a/compass-tasks-base/utils/flags.py b/compass-tasks-base/utils/flags.py new file mode 100644 index 0000000..a3169f5 --- /dev/null +++ b/compass-tasks-base/utils/flags.py @@ -0,0 +1,91 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to load flags. + + .. moduleauthor:: Xiaodong Wang +""" +import sys + +from optparse import OptionParser + + +class Flags(object): + """Class to store flags.""" + + PARSER = OptionParser() + PARSED_OPTIONS = None + + @classmethod + def parse_args(cls): + """parse args.""" + (options, argv) = Flags.PARSER.parse_args() + sys.argv = [sys.argv[0]] + argv + Flags.PARSED_OPTIONS = options + + def __getattr__(self, name): + if Flags.PARSED_OPTIONS and hasattr(Flags.PARSED_OPTIONS, name): + return getattr(Flags.PARSED_OPTIONS, name) + + for option in Flags.PARSER.option_list: + if option.dest == name: + return option.default + + raise AttributeError('Option instance has no attribute %s' % name) + + def __setattr__(self, name, value): + if Flags.PARSED_OPTIONS and hasattr(Flags.PARSED_OPTIONS, name): + setattr(Flags.PARSED_OPTIONS, name, value) + return + + for option in Flags.PARSER.option_list: + if option.dest == name: + option.default = value + return + + object.__setattr__(self, name, value) + + +OPTIONS = Flags() + + +def init(): + """Init flag parsing.""" + OPTIONS.parse_args() + + +def add(flagname, **kwargs): + """Add a flag name and its setting. + + :param flagname: flag name declared in cmd as --=... + :type flagname: str + """ + Flags.PARSER.add_option('--%s' % flagname, + dest=flagname, **kwargs) + + +def add_bool(flagname, default=True, **kwargs): + """Add a bool flag name and its setting. + + :param flagname: flag name declared in cmd as --[no]. + :type flagname: str + :param default: default value + :type default: bool + """ + Flags.PARSER.add_option('--%s' % flagname, + dest=flagname, default=default, + action="store_true", **kwargs) + Flags.PARSER.add_option('--no%s' % flagname, + dest=flagname, + action="store_false", **kwargs) diff --git a/compass-tasks-base/utils/logsetting.py b/compass-tasks-base/utils/logsetting.py new file mode 100644 index 0000000..836ebcb --- /dev/null +++ b/compass-tasks-base/utils/logsetting.py @@ -0,0 +1,108 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to setup logging configuration. + + .. moduleauthor:: Xiaodong Wang +""" + +import logging +import logging.handlers +import os +import os.path +import sys + +from compass.utils import flags +from compass.utils import setting_wrapper as setting + + +flags.add('loglevel', + help='logging level', default=setting.DEFAULT_LOGLEVEL) +flags.add('logdir', + help='logging directory', default=setting.DEFAULT_LOGDIR) +flags.add('logfile', + help='logging filename', default=None) +flags.add('log_interval', type='int', + help='log interval', default=setting.DEFAULT_LOGINTERVAL) +flags.add('log_interval_unit', + help='log interval unit', default=setting.DEFAULT_LOGINTERVAL_UNIT) +flags.add('log_format', + help='log format', default=setting.DEFAULT_LOGFORMAT) +flags.add('log_backup_count', type='int', + help='log backup count', default=setting.DEFAULT_LOGBACKUPCOUNT) + + +# mapping str setting in flag --loglevel to logging level. +LOGLEVEL_MAPPING = { + 'finest': logging.DEBUG - 2, # more detailed log. + 'fine': logging.DEBUG - 1, # detailed log. + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warning': logging.WARNING, + 'error': logging.ERROR, + 'critical': logging.CRITICAL, +} + + +logging.addLevelName(LOGLEVEL_MAPPING['fine'], 'fine') +logging.addLevelName(LOGLEVEL_MAPPING['finest'], 'finest') + + +# disable logging when logsetting.init not called +logging.getLogger().setLevel(logging.CRITICAL) + + +def getLevelByName(level_name): + """Get log level by level name.""" + return LOGLEVEL_MAPPING[level_name] + + +def init(): + """Init loggsetting. It should be called after flags.init.""" + loglevel = flags.OPTIONS.loglevel.lower() + logdir = flags.OPTIONS.logdir + logfile = flags.OPTIONS.logfile + logger = logging.getLogger() + if logger.handlers: + for handler in logger.handlers: + logger.removeHandler(handler) + + if logdir: + if not logfile: + logfile = '%s.log' % os.path.basename(sys.argv[0]) + + handler = logging.handlers.TimedRotatingFileHandler( + os.path.join(logdir, logfile), + when=flags.OPTIONS.log_interval_unit, + interval=flags.OPTIONS.log_interval, + backupCount=flags.OPTIONS.log_backup_count) + else: + if not logfile: + handler = logging.StreamHandler(sys.stderr) + else: + handler = logging.handlers.TimedRotatingFileHandler( + logfile, + when=flags.OPTIONS.log_interval_unit, + interval=flags.OPTIONS.log_interval, + backupCount=flags.OPTIONS.log_backup_count) + + if loglevel in LOGLEVEL_MAPPING: + logger.setLevel(LOGLEVEL_MAPPING[loglevel]) + handler.setLevel(LOGLEVEL_MAPPING[loglevel]) + + formatter = logging.Formatter( + flags.OPTIONS.log_format) + + handler.setFormatter(formatter) + logger.addHandler(handler) diff --git a/compass-tasks-base/utils/setting_wrapper.py b/compass-tasks-base/utils/setting_wrapper.py new file mode 100644 index 0000000..0b3e9f7 --- /dev/null +++ b/compass-tasks-base/utils/setting_wrapper.py @@ -0,0 +1,175 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""comapss setting wrapper. + + .. moduleauthor:: Xiaodong Wang ,xiaodongwang@huawei.com> +""" +import datetime +import lazypy +import logging +import os +import os.path + + +# default setting +CONFIG_DIR = os.environ.get('COMPASS_CONFIG_DIR', '/etc/compass') +SQLALCHEMY_DATABASE_URI = 'sqlite://' +SQLALCHEMY_DATABASE_POOL_TYPE = 'static' +COBBLER_INSTALLATION_LOGDIR = '/var/log/cobbler/anamon' +CHEF_INSTALLATION_LOGDIR = '/var/log/chef' +INSTALLATION_LOGDIR = { + 'CobblerInstaller': COBBLER_INSTALLATION_LOGDIR, + 'ChefInstaller': CHEF_INSTALLATION_LOGDIR +} +CLUSTERHOST_INATALLATION_LOGDIR_NAME = 'name' +HOST_INSTALLATION_LOGDIR_NAME = 'name' +DEFAULT_LOGLEVEL = 'debug' +DEFAULT_LOGDIR = '/tmp' +DEFAULT_LOGINTERVAL = 1 +DEFAULT_LOGINTERVAL_UNIT = 'h' +DEFAULT_LOGFORMAT = ( + '%(asctime)s - %(filename)s - %(lineno)d - %(levelname)s - %(message)s') +DEFAULT_LOGBACKUPCOUNT = 5 +WEB_LOGFILE = '' +CELERY_LOGFILE = '' +CELERYCONFIG_DIR = lazypy.delay(lambda: CONFIG_DIR) +CELERYCONFIG_FILE = '' +PROGRESS_UPDATE_INTERVAL = 30 +POLLSWITCH_INTERVAL = 60 +SWITCHES = [ +] + +USER_AUTH_HEADER_NAME = 'X-Auth-Token' +USER_TOKEN_DURATION = '2h' +COMPASS_ADMIN_EMAIL = 'admin@huawei.com' +COMPASS_ADMIN_PASSWORD = 'admin' +COMPASS_DEFAULT_PERMISSIONS = [ + 'list_permissions', +] +SWITCHES_DEFAULT_FILTERS = [] +DEFAULT_SWITCH_IP = '0.0.0.0' +DEFAULT_SWITCH_PORT = 0 + +COMPASS_SUPPORTED_PROXY = 'http://127.0.0.1:3128' +COMPASS_SUPPORTED_DEFAULT_NOPROXY = ['127.0.0.1'] +COMPASS_SUPPORTED_NTP_SERVER = '127.0.0.1' +COMPASS_SUPPORTED_DNS_SERVERS = ['127.0.0.1'] +COMPASS_SUPPORTED_DOMAINS = [] +COMPASS_SUPPORTED_DEFAULT_GATEWAY = '127.0.0.1' +COMPASS_SUPPORTED_LOCAL_REPO = 'http://127.0.0.1' + +PROGRESS_UPDATE_PID_FILE = '/var/run/progress_update.pid' + +PROXY_URL_PREFIX = 'http://10.145.81.205:5000' + +OS_INSTALLER_DIR = '' +PACKAGE_INSTALLER_DIR = '' +OS_DIR = '' +ADAPTER_DIR = '' +OS_METADATA_DIR = '' +PACKAGE_METADATA_DIR = '' +FLAVOR_METADATA_DIR = '' +OS_FIELD_DIR = '' +PACKAGE_FIELD_DIR = '' +FLAVOR_FIELD_DIR = '' +ADAPTER_ROLE_DIR = '' +ADAPTER_FLAVOR_DIR = '' +VALIDATOR_DIR = '' +CALLBACK_DIR = '' +TMPL_DIR = '' +MACHINE_LIST_DIR = '' +PROGRESS_CALCULATOR_DIR = '' +OS_MAPPING_DIR = '' +FLAVOR_MAPPING_DIR = '' +PLUGINS_DIR = '' + +if ( + 'COMPASS_IGNORE_SETTING' in os.environ and + os.environ['COMPASS_IGNORE_SETTING'] +): + pass +else: + if 'COMPASS_SETTING' in os.environ: + SETTING = os.environ['COMPASS_SETTING'] + else: + SETTING = '/etc/compass/setting' + + try: + logging.info('load setting from %s', SETTING) + execfile(SETTING, globals(), locals()) + except Exception as error: + logging.exception(error) + raise error + +if not OS_INSTALLER_DIR: + OS_INSTALLER_DIR = os.path.join(CONFIG_DIR, 'os_installer') + +if not PACKAGE_INSTALLER_DIR: + PACKAGE_INSTALLER_DIR = os.path.join(CONFIG_DIR, 'package_installer') + +if not OS_DIR: + OS_DIR = os.path.join(CONFIG_DIR, 'os') + +if not ADAPTER_DIR: + ADAPTER_DIR = os.path.join(CONFIG_DIR, 'adapter') + +if not OS_METADATA_DIR: + OS_METADATA_DIR = os.path.join(CONFIG_DIR, 'os_metadata') + +if not PACKAGE_METADATA_DIR: + PACKAGE_METADATA_DIR = os.path.join(CONFIG_DIR, 'package_metadata') + +if not FLAVOR_METADATA_DIR: + FLAVOR_METADATA_DIR = os.path.join(CONFIG_DIR, 'flavor_metadata') + +if not OS_FIELD_DIR: + OS_FIELD_DIR = os.path.join(CONFIG_DIR, 'os_field') + +if not PACKAGE_FIELD_DIR: + PACKAGE_FIELD_DIR = os.path.join(CONFIG_DIR, 'package_field') + +if not FLAVOR_FIELD_DIR: + FLAVOR_FIELD_DIR = os.path.join(CONFIG_DIR, 'flavor_field') + +if not ADAPTER_ROLE_DIR: + ADAPTER_ROLE_DIR = os.path.join(CONFIG_DIR, 'role') + +if not ADAPTER_FLAVOR_DIR: + ADAPTER_FLAVOR_DIR = os.path.join(CONFIG_DIR, 'flavor') + +if not VALIDATOR_DIR: + VALIDATOR_DIR = os.path.join(CONFIG_DIR, 'validator') + +if not CALLBACK_DIR: + CALLBACK_DIR = os.path.join(CONFIG_DIR, 'callback') + +if not TMPL_DIR: + TMPL_DIR = os.path.join(CONFIG_DIR, 'templates') + +if not MACHINE_LIST_DIR: + MACHINE_LIST_DIR = os.path.join(CONFIG_DIR, 'machine_list') + +if not PROGRESS_CALCULATOR_DIR: + PROGRESS_CALCULATOR_DIR = os.path.join(CONFIG_DIR, 'progress_calculator') + +if not OS_MAPPING_DIR: + OS_MAPPING_DIR = os.path.join(CONFIG_DIR, 'os_mapping') + +if not FLAVOR_MAPPING_DIR: + FLAVOR_MAPPING_DIR = os.path.join(CONFIG_DIR, 'flavor_mapping') + +if not PLUGINS_DIR: + PLUGINS_DIR = os.environ.get('COMPASS_PLUGINS_DIR', + os.path.join(CONFIG_DIR, 'plugins')) diff --git a/compass-tasks-base/utils/util.py b/compass-tasks-base/utils/util.py new file mode 100644 index 0000000..39978ca --- /dev/null +++ b/compass-tasks-base/utils/util.py @@ -0,0 +1,395 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to provider util functions in all compass code + + .. moduleauthor:: Xiaodong Wang +""" + +import crypt +import datetime +import logging +import os +import os.path +import re +import setting_wrapper as setting +import sys +import warnings + + +def deprecated(func): + """This is a decorator which can be used to mark functions as deprecated. + + It will result in a warning being emitted when the function is used. + """ + def new_func(*args, **kwargs): + warnings.warn( + "Call to deprecated function %s." % func.__name__, + category=DeprecationWarning + ) + return func(*args, **kwargs) + + new_func.__name__ = func.__name__ + new_func.__doc__ = func.__doc__ + new_func.__dict__.update(func.__dict__) + return new_func + + +def parse_datetime(date_time, exception_class=Exception): + """Parse datetime str to get datetime object. + + The date time format is %Y-%m-%d %H:%M:%S + """ + try: + return datetime.datetime.strptime( + date_time, '%Y-%m-%d %H:%M:%S' + ) + except Exception as error: + logging.exception(error) + raise exception_class( + 'date time %s format is invalid' % date_time + ) + + +def parse_datetime_range(date_time_range, exception_class=Exception): + """parse datetime range str to pair of datetime objects. + + The date time range format is %Y-%m-%d %H:%M:%S,%Y-%m-%d %H:%M:%S + """ + try: + start, end = date_time_range.split(',') + except Exception as error: + logging.exception(error) + raise exception_class( + 'there is no `,` in date time range %s' % date_time_range + ) + if start: + start_datetime = parse_datetime(start, exception_class) + else: + start_datetime = None + if end: + end_datetime = parse_datetime(end, exception_class) + else: + end_datetime = None + return start_datetime, end_datetime + + +def parse_request_arg_dict(arg, exception_class=Exception): + """parse string to dict. + + The str is formatted like a=b;c=d and parsed to + {'a': 'b', 'c': 'd'} + """ + arg_dict = {} + arg_pairs = arg.split(';') + for arg_pair in arg_pairs: + try: + arg_name, arg_value = arg_pair.split('=', 1) + except Exception as error: + logging.exception(error) + raise exception_class( + 'there is no `=` in %s' % arg_pair + ) + arg_dict[arg_name] = arg_value + return arg_dict + + +def format_datetime(date_time): + """Generate string from datetime object.""" + return date_time.strftime("%Y-%m-%d %H:%M:%S") + + +def merge_dict(lhs, rhs, override=True): + """Merge nested right dict into left nested dict recursively. + + :param lhs: dict to be merged into. + :type lhs: dict + :param rhs: dict to merge from. + :type rhs: dict + :param override: the value in rhs overide the value in left if True. + :type override: boolean + """ + if not isinstance(lhs, dict) or not isinstance(rhs, dict): + if override: + return rhs + else: + return lhs + + for key, value in rhs.items(): + if key not in lhs: + lhs[key] = rhs[key] + else: + lhs[key] = merge_dict(lhs[key], value, override) + + return lhs + + +def recursive_merge_dict(name, all_dicts, parents): + """Recursively merge parent dict into base dict.""" + parent_name = parents.get(name, None) + base_dict = all_dicts.get(name, {}) + if not parent_name: + return base_dict + merged = recursive_merge_dict(parent_name, all_dicts, parents) + return merge_dict(base_dict, merged, override=False) + + +def encrypt(value, crypt_method=None): + """Get encrypted value.""" + if not crypt_method: + if hasattr(crypt, 'METHOD_MD5'): + crypt_method = crypt.METHOD_MD5 + else: + # for python2.7, copy python2.6 METHOD_MD5 logic here. + from random import choice + import string + + _saltchars = string.ascii_letters + string.digits + './' + + def _mksalt(): + """generate salt.""" + salt = '$1$' + salt += ''.join(choice(_saltchars) for _ in range(8)) + return salt + + crypt_method = _mksalt() + + return crypt.crypt(value, crypt_method) + + +def parse_time_interval(time_interval_str): + """parse string of time interval to time interval. + + supported time interval unit: ['d', 'w', 'h', 'm', 's'] + Examples: + time_interval_str: '3d 2h' time interval to 3 days and 2 hours. + """ + if not time_interval_str: + return 0 + + time_interval_tuple = [ + time_interval_element + for time_interval_element in time_interval_str.split(' ') + if time_interval_element + ] + time_interval_dict = {} + time_interval_unit_mapping = { + 'd': 'days', + 'w': 'weeks', + 'h': 'hours', + 'm': 'minutes', + 's': 'seconds' + } + for time_interval_element in time_interval_tuple: + mat = re.match(r'^([+-]?\d+)(w|d|h|m|s).*', time_interval_element) + if not mat: + continue + + time_interval_value = int(mat.group(1)) + time_interval_unit = time_interval_unit_mapping[mat.group(2)] + time_interval_dict[time_interval_unit] = ( + time_interval_dict.get(time_interval_unit, 0) + time_interval_value + ) + + time_interval = datetime.timedelta(**time_interval_dict) + if sys.version_info[0:2] > (2, 6): + return time_interval.total_seconds() + else: + return ( + time_interval.microseconds + ( + time_interval.seconds + time_interval.days * 24 * 3600 + ) * 1e6 + ) / 1e6 + + +def get_plugins_config_files(name, suffix=".conf"): + """walk through each of plugin to find all the config files in the""" + """name directory""" + + plugins_path = setting.PLUGINS_DIR + files = [] + if os.path.exists(plugins_path): + for plugin in os.listdir(plugins_path): + plugin_path = os.path.join(plugins_path, plugin) + plugin_config = os.path.join(plugin_path, name) + if os.path.exists(plugin_config): + for component in os.listdir(plugin_config): + if not component.endswith(suffix): + continue + files.append(os.path.join(plugin_config, component)) + return files + + +def load_configs( + config_dir, config_name_suffix='.conf', + env_globals={}, env_locals={} +): + """Load configurations from config dir.""" + """The config file could be in the config_dir or in plugins config_dir""" + """The plugins config_dir is formed as, for example /etc/compass/adapter""" + """Then the plugins config_dir is /etc/compass/plugins/xxx/adapter""" + + # TODO(Carl) instead of using config_dir, it should use a name such as + # adapter etc, however, doing it requires a lot client sites changes, + # will do it later. + + configs = [] + config_files = [] + config_dir = str(config_dir) + + """search for config_dir""" + if os.path.exists(config_dir): + for component in os.listdir(config_dir): + if not component.endswith(config_name_suffix): + continue + config_files.append(os.path.join(config_dir, component)) + + """search for plugins config_dir""" + index = config_dir.rfind("/") + + config_files.extend(get_plugins_config_files(config_dir[index + 1:], + config_name_suffix)) + + if not config_files: + logging.error('path %s and plugins does not exist', config_dir) + for path in config_files: + logging.debug('load config from %s', path) + config_globals = {} + config_globals.update(env_globals) + config_locals = {} + config_locals.update(env_locals) + try: + execfile(path, config_globals, config_locals) + except Exception as error: + logging.exception(error) + raise error + configs.append(config_locals) + return configs + + +def pretty_print(*contents): + """pretty print contents.""" + if len(contents) == 0: + print "" + else: + print "\n".join(content for content in contents) + + +def get_switch_machines_from_file(filename): + """get switch machines from file.""" + switches = [] + switch_machines = {} + with open(filename) as switch_file: + for line in switch_file: + line = line.strip() + if not line: + # ignore empty line + continue + + if line.startswith('#'): + # ignore comments + continue + + columns = [column for column in line.split(',')] + if not columns: + # ignore empty line + continue + + if columns[0] == 'switch': + (switch_ip, switch_vendor, switch_version, + switch_community, switch_state) = columns[1:] + switches.append({ + 'ip': switch_ip, + 'vendor': switch_vendor, + 'credentials': { + 'version': switch_version, + 'community': switch_community, + }, + 'state': switch_state, + }) + elif columns[0] == 'machine': + switch_ip, switch_port, mac = columns[1:] + switch_machines.setdefault(switch_ip, []).append({ + 'mac': mac, + 'port': switch_port, + }) + + return (switches, switch_machines) + + +def execute_cli_by_ssh(cmd, host, username, password=None, + keyfile='/root/.ssh/id_rsa', nowait=False): + """SSH to execute script on remote machine + + :param host: ip of the remote machine + :param username: username to access the remote machine + :param password: password to access the remote machine + :param cmd: command to execute + + """ + if not cmd: + logging.error("No command found!") + raise Exception('No command found!') + + if nowait: + cmd = "nohup %s >/dev/null 2>&1 &" % cmd + + stdin = None + stdout = None + stderr = None + try: + import paramiko + from paramiko import ssh_exception + + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + if password: + client.connect(host, username=username, password=password) + else: + client.load_system_host_keys() + client.connect( + host, username=username, + key_filename=keyfile, look_for_keys=True + ) + stdin, stdout, stderr = client.exec_command(cmd) + result = stdout.readlines() + logging.info("result of command '%s' is '%s'!" % (cmd, result)) + return result + + except ImportError: + err_msg = "Cannot find Paramiko package!" + logging.error(err_msg) + raise ImportError(err_msg) + + except (ssh_exception.BadHostKeyException, + ssh_exception.AuthenticationException, + ssh_exception.SSHException): + + err_msg = 'SSH connection error or command execution failed!' + logging.error(err_msg) + raise Exception(err_msg) + + except Exception as exc: + logging.error( + 'Failed to execute command "%s", exception is %s' % (cmd, exc) + ) + raise Exception(exc) + + finally: + for resource in [stdin, stdout, stderr]: + if resource: + resource.close() + + client.close() diff --git a/compass-tasks-k8s/Dockerfile b/compass-tasks-k8s/Dockerfile deleted file mode 100644 index cd74cf8..0000000 --- a/compass-tasks-k8s/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM opnfv/compass-tasks:latest -#FROM localbuild/compass-tasks - -ARG BRANCH=master - -ADD ./run.sh /root/ -RUN chmod +x /root/run.sh -RUN /root/run.sh diff --git a/compass-tasks-k8s/README.md b/compass-tasks-k8s/README.md deleted file mode 100644 index 2dab2ee..0000000 --- a/compass-tasks-k8s/README.md +++ /dev/null @@ -1 +0,0 @@ -# compass-tasks-k8s diff --git a/compass-tasks-k8s/run.sh b/compass-tasks-k8s/run.sh deleted file mode 100644 index 516ead0..0000000 --- a/compass-tasks-k8s/run.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -############################################################################## -# Copyright (c) 2016-2017 HUAWEI TECHNOLOGIES CO.,LTD and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -set -x -COMPASS_DIR=${BASH_SOURCE[0]%/*} - -rm -rf /opt/kargo_k8s -git clone https://github.com/kubernetes-incubator/kubespray.git /opt/kargo_k8s - -cd /opt/kargo_k8s -git checkout v2.2.0 - -pip uninstall ansible -y - -pip install ansible==2.3.1.0 - - - - diff --git a/compass-tasks-osa/Dockerfile b/compass-tasks-osa/Dockerfile deleted file mode 100644 index 77cf585..0000000 --- a/compass-tasks-osa/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM opnfv/compass-tasks:latest -#FROM localbuild/compass-tasks -ARG BRANCH=master - -ADD ./run.sh /root/ -ADD ./tacker_conf /opt/tacker_conf -RUN chmod +x /root/run.sh -RUN /root/run.sh diff --git a/compass-tasks-osa/run.sh b/compass-tasks-osa/run.sh deleted file mode 100644 index fd6a5c9..0000000 --- a/compass-tasks-osa/run.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -yum install https://rdoproject.org/repos/openstack-ocata/rdo-release-ocata.rpm -y -yum install git ntp wget ntpdate openssh-server python-devel sudo '@Development Tools' -y - -mkdir -p /opt/git/ -cd /opt/git/ -wget artifacts.opnfv.org/compass4nfv/package/openstack.tar.gz -tar -zxvf openstack.tar.gz -rm -rf openstack.tar.gz -cd openstack -git clone https://github.com/openstack/tacker.git -b stable/ocata -cd tacker -git checkout a0f1e680d81c7db66ae7a2a08c3d069901d0765a - - -git clone https://git.openstack.org/openstack/openstack-ansible /opt/openstack-ansible - -cd /opt/openstack-ansible - -git checkout 7beba50a8345616ef27c70cbbcac962b56b8adc5 - -/bin/cp -rf /opt/tacker_conf/ansible-role-requirements.yml /opt/openstack-ansible/ -/bin/cp -rf /opt/tacker_conf/openstack_services.yml /opt/openstack-ansible/playbooks/defaults/repo_packages/ -/bin/cp -rf /opt/tacker_conf/os-tacker-install.yml /opt/openstack-ansible/playbooks/ -/bin/cp -rf /opt/tacker_conf/setup-openstack.yml /opt/openstack-ansible/playbooks/ -/bin/cp -rf /opt/tacker_conf/tacker.yml /opt/openstack-ansible/playbooks/inventory/env.d/ -/bin/cp -rf /opt/tacker_conf/tacker_all.yml /opt/openstack-ansible/playbooks/inventory/group_vars/ -/bin/cp -rf /opt/tacker_conf/user_secrets.yml /opt/openstack-ansible/etc/openstack_deploy/ -/bin/cp -rf /opt/tacker_conf/haproxy_config.yml /opt/openstack-ansible/playbooks/vars/configs/ - -/bin/cp -rf /opt/openstack-ansible/etc/openstack_deploy /etc/openstack_deploy - -cd /opt/openstack-ansible - -scripts/bootstrap-ansible.sh - -rm -f /usr/local/bin/ansible-playbook - -cd /opt/openstack-ansible/scripts/ -python pw-token-gen.py --file /etc/openstack_deploy/user_secrets.yml - -cd /opt/openstack-ansible/playbooks/inventory/group_vars -sed -i 's/#repo_build_git_cache/repo_build_git_cache/g' repo_all.yml diff --git a/compass-tasks-osa/tacker_conf/ansible-role-requirements.yml b/compass-tasks-osa/tacker_conf/ansible-role-requirements.yml deleted file mode 100644 index 40e0f91..0000000 --- a/compass-tasks-osa/tacker_conf/ansible-role-requirements.yml +++ /dev/null @@ -1,192 +0,0 @@ -- name: apt_package_pinning - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-apt_package_pinning - version: d5c0d7e8d21fc1539cc043097e23851296e2dd95 -- name: pip_install - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-pip_install - version: e142f5642646449d98cd65d8fd0690907c403801 -- name: galera_client - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-galera_client - version: 1d3cdcd33c75a668ac3be046ac53fe1842780058 -- name: galera_server - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-galera_server - version: e47877adc99bf64b900c83056716dfc09f2bcc0b -- name: ceph_client - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-ceph_client - version: d1b30ae7b660acbb4f9d6e4f4ab702581c6748d9 -- name: haproxy_server - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-haproxy_server - version: ffba50ecf0c99f41d1d5a1195268a2c23c38f89f -- name: keepalived - scm: git - src: https://github.com/evrardjp/ansible-keepalived - version: 2.6.0 -- name: lxc_container_create - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-lxc_container_create - version: 540bd82f8ded168ccab25a4a762a5a88e5a4ecbd -- name: lxc_hosts - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-lxc_hosts - version: 73a62894aa26e4b63bb69336411209fc0df6abb2 -- name: memcached_server - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-memcached_server - version: ddd1dc402b7c825265a9e59483272f2dfcd65a93 -- name: openstack-ansible-security - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-security - version: c58c839f9cc9fcb75d32fc8de00b4f27add36661 -- name: openstack_hosts - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-openstack_hosts - version: 1ba80641ed5c4d55a41906e9d37df10b9fd720ee -- name: os_keystone - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_keystone - version: 600e0e58102ccbdd082b5c90237cb7d27c63e5fa -- name: openstack_openrc - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-openstack_openrc - version: dfdf65b3db7c6240fa173a3a62af4e5028023626 -- name: os_aodh - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_aodh - version: a1ec96ee54d7b8c7fd24f1148545d3556a0a8b5b -- name: os_barbican - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_barbican - version: 2d087176610c712f51dec8722b6129e648b149e2 -- name: os_ceilometer - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_ceilometer - version: 1d709bd6eace68bfd7fb7d1e52df8d5b6f28810b -- name: os_cinder - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_cinder - version: 66542db4c7045dc559a1b3e6bd293ca28e09c34e -- name: os_glance - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_glance - version: 79d6b78e894715ba3a17b50cc18917b251025069 -- name: os_gnocchi - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_gnocchi - version: 5932746fe1717ef2798b46c374c268945f34814e -- name: os_heat - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_heat - version: cbe917fefeb90c506e1c73c31b9a1d00fbf9beff -- name: os_horizon - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_horizon - version: 25435372547d217116dd4ba10b5d0e8c39307b52 -- name: os_ironic - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_ironic - version: 4110fca2b9e1c6a1698b9be9797253db4a1cad2e -- name: os_magnum - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_magnum - version: f452cc54d1b3240f81a82fd110a467cd996a3331 -- name: os_trove - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_trove - version: 51b38916ad99f3ffbc1723a7d97f9034b4c02dc4 -- name: os_neutron - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_neutron - version: 4b43221adae78627d8035cd82e791662821a461c -- name: os_nova - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_nova - version: 8aacdd01ad98d82b3b817895d39af44b3a862847 -- name: os_rally - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_rally - version: 1410d71f60aa354259fa11d9dddfe4ed743177ab -- name: os_sahara - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_sahara - version: 5e29921df9cc7d3ba2a89b0d6cdddcf02553c423 -- name: os_swift - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_swift - version: 70c3795215b5b3ea90090ffd99381c97bc45be80 -- name: os_tacker - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_tacker - version: c6a563731b4482a28b7a01982252d79f52fe586b -- name: os_tempest - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_tempest - version: 5934f81385870cc836f7d1cd84b8dbc6a97cedad -- name: plugins - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-plugins - version: 9ce61bdc6cb537c2377e95da1c3bfc9e837c9784 -- name: rabbitmq_server - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-rabbitmq_server - version: 3d1de52f75c08b3265d2b35c56c9217d60c2c840 -- name: repo_build - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-repo_build - version: 6d00b7d6f431fc8706be81e1089ee351d2172e48 -- name: repo_server - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-repo_server - version: 860ff80b348f1c9e93745da55b1df69d05283afb -- name: rsyslog_client - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-rsyslog_client - version: 5f5275720d13bd22945348751131c39877b731ab -- name: rsyslog_server - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-rsyslog_server - version: a7d4b0df2e9ed3c2a1414613e96c7c9db0f2db60 -- name: sshd - scm: git - src: https://github.com/willshersystems/ansible-sshd - version: 0.4.5 -- name: bird - scm: git - src: https://github.com/logan2211/ansible-bird - version: '1.2' -- name: etcd - scm: git - src: https://github.com/logan2211/ansible-etcd - version: '1.2' -- name: unbound - scm: git - src: https://github.com/logan2211/ansible-unbound - version: '1.3' -- name: resolvconf - scm: git - src: https://github.com/logan2211/ansible-resolvconf - version: '1.2' -- name: os_designate - scm: git - src: https://git.openstack.org/openstack/openstack-ansible-os_designate - version: fc4f0fb40197954c96f486f4ebbd679162b27a6d -- name: ceph.ceph-common - scm: git - src: https://github.com/ceph/ansible-ceph-common - version: v2.2.4 -- name: ceph.ceph-docker-common - scm: git - src: https://github.com/ceph/ansible-ceph-docker-common - version: ca86fd0ef6d24aa2c750a625acdcb8012c374aa0 -- name: ceph-mon - scm: git - src: https://github.com/ceph/ansible-ceph-mon - version: v2.2.4 -- name: ceph-osd - scm: git - src: https://github.com/ceph/ansible-ceph-osd - version: v2.2.4 diff --git a/compass-tasks-osa/tacker_conf/haproxy_config.yml b/compass-tasks-osa/tacker_conf/haproxy_config.yml deleted file mode 100644 index bbac6d7..0000000 --- a/compass-tasks-osa/tacker_conf/haproxy_config.yml +++ /dev/null @@ -1,285 +0,0 @@ -# Copyright 2014, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -haproxy_default_services: - - service: - haproxy_service_name: galera - haproxy_backend_nodes: "{{ [groups['galera_all'][0]] | default([]) }}" # list expected - haproxy_backup_nodes: "{{ groups['galera_all'][1:] | default([]) }}" - haproxy_bind: "{{ [internal_lb_vip_address] }}" - haproxy_port: 3306 - haproxy_balance_type: tcp - haproxy_timeout_client: 5000s - haproxy_timeout_server: 5000s - haproxy_backend_options: - - "mysql-check user {{ galera_monitoring_user }}" - haproxy_whitelist_networks: "{{ haproxy_galera_whitelist_networks }}" - - service: - haproxy_service_name: repo_git - haproxy_backend_nodes: "{{ groups['repo_all'] | default([]) }}" - haproxy_bind: "{{ [internal_lb_vip_address] }}" - haproxy_port: 9418 - haproxy_balance_type: tcp - haproxy_backend_options: - - tcp-check - haproxy_whitelist_networks: "{{ haproxy_repo_git_whitelist_networks }}" - - service: - haproxy_service_name: repo_all - haproxy_backend_nodes: "{{ groups['repo_all'] | default([]) }}" - haproxy_bind: "{{ [internal_lb_vip_address] }}" - haproxy_port: 8181 - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk HEAD /" - - service: - haproxy_service_name: repo_cache - haproxy_backend_nodes: "{{ [groups['repo_all'][0]] | default([]) }}" # list expected - haproxy_backup_nodes: "{{ groups['repo_all'][1:] | default([]) }}" - haproxy_bind: "{{ [internal_lb_vip_address] }}" - haproxy_port: "{{ repo_pkg_cache_port }}" - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk HEAD /acng-report.html" - haproxy_whitelist_networks: "{{ haproxy_repo_cache_whitelist_networks }}" - - service: - haproxy_service_name: glance_api - haproxy_backend_nodes: "{{ groups['glance_api'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_port: 9292 - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk /healthcheck" - - service: - haproxy_service_name: glance_registry - haproxy_backend_nodes: "{{ groups['glance_registry'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_port: 9191 - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk /healthcheck" - haproxy_whitelist_networks: "{{ haproxy_glance_registry_whitelist_networks }}" - - service: - haproxy_service_name: gnocchi - haproxy_backend_nodes: "{{ groups['gnocchi_all'] | default([]) }}" - haproxy_port: 8041 - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk /healthcheck" - - service: - haproxy_service_name: heat_api_cfn - haproxy_backend_nodes: "{{ groups['heat_api_cfn'] | default([]) }}" - haproxy_port: 8000 - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk HEAD /" - - service: - haproxy_service_name: heat_api_cloudwatch - haproxy_backend_nodes: "{{ groups['heat_api_cloudwatch'] | default([]) }}" - haproxy_port: 8003 - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk HEAD /" - - service: - haproxy_service_name: heat_api - haproxy_backend_nodes: "{{ groups['heat_api'] | default([]) }}" - haproxy_port: 8004 - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk HEAD /" - - service: - haproxy_service_name: keystone_service - haproxy_backend_nodes: "{{ groups['keystone_all'] | default([]) }}" - haproxy_port: 5000 - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_balance_type: "http" - haproxy_backend_options: - - "httpchk HEAD /" - - service: - haproxy_service_name: keystone_admin - haproxy_backend_nodes: "{{ groups['keystone_all'] | default([]) }}" - haproxy_port: 35357 - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_balance_type: "http" - haproxy_backend_options: - - "httpchk HEAD /" - haproxy_whitelist_networks: "{{ haproxy_keystone_admin_whitelist_networks }}" - - service: - haproxy_service_name: neutron_server - haproxy_backend_nodes: "{{ groups['neutron_server'] | default([]) }}" - haproxy_port: 9696 - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk HEAD /" - - service: - haproxy_service_name: nova_api_metadata - haproxy_backend_nodes: "{{ groups['nova_api_metadata'] | default([]) }}" - haproxy_port: 8775 - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk HEAD /" - haproxy_whitelist_networks: "{{ haproxy_nova_metadata_whitelist_networks }}" - - service: - haproxy_service_name: nova_api_os_compute - haproxy_backend_nodes: "{{ groups['nova_api_os_compute'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_port: 8774 - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk HEAD /" - - service: - haproxy_service_name: nova_api_placement - haproxy_backend_nodes: "{{ groups['nova_api_placement'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_port: 8780 - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk HEAD /" - - service: - haproxy_service_name: nova_console - haproxy_backend_nodes: "{{ groups['nova_console'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_port: "{{ nova_console_port }}" - haproxy_balance_type: http - haproxy_timeout_client: 60m - haproxy_timeout_server: 60m - haproxy_balance_alg: source - haproxy_backend_options: - - "httpchk HEAD /" - haproxy_backend_httpcheck_options: - - "expect status 404" - - service: - haproxy_service_name: cinder_api - haproxy_backend_nodes: "{{ groups['cinder_api'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_port: 8776 - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk HEAD /" - - service: - haproxy_service_name: horizon - haproxy_backend_nodes: "{{ groups['horizon_all'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_ssl_all_vips: true - haproxy_port: "{{ haproxy_ssl | ternary(443,80) }}" - haproxy_backend_port: 80 - haproxy_redirect_http_port: 80 - haproxy_balance_type: http - haproxy_balance_alg: source - haproxy_backend_options: - - "httpchk HEAD /" - - service: - haproxy_service_name: sahara_api - haproxy_backend_nodes: "{{ groups['sahara_api'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_balance_alg: source - haproxy_port: 8386 - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk /healthcheck" - - service: - haproxy_service_name: swift_proxy - haproxy_backend_nodes: "{{ groups['swift_proxy'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_balance_alg: source - haproxy_port: 8080 - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk /healthcheck" - - service: - haproxy_service_name: ceilometer_api - haproxy_backend_nodes: "{{ groups['ceilometer_api_container'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_port: 8777 - haproxy_balance_type: tcp - haproxy_backend_options: - - tcp-check - - service: - haproxy_service_name: aodh_api - haproxy_backend_nodes: "{{ groups['aodh_api'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_port: 8042 - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk HEAD /" - haproxy_backend_httpcheck_options: - - "expect status 401" - - service: - haproxy_service_name: ironic_api - haproxy_backend_nodes: "{{ groups['ironic_api'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_port: 6385 - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk GET /" - - service: - haproxy_service_name: rabbitmq_mgmt - haproxy_backend_nodes: "{{ groups['rabbitmq'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_port: 15672 - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk HEAD /" - haproxy_whitelist_networks: "{{ haproxy_rabbitmq_management_whitelist_networks }}" - - service: - haproxy_service_name: magnum - haproxy_backend_nodes: "{{ groups['magnum_all'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_port: 9511 - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk GET /" - - service: - haproxy_service_name: trove - haproxy_backend_nodes: "{{ groups['trove_api'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_port: 8779 - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk HEAD /" - - service: - haproxy_service_name: barbican - haproxy_backend_nodes: "{{ groups['barbican_api'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_port: 9311 - haproxy_balance_type: http - haproxy_backend_options: - - "httpchk HEAD /" - haproxy_backend_httpcheck_options: - - "expect status 401" - - service: - haproxy_service_name: designate_api - haproxy_backend_nodes: "{{ groups['designate_api'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_port: 9001 - haproxy_balance_type: http - haproxy_backend_options: - - "forwardfor" - - "httpchk /versions" - - "httplog" - - service: - haproxy_service_name: tacker - haproxy_backend_nodes: "{{ groups['tacker_all'] | default([]) }}" - haproxy_ssl: "{{ haproxy_ssl }}" - haproxy_port: 9890 - haproxy_balance_type: http - haproxy_backend_options: - - "forwardfor" - - "httpchk" - - "httplog" diff --git a/compass-tasks-osa/tacker_conf/openstack_services.yml b/compass-tasks-osa/tacker_conf/openstack_services.yml deleted file mode 100644 index 5a772e5..0000000 --- a/compass-tasks-osa/tacker_conf/openstack_services.yml +++ /dev/null @@ -1,228 +0,0 @@ ---- -# Copyright 2014, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -## NOTICE on items in this file: -## * If you use anything in the *._git_install_branch field that is not a TAG -## make sure to leave an in-line comment as to "why". - -## For the sake of anyone else editing this file: -## * If you add services to this file please do so in alphabetical order. -## * Every entry should be name spaced with the name of the client followed by an "_" -## * All items with this file should be separated by `name_` note that the name of the -## package should be one long name with no additional `_` separating it. - - -### Before this is shipped all of these services should have a tag set as the branch, -### or have a comment / reason attached to them as to why a tag can not work. - - -## Global Requirements -requirements_git_repo: https://git.openstack.org/openstack/requirements -requirements_git_install_branch: 90094c5d578ecfc4ab1e9f38a86bca5b615a3527 # HEAD of "stable/ocata" as of 15.05.2017 -requirements_git_dest: "/opt/requirements_{{ requirements_git_install_branch | replace('/', '_') }}" - - -## Aodh service -aodh_git_repo: https://git.openstack.org/openstack/aodh -aodh_git_install_branch: 5c6b46ecc85743798c04b2d5fd094a0e9ef661a7 # HEAD of "stable/ocata" as of 15.05.2017 -aodh_git_dest: "/opt/aodh_{{ aodh_git_install_branch | replace('/', '_') }}" -aodh_git_project_group: aodh_all - - -## Barbican service -barbican_git_repo: https://git.openstack.org/openstack/barbican -barbican_git_install_branch: b04aa152843d9cda3b11e8736e740c71fd798d0f # HEAD of "stable/ocata" as of 15.05.2017 -barbican_git_dest: "/opt/barbican_{{ barbican_git_install_branch | replace('/', '_') }}" -barbican_git_project_group: barbican_all - - -## Ceilometer service -ceilometer_git_repo: https://git.openstack.org/openstack/ceilometer -ceilometer_git_install_branch: c25e934b8c48b00889f8fa6bb3e5bcb9591df618 # HEAD of "stable/ocata" as of 15.05.2017 -ceilometer_git_dest: "/opt/ceilometer_{{ceilometer_git_install_branch | replace('/', '_') }}" -ceilometer_git_project_group: ceilometer_all - - -## Cinder service -cinder_git_repo: https://git.openstack.org/openstack/cinder -cinder_git_install_branch: 501485b25087ff398986aedbbdb893741f8aabae # HEAD of "stable/ocata" as of 15.05.2017 -cinder_git_dest: "/opt/cinder_{{ cinder_git_install_branch | replace('/', '_') }}" -cinder_git_project_group: cinder_all - - -## Designate service -designate_git_repo: https://git.openstack.org/openstack/designate -designate_git_install_branch: 021aa52325c2b58be6317a014cb3fd44be02f29f # HEAD of "stable/ocata" as of 15.05.2017 -designate_git_dest: "/opt/designate_{{ designate_git_install_branch | replace('/', '_') }}" -designate_git_project_group: designate_all - - -## Horizon Designate dashboard plugin -designate_dashboard_git_repo: https://git.openstack.org/openstack/designate-dashboard -designate_dashboard_git_install_branch: fb8108354937ef6d3a1bb389481af79681f7c5b3 # HEAD of "stable/ocata" as of 15.05.2017 -designate_dashboard_git_dest: "/opt/designate_dashboard_{{ designate_dashboard_git_install_branch | replace('/', '_') }}" -designate_dashboard_git_project_group: horizon_all - - -## Dragonflow service -dragonflow_git_repo: https://git.openstack.org/openstack/dragonflow -dragonflow_git_install_branch: b72ffa06721faf95f42c9702f60a9acc3a7d8b61 # HEAD of "stable/ocata" as of 15.05.2017 -dragonflow_git_dest: "/opt/dragonflow_{{ dragonflow_git_install_branch | replace('/', '_') }}" -dragonflow_git_project_group: neutron_all - - -## Glance service -glance_git_repo: https://git.openstack.org/openstack/glance -glance_git_install_branch: 57c4d7d78f37e840660719b944ebabe91cbf231b # HEAD of "stable/ocata" as of 15.05.2017 -glance_git_dest: "/opt/glance_{{ glance_git_install_branch | replace('/', '_') }}" -glance_git_project_group: glance_all - - -## Heat service -heat_git_repo: https://git.openstack.org/openstack/heat -heat_git_install_branch: fe6ae385bc7bba77864b32061600ac40b24f2a10 # HEAD of "stable/ocata" as of 15.05.2017 -heat_git_dest: "/opt/heat_{{ heat_git_install_branch | replace('/', '_') }}" -heat_git_project_group: heat_all - - -## Horizon service -horizon_git_repo: https://git.openstack.org/openstack/horizon -horizon_git_install_branch: 2684552f09ee5d597a507833cf232f0f9d48d9b1 # HEAD of "stable/ocata" as of 15.05.2017 -horizon_git_dest: "/opt/horizon_{{ horizon_git_install_branch | replace('/', '_') }}" -horizon_git_project_group: horizon_all - -## Horizon Ironic dashboard plugin -ironic_dashboard_git_repo: https://git.openstack.org/openstack/ironic-ui -ironic_dashboard_git_install_branch: 6a0cdbff587df20ae913ca7f0d434ad8381b761c # HEAD of "stable/ocata" as of 15.05.2017 -ironic_dashboard_git_dest: "/opt/ironic_dashboard_{{ ironic_dashboard_git_install_branch | replace('/', '_') }}" -ironic_dashboard_git_project_group: horizon_all - -## Horizon Magnum dashboard plugin -magnum_dashboard_git_repo: https://git.openstack.org/openstack/magnum-ui -magnum_dashboard_git_install_branch: db3bfff643ceb9353c7c0cafdd4e15e0fcf01fb1 # HEAD of "stable/ocata" as of 15.05.2017 -magnum_dashboard_git_dest: "/opt/magnum_dashboard_{{ magnum_dashboard_git_install_branch | replace('/', '_') }}" -magnum_dashboard_git_project_group: horizon_all - -## Horizon LBaaS dashboard plugin -neutron_lbaas_dashboard_git_repo: https://git.openstack.org/openstack/neutron-lbaas-dashboard -neutron_lbaas_dashboard_git_install_branch: 3653432bb8287fa0ec1248d866e4717b90ab824b # HEAD of "stable/ocata" as of 15.05.2017 -neutron_lbaas_dashboard_git_dest: "/opt/neutron_lbaas_dashboard_{{ neutron_lbaas_dashboard_git_install_branch | replace('/', '_') }}" -neutron_lbaas_dashboard_git_project_group: horizon_all - -## Horizon Sahara dashboard plugin -sahara_dashboard_git_repo: https://git.openstack.org/openstack/sahara-dashboard -sahara_dashboard_git_install_branch: 5270e94b168b3c20e5c787a3f43d44b7e101c448 # HEAD of "stable/ocata" as of 15.05.2017 -sahara_dashboard_git_dest: "/opt/sahara_dashboard_{{ sahara_dashboard_git_install_branch | replace('/', '_') }}" -sahara_dashboard_git_project_group: horizon_all - - -## Keystone service -keystone_git_repo: https://git.openstack.org/openstack/keystone -keystone_git_install_branch: 955fd6ca3758e217d9d98480852e0014dc11e988 # HEAD of "stable/ocata" as of 15.05.2017 -keystone_git_dest: "/opt/keystone_{{ keystone_git_install_branch | replace('/', '_') }}" -keystone_git_project_group: keystone_all - - -## Neutron service -neutron_git_repo: https://git.openstack.org/openstack/neutron -neutron_git_install_branch: 515a2ff4ce3239f6a077d8b07ff80544023f0631 # HEAD of "stable/ocata" as of 15.05.2017 -neutron_git_dest: "/opt/neutron_{{ neutron_git_install_branch | replace('/', '_') }}" -neutron_git_project_group: neutron_all - -neutron_lbaas_git_repo: https://git.openstack.org/openstack/neutron-lbaas -neutron_lbaas_git_install_branch: 674e3604a6285655f2dc7f2949629cc4de06bbf4 # HEAD of "stable/ocata" as of 15.05.2017 -neutron_lbaas_git_dest: "/opt/neutron_lbaas_{{ neutron_lbaas_git_install_branch | replace('/', '_') }}" -neutron_lbaas_git_project_group: neutron_all - -neutron_vpnaas_git_repo: https://git.openstack.org/openstack/neutron-vpnaas -neutron_vpnaas_git_install_branch: 4a639ac09ccfc7573452d0ac9d189d934c169d34 # HEAD of "stable/ocata" as of 15.05.2017 -neutron_vpnaas_git_dest: "/opt/neutron_vpnaas_{{ neutron_vpnaas_git_install_branch | replace('/', '_') }}" -neutron_vpnaas_git_project_group: neutron_all - -neutron_fwaas_git_repo: https://git.openstack.org/openstack/neutron-fwaas -neutron_fwaas_git_install_branch: 3301972f46d877d0f68d9cf7f9246e0df897a91e # HEAD of "stable/ocata" as of 15.05.2017 -neutron_fwaas_git_dest: "/opt/neutron_fwaas_{{ neutron_fwaas_git_install_branch | replace('/', '_') }}" -neutron_fwaas_git_project_group: neutron_all - -neutron_dynamic_routing_git_repo: https://git.openstack.org/openstack/neutron-dynamic-routing -neutron_dynamic_routing_git_install_branch: da877412200e460fca34edaf56f84286557bd486 # HEAD of "stable/ocata" as of 15.05.2017 -neutron_dynamic_routing_git_dest: "/opt/neutron_dynamic_routing_{{ neutron_dynamic_routing_git_install_branch | replace('/', '_') }}" -neutron_dynamic_routing_git_project_group: neutron_all - - -## Nova service -nova_git_repo: https://git.openstack.org/openstack/nova -nova_git_install_branch: 506465a027dd1ba1c90949dc58297edae32da7e4 # HEAD of "stable/ocata" as of 15.05.2017 -nova_git_dest: "/opt/nova_{{ nova_git_install_branch | replace('/', '_') }}" -nova_git_project_group: nova_all - - -## PowerVM Virt Driver -nova_powervm_git_repo: https://git.openstack.org/openstack/nova-powervm -nova_powervm_git_install_branch: 53a5c75a0db38b606f51a53fd159b7ffab7c3a33 # HEAD of "stable/ocata" as of 15.05.2017 -nova_powervm_git_dest: "/opt/nova_powervm_{{ nova_powervm_git_install_branch | replace('/', '_') }}" -nova_powervm_git_project_group: nova_compute - - -## LXD Virt Driver -nova_lxd_git_repo: https://git.openstack.org/openstack/nova-lxd -nova_lxd_git_install_branch: ed05fa417c4a78970dd5bdcdd3e1922f3c07f0ac # HEAD of "stable/ocata" as of 15.05.2017 -nova_lxd_git_dest: "/opt/nova_lxd_{{ nova_lxd_git_install_branch | replace('/', '_') }}" -nova_lxd_git_project_group: nova_compute - - -## Sahara service -sahara_git_repo: https://git.openstack.org/openstack/sahara -sahara_git_install_branch: 5241340d3e668bcb3b0e9f09c38afdc21e6c136d # HEAD of "stable/ocata" as of 15.05.2017 -sahara_git_dest: "/opt/sahara_{{ sahara_git_install_branch | replace('/', '_') }}" -sahara_git_project_group: sahara_all - - -## Swift service -swift_git_repo: https://git.openstack.org/openstack/swift -swift_git_install_branch: 177fca2b3a6a7a6e17e20d161c23ac8a10500939 # HEAD of "stable/ocata" as of 15.05.2017 -swift_git_dest: "/opt/swift_{{ swift_git_install_branch | replace('/', '_') }}" -swift_git_project_group: swift_all - - -## Ironic service -ironic_git_repo: https://git.openstack.org/openstack/ironic -ironic_git_install_branch: ed2f6cea892ccb2a1343dd935ad279de8fd3a471 # HEAD of "stable/ocata" as of 15.05.2017 -ironic_git_dest: "/opt/ironic_{{ ironic_git_install_branch | replace('/', '_') }}" -ironic_git_project_group: ironic_all - -## Magnum service -magnum_git_repo: https://git.openstack.org/openstack/magnum -magnum_git_install_branch: 1a685113d8df479c56ad85aa001930f6b8f1e2dd # HEAD of "stable/ocata" as of 15.05.2017 -magnum_git_dest: "/opt/magnum_{{ magnum_git_install_branch | replace('/', '_') }}" -magnum_git_project_group: magnum_all - -## Trove service -trove_git_repo: https://git.openstack.org/openstack/trove -trove_git_install_branch: ca0a1128f91aa5f54b9f57fedf2f5db4b6f82e20 # HEAD of "stable/ocata" as of 15.05.2017 -trove_git_dest: "/opt/trove_{{ trove_git_install_branch | replace('/', '_') }}" -trove_git_project_group: trove_all - -## Horizon Trove dashboard plugin -trove_dashboard_git_repo: https://git.openstack.org/openstack/trove-dashboard -trove_dashboard_git_install_branch: 5fd256e58219074b55cc9474a74516105787ac03 # HEAD of "stable/ocata" as of 15.05.2017 -trove_dashboard_git_dest: "/opt/trove_dashboard_{{ trove_dashboard_git_install_branch | replace('/', '_') }}" -trove_dashboard_git_project_group: horizon_all - -## Tacker service -tacker_git_repo: https://git.openstack.org/openstack/tacker -tacker_git_install_branch: a0f1e680d81c7db66ae7a2a08c3d069901d0765a # HEAD of "ocata" as of 04.09.2017 -tacker_git_project_group: tacker_all diff --git a/compass-tasks-osa/tacker_conf/os-tacker-install.yml b/compass-tasks-osa/tacker_conf/os-tacker-install.yml deleted file mode 100644 index dd96595..0000000 --- a/compass-tasks-osa/tacker_conf/os-tacker-install.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- -# Copyright 2017, SUSE LINUX GmbH. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Install the tacker components - hosts: tacker_all - gather_facts: "{{ gather_facts | default(True) }}" - max_fail_percentage: 20 - user: root - pre_tasks: - - include: common-tasks/os-lxc-container-setup.yml - - include: common-tasks/rabbitmq-vhost-user.yml - static: no - vars: - user: "{{ tacker_rabbitmq_userid }}" - password: "{{ tacker_rabbitmq_password }}" - vhost: "{{ tacker_rabbitmq_vhost }}" - _rabbitmq_host_group: "{{ tacker_rabbitmq_host_group }}" - when: - - inventory_hostname == groups['tacker_all'][0] - - groups[tacker_rabbitmq_host_group] | length > 0 - - include: common-tasks/os-log-dir-setup.yml - vars: - log_dirs: - - src: "/openstack/log/{{ inventory_hostname }}-tacker" - dest: "/var/log/tacker" - - include: common-tasks/mysql-db-user.yml - static: no - vars: - user_name: "{{ tacker_galera_user }}" - password: "{{ tacker_container_mysql_password }}" - login_host: "{{ tacker_galera_address }}" - db_name: "{{ tacker_galera_database }}" - when: inventory_hostname == groups['tacker_all'][0] - - include: common-tasks/package-cache-proxy.yml - roles: - - role: "os_tacker" - - role: "openstack_openrc" - tags: - - openrc - - role: "rsyslog_client" - rsyslog_client_log_rotate_file: tacker_log_rotate - rsyslog_client_log_dir: "/var/log/tacker" - rsyslog_client_config_name: "99-tacker-rsyslog-client.conf" - tags: - - rsyslog - vars: - is_metal: "{{ properties.is_metal|default(false) }}" - tacker_galera_address: "{{ internal_lb_vip_address }}" - environment: "{{ deployment_environment_variables | default({}) }}" - tags: - - tacker diff --git a/compass-tasks-osa/tacker_conf/setup-openstack.yml b/compass-tasks-osa/tacker_conf/setup-openstack.yml deleted file mode 100644 index d32eb15..0000000 --- a/compass-tasks-osa/tacker_conf/setup-openstack.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -# Copyright 2014, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- include: os-keystone-install.yml -- include: os-barbican-install.yml -- include: os-glance-install.yml -- include: os-cinder-install.yml -- include: os-nova-install.yml -- include: os-neutron-install.yml -- include: os-heat-install.yml -- include: os-horizon-install.yml -- include: os-ceilometer-install.yml -- include: os-aodh-install.yml -- include: os-designate-install.yml -#NOTE(stevelle) Ensure Gnocchi identities exist before Swift -- include: os-gnocchi-install.yml - when: - - gnocchi_storage_driver is defined - - gnocchi_storage_driver == 'swift' - vars: - gnocchi_identity_only: True -- include: os-swift-install.yml -- include: os-gnocchi-install.yml -- include: os-ironic-install.yml -- include: os-magnum-install.yml -- include: os-trove-install.yml -- include: os-sahara-install.yml -- include: os-tacker-install.yml -- include: os-tempest-install.yml - when: (tempest_install | default(False)) | bool or (tempest_run | default(False)) | bool - diff --git a/compass-tasks-osa/tacker_conf/tacker.yml b/compass-tasks-osa/tacker_conf/tacker.yml deleted file mode 100644 index 9ceabbc..0000000 --- a/compass-tasks-osa/tacker_conf/tacker.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -# Copyright 2017, SUSE Linux GmbH -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -component_skel: - tacker_server: - belongs_to: - - tacker_all - - -container_skel: - tacker_container: - belongs_to: - - mano_containers - contains: - - tacker_server - - -physical_skel: - mano_containers: - belongs_to: - - all_containers - mano_hosts: - belongs_to: - - hosts diff --git a/compass-tasks-osa/tacker_conf/tacker_all.yml b/compass-tasks-osa/tacker_conf/tacker_all.yml deleted file mode 100644 index 95d5b83..0000000 --- a/compass-tasks-osa/tacker_conf/tacker_all.yml +++ /dev/null @@ -1,46 +0,0 @@ ---- -# Copyright 2017, SUSE LINUX GmbH -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -tacker_service_user_name: tacker -tacker_service_tenant_name: service - -tacker_rabbitmq_userid: tacker -tacker_rabbitmq_vhost: /tacker -tacker_rabbitmq_port: "{{ rabbitmq_port }}" -tacker_rabbitmq_use_ssl: "{{ rabbitmq_use_ssl }}" -tacker_rabbitmq_servers: "{{ rabbitmq_servers }}" -tacker_rabbitmq_host_group: "{{ rabbitmq_host_group }}" - -tacker_service_publicuri: "{{ openstack_service_publicuri_proto|default(tacker_service_proto) }}://{{ external_lb_vip_address }}:{{ tacker_service_port }}" -tacker_service_adminurl: "{{ tacker_service_adminuri }}/" -tacker_service_region: "{{ service_region }}" -tacker_service_in_ldap: "{{ service_ldap_backend_enabled }}" - -tacker_aodh_enabled: "{{ groups['aodh_all'] is defined and groups['aodh_all'] | length > 0 }}" -tacker_gnocchi_enabled: "{{ groups['gnocchi_all'] is defined and groups['gnocchi_all'] | length > 0 }}" - -# NOTE: these and their swift_all.yml counterpart should be moved back to all.yml once swift with tacker gets proper SSL support -# swift_rabbitmq_telemetry_port: "{{ rabbitmq_port }}" -# swift_rabbitmq_telemetry_use_ssl: "{{ rabbitmq_use_ssl }}" - -# Ensure that the package state matches the global setting -tacker_package_state: "{{ package_state }}" - -# venv fetch configuration -tacker_venv_tag: "{{ venv_tag }}" -tacker_venv_download_url: "{{ venv_base_download_url }}/tacker-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz" - -# locations for fetching the default files from the git source -tacker_git_config_lookup_location: "{{ openstack_repo_url }}/openstackgit/tacker" diff --git a/compass-tasks-osa/tacker_conf/user_secrets.yml b/compass-tasks-osa/tacker_conf/user_secrets.yml deleted file mode 100644 index b5a5796..0000000 --- a/compass-tasks-osa/tacker_conf/user_secrets.yml +++ /dev/null @@ -1,155 +0,0 @@ ---- -# Copyright 2014, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -############################# WARNING ######################################## -# The playbooks do not currently manage changing passwords in an existing -# environment. Changing passwords and re-running the playbooks will fail -# and may break your OpenStack environment. -############################# WARNING ######################################## - - -## Rabbitmq Options -rabbitmq_cookie_token: -rabbitmq_monitoring_password: - -## Tokens -memcached_encryption_key: - -## Galera Options -galera_root_password: - -## Keystone Options -keystone_container_mysql_password: -keystone_auth_admin_password: -keystone_service_password: -keystone_rabbitmq_password: - -## Ceilometer Options: -ceilometer_container_db_password: -ceilometer_service_password: -ceilometer_telemetry_secret: -ceilometer_rabbitmq_password: - -## Aodh Options: -aodh_container_db_password: -aodh_service_password: -aodh_rabbitmq_password: - -## Cinder Options -cinder_container_mysql_password: -cinder_service_password: -cinder_profiler_hmac_key: -cinder_rabbitmq_password: - -## Ceph/rbd: a UUID to be used by libvirt to refer to the client.cinder user -cinder_ceph_client_uuid: - -## Glance Options -glance_container_mysql_password: -glance_service_password: -glance_profiler_hmac_key: -glance_rabbitmq_password: - -## Gnocchi Options: -gnocchi_container_mysql_password: -gnocchi_service_password: - -## Heat Options -heat_stack_domain_admin_password: -heat_container_mysql_password: -### THE HEAT AUTH KEY NEEDS TO BE 32 CHARACTERS LONG ## -heat_auth_encryption_key: -### THE HEAT AUTH KEY NEEDS TO BE 32 CHARACTERS LONG ## -heat_service_password: -heat_rabbitmq_password: - -## Ironic options -ironic_rabbitmq_password: -ironic_container_mysql_password: -ironic_service_password: -ironic_swift_temp_url_secret_key: - -## Horizon Options -horizon_container_mysql_password: -horizon_secret_key: - -## Neutron Options -neutron_container_mysql_password: -neutron_service_password: -neutron_rabbitmq_password: -neutron_ha_vrrp_auth_password: - -## Nova Options -nova_container_mysql_password: -nova_api_container_mysql_password: -nova_metadata_proxy_secret: -nova_service_password: -nova_rabbitmq_password: -nova_placement_service_password: -nova_placement_container_mysql_password: - -# LXD Options for nova compute -lxd_trust_password: - -## Sahara Options -sahara_container_mysql_password: -sahara_rabbitmq_password: -sahara_service_password: - -## Swift Options: -swift_service_password: -swift_dispersion_password: -### Once the swift cluster has been setup DO NOT change these hash values! -swift_hash_path_suffix: -swift_hash_path_prefix: -# Swift needs a telemetry password when using ceilometer -swift_rabbitmq_telemetry_password: - -## haproxy stats password -haproxy_stats_password: -haproxy_keepalived_authentication_password: - -## Magnum Options -magnum_service_password: -magnum_galera_password: -magnum_rabbitmq_password: -magnum_trustee_password: - -## Rally Options: -rally_galera_password: - -## Trove Options -trove_galera_password: -trove_rabbitmq_password: -trove_service_password: -trove_admin_user_password: -trove_regular_user_password: -trove_taskmanager_rpc_encr_key: -trove_inst_rpc_key_encr_key: - -## Barbican Options -barbican_galera_password: -barbican_rabbitmq_password: -barbican_service_password: - -## Designate Options -designate_galera_password: -designate_rabbitmq_password: -designate_service_password: - -# Tacker options -tacker_rabbitmq_password: -tacker_service_password: -tacker_container_mysql_password: diff --git a/compass-tasks/Dockerfile b/compass-tasks/Dockerfile index 4fcfbbc..fc9f06d 100644 --- a/compass-tasks/Dockerfile +++ b/compass-tasks/Dockerfile @@ -1,13 +1,7 @@ -FROM centos:7 +FROM opnfv/compass-tasks-base -ARG BRANCH=master -ADD . /root/compass-tasks - -RUN /root/compass-tasks/build.sh - -EXPOSE 6379 - -VOLUME ["/var/ansible", "/etc/compass/machine_list", "/etc/compass/switch_list"] - -ENTRYPOINT ["/bin/bash", "-c"] -CMD ["/usr/local/bin/start.sh"] +ADD ./run.sh /root/ +#ADD ./tacker_conf /opt/tacker_conf +ADD ./setup-complete.yml /opt/ +RUN chmod +x /root/run.sh +RUN /root/run.sh diff --git a/compass-tasks/README.md b/compass-tasks/README.md deleted file mode 100644 index 952f4b5..0000000 --- a/compass-tasks/README.md +++ /dev/null @@ -1 +0,0 @@ -# compass-tasks \ No newline at end of file diff --git a/compass-tasks/actions/__init__.py b/compass-tasks/actions/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/actions/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/actions/clean.py b/compass-tasks/actions/clean.py deleted file mode 100644 index a4e9bc9..0000000 --- a/compass-tasks/actions/clean.py +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to clean installers -""" -import logging -import xmlrpclib - -from compass.actions import util - -try: - import chef -except ImportError: - pass - -class CobblerInstaller(object): - """cobbler installer""" - CREDENTIALS = "credentials" - USERNAME = 'username' - PASSWORD = 'password' - - INSTALLER_URL = "cobbler_url" - - def __init__(self, settings): - username = settings[self.CREDENTIALS][self.USERNAME] - password = settings[self.CREDENTIALS][self.PASSWORD] - cobbler_url = settings[self.INSTALLER_URL] - try: - self.remote = xmlrpclib.Server(cobbler_url) - self.token = self.remote.login(username, password) - logging.info('cobbler %s client created', cobbler_url) - except Exception as error: - logging.error( - 'failed to login %s with (%s, %s)', - cobbler_url, username, password - ) - logging.exception(error) - - def clean(self): - systems = self.remote.get_systems() - for system in systems: - system_name = system['name'] - try: - self.remote.remove_system(system_name, self.token) - logging.info('system %s is removed', system_name) - except Exception as error: - logging.error( - 'failed to remove system %s', system_name - ) - logging.exception(error) - - -class AnsibleInstaller(object): - - def __init__(self, settings): - return - - def clean(self): - pass - - -class ChefInstaller(object): - DATABAGS = "databags" - CHEFSERVER_URL = "chef_url" - CHEFSERVER_DNS = "chef_server_dns" - CHEFSERVER_IP = "chef_server_ip" - KEY_DIR = "key_dir" - CLIENT = "client_name" - - def __init__(self, settings): - installer_url = settings.get(self.CHEFSERVER_URL, None) - key_dir = settings.get(self.KEY_DIR, None) - client = settings.get(self.CLIENT, None) - try: - if installer_url and key_dir and client: - self.api = chef.ChefAPI(installer_url, key_dir, client) - else: - self.api = chef.autoconfigure() - logging.info( - 'chef client created %s(%s, %s)', - installer_url, key_dir, client - ) - except Exception as error: - logging.error( - 'failed to create chef client %s(%s, %s)', - installer_url, key_dir, client - ) - logging.exception(error) - - def clean(self): - try: - for node_name in chef.Node.list(api=self.api): - node = chef.Node(node_name, api=self.api) - node.delete() - logging.info('delete node %s', node_name) - except Exception as error: - logging.error('failed to delete some nodes') - logging.exception(error) - - try: - for client_name in chef.Client.list(api=self.api): - if client_name in ['chef-webui', 'chef-validator']: - continue - client = chef.Client(client_name, api=self.api) - client.delete() - logging.info('delete client %s', client_name) - except Exception as error: - logging.error('failed to delete some clients') - logging.exception(error) - - try: - for env_name in chef.Environment.list(api=self.api): - if env_name == '_default': - continue - env = chef.Environment(env_name, api=self.api) - env.delete() - logging.info('delete env %s', env_name) - except Exception as error: - logging.error('failed to delete some envs') - logging.exception(error) - - try: - for databag_name in chef.DataBag.list(api=self.api): - databag = chef.DataBag(databag_name, api=self.api) - for item_name, item in databag.items(): - item.delete() - logging.info( - 'delete item %s from databag %s', - item_name, databag_name - ) - except Exception as error: - logging.error('failed to delete some databag items') - logging.exception(error) - - -OS_INSTALLERS = { - 'cobbler': CobblerInstaller -} -PK_INSTALLERS = { - 'chef_installer': ChefInstaller, - 'ansible_installer': AnsibleInstaller -} - - -def clean_os_installer( - os_installer_name, os_installer_settings -): - with util.lock('serialized_action', timeout=100) as lock: - if not lock: - raise Exception( - 'failed to acquire lock to clean os installer' - ) - - if os_installer_name not in OS_INSTALLERS: - logging.error( - '%s not found in os_installers', - os_installer_name - ) - - os_installer = OS_INSTALLERS[os_installer_name]( - os_installer_settings - ) - os_installer.clean() - - -def clean_package_installer( - package_installer_name, package_installer_settings -): - with util.lock('serialized_action', timeout=100) as lock: - if not lock: - raise Exception( - 'failed to acquire lock to clean package installer' - ) - - if package_installer_name not in PK_INSTALLERS: - logging.error( - '%s not found in os_installers', - package_installer_name - ) - - package_installer = PK_INSTALLERS[package_installer_name]( - package_installer_settings - ) - package_installer.clean() diff --git a/compass-tasks/actions/cli.py b/compass-tasks/actions/cli.py deleted file mode 100644 index c9058ed..0000000 --- a/compass-tasks/actions/cli.py +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Compass Command Line Interface""" -import logging -import subprocess -import sys - -from compass.actions.health_check import check -from compass.db.api import database - -from compass.utils import flags -from compass.utils import logsetting -from compass.utils import setting_wrapper as setting -from compass.utils.util import pretty_print - - -ACTION_MAP = { - "check": "apache celery dhcp dns hds misc os_installer " - "package_installer squid tftp".split(" "), - "refresh": "db sync".split(" "), -} - - -class BootCLI(object): - """CLI to do compass check.""" - - def __init__(self): - return - - def run(self, args): - """cli takes the commands and calls respective modules.""" - action = self.get_action(args) - if action is None: - self.print_help() - else: - module = self.get_module(action, args) - if module == "invalid": - self.print_help(action) - else: - method = "self.run_" + action + "(module)" - eval(method) - - @classmethod - def get_action(cls, args): - """This method returns an action type. - - .. note:: - For 'compass check dhcp' command, it will return 'check'. - """ - if len(args) == 1: - return None - elif args[1] in ACTION_MAP.keys(): - return args[1] - return None - - @classmethod - def get_module(cls, action, args): - """This method returns a module. - - .. note:: - For 'compass check dhcp' command, it will return 'dhcp'. - """ - if len(args) <= 2: - return None - elif args[2] in ACTION_MAP[action]: - return args[2] - return "invalid" - - def run_check(self, module=None): - """This provides a flexible sanity check. - - .. note:: - param module default set to None. - if parameter module is none. Compass checks all modules. - If module specified, Compass will only check such module. - """ - if module is None: - pretty_print("Starting: Compass Health Check", - "==============================") - chk = check.BootCheck() - res = chk.run() - self.output_check_result(res) - - else: - pretty_print("Checking Module: %s" % module, - "============================") - chk = check.BootCheck() - method = "chk._check_" + module + "()" - res = eval(method) - print "\n".join(msg for msg in res[1]) - - @classmethod - def output_check_result(cls, result): - """output check result.""" - if result == {}: - return - pretty_print("\n", - "===============================", - "* Compass Health Check Report *", - "===============================") - successful = True - for key in result.keys(): - if result[key][0] == 0: - successful = False - print "%s" % "\n".join(item for item in result[key][1]) - - print "====================" - if successful is True: - print "Compass Check completes. No problems found, all systems go" - sys.exit(0) - else: - print ( - "Compass has ERRORS shown above. Please fix them before " - "deploying!") - sys.exit(1) - - @classmethod - def run_refresh(cls, action=None): - """Run refresh.""" - # TODO(xicheng): replace refresh.sh with refresh.py - if action is None: - pretty_print("Refreshing Compass...", - "=================") - subprocess.Popen( - ['/opt/compass/bin/refresh.sh'], shell=True) - elif action == "db": - pretty_print("Refreshing Compass Database...", - "===================") - subprocess.Popen( - ['/opt/compass/bin/manage_db.py createdb'], shell=True) - else: - pretty_print("Syncing with Installers...", - "================") - subprocess.Popen( - ['/opt/compass/bin/manage_db.py sync_from_installers'], - shell=True - ) - - @classmethod - def print_help(cls, module_help=""): - """print help.""" - if module_help == "": - pretty_print("usage\n=====", - "compass ", - "type 'compass {action} --help' for detailed " - "command list") - - elif module_help == "refresh": - pretty_print("usage\n=====", - "compass refresh [%s]" % - "|".join(action for action in ACTION_MAP['refresh'])) - - else: - pretty_print("usage\n=====", - "compass check [%s]" % - "|".join(action for action in ACTION_MAP['check'])) - sys.exit(2) - - -def main(): - """Compass cli entry point.""" - flags.init() - logsetting.init() - database.init() - cli = BootCLI() - output = cli.run(sys.argv) - return sys.exit(output) diff --git a/compass-tasks/actions/delete.py b/compass-tasks/actions/delete.py deleted file mode 100644 index d89994d..0000000 --- a/compass-tasks/actions/delete.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to delete a given cluster -""" -import logging - -from compass.actions import util -from compass.db.api import cluster as cluster_api -from compass.db.api import host as host_api -from compass.db.api import user as user_db -from compass.deployment.deploy_manager import DeployManager -from compass.deployment.utils import constants as const - - -def delete_cluster( - cluster_id, host_id_list, - username=None, delete_underlying_host=False -): - """Delete cluster and all clusterhosts on it. - - :param cluster_id: id of the cluster. - :type cluster_id: int - :param host_id_list: list of host id. - :type host_id_list: list of int. - - If delete_underlying_host is set, all underlying hosts will - be deleted. - - .. note:: - The function should be called out of database session. - """ - with util.lock('serialized_action', timeout=100) as lock: - if not lock: - raise Exception('failed to acquire lock to delete cluster') - - user = user_db.get_user_object(username) - - cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) - adapter_id = cluster_info[const.ADAPTER_ID] - - adapter_info = util.ActionHelper.get_adapter_info( - adapter_id, cluster_id, user) - hosts_info = util.ActionHelper.get_hosts_info( - cluster_id, host_id_list, user) - - deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) - - deploy_manager.remove_hosts( - package_only=not delete_underlying_host, - delete_cluster=True - ) - util.ActionHelper.delete_cluster( - cluster_id, host_id_list, user, - delete_underlying_host - ) - - -def delete_cluster_host( - cluster_id, host_id, - username=None, delete_underlying_host=False -): - """Delete clusterhost. - - :param cluster_id: id of the cluster. - :type cluster_id: int - :param host_id: id of the host. - :type host_id: int - - If delete_underlying_host is set, the underlying host - will be deleted too. - - .. note:: - The function should be called out of database session. - """ - with util.lock('serialized_action', timeout=100) as lock: - if not lock: - raise Exception('failed to acquire lock to delete clusterhost') - - user = user_db.get_user_object(username) - cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) - adapter_id = cluster_info[const.ADAPTER_ID] - - adapter_info = util.ActionHelper.get_adapter_info( - adapter_id, cluster_id, user) - hosts_info = util.ActionHelper.get_hosts_info( - cluster_id, [host_id], user) - - deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) - - deploy_manager.remove_hosts( - package_only=not delete_underlying_host, - delete_cluster=False - ) - util.ActionHelper.delete_cluster_host( - cluster_id, host_id, user, - delete_underlying_host - ) - - -def delete_host( - host_id, cluster_id_list, username=None -): - """Delete host and all clusterhosts on it. - - :param host_id: id of the host. - :type host_id: int - - .. note:: - The function should be called out of database session. - """ - with util.lock('serialized_action', timeout=100) as lock: - if not lock: - raise Exception('failed to acquire lock to delete host') - - user = user_db.get_user_object(username) - for cluster_id in cluster_id_list: - cluster_info = util.ActionHelper.get_cluster_info( - cluster_id, user) - adapter_id = cluster_info[const.ADAPTER_ID] - - adapter_info = util.ActionHelper.get_adapter_info( - adapter_id, cluster_id, user) - hosts_info = util.ActionHelper.get_hosts_info( - cluster_id, [host_id], user) - - deploy_manager = DeployManager( - adapter_info, cluster_info, hosts_info) - - deploy_manager.remove_hosts( - package_only=True, - delete_cluster=False - ) - - util.ActionHelper.delete_host( - host_id, user - ) diff --git a/compass-tasks/actions/deploy.py b/compass-tasks/actions/deploy.py deleted file mode 100644 index 53179f5..0000000 --- a/compass-tasks/actions/deploy.py +++ /dev/null @@ -1,182 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to deploy a given cluster -""" -import logging - -from compass.actions import util -from compass.db.api import cluster as cluster_db -from compass.db.api import health_check_report as health_check_db -from compass.db.api import user as user_db -from compass.deployment.deploy_manager import DeployManager -from compass.deployment.utils import constants as const - - -def deploy(cluster_id, hosts_id_list, username=None): - """Deploy clusters. - - :param cluster_hosts: clusters and hosts in each cluster to deploy. - :type cluster_hosts: dict of int or str to list of int or str - - .. note:: - The function should be called out of database session. - """ - with util.lock('serialized_action', timeout=1000) as lock: - if not lock: - raise Exception('failed to acquire lock to deploy') - - user = user_db.get_user_object(username) - - cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) - adapter_id = cluster_info[const.ADAPTER_ID] - - adapter_info = util.ActionHelper.get_adapter_info( - adapter_id, cluster_id, user) - hosts_info = util.ActionHelper.get_hosts_info( - cluster_id, hosts_id_list, user) - - deploy_successful = True - try: - deploy_manager = DeployManager( - adapter_info, cluster_info, hosts_info) - # deploy_manager.prepare_for_deploy() - logging.debug('Created deploy manager with %s %s %s' - % (adapter_info, cluster_info, hosts_info)) - deployed_config = deploy_manager.deploy() - except Exception as error: - logging.exception(error) - deploy_successful = False - - if deploy_successful: - util.ActionHelper.save_deployed_config(deployed_config, user) - util.ActionHelper.update_state( - cluster_id, hosts_id_list, user, state='INSTALLING' - ) - else: - util.ActionHelper.update_state( - cluster_id, hosts_id_list, user, state='ERROR', - message='failed to start deployment', severity='ERROR' - ) - - -def redeploy(cluster_id, username=None): - """Deploy clusters. - - :param cluster_hosts: clusters and hosts in each cluster to deploy. - :type cluster_hosts: dict of int or str to list of int or str - """ - with util.lock('serialized_action') as lock: - if not lock: - raise Exception('failed to acquire lock to deploy') - - user = user_db.get_user_object(username) - cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) - adapter_id = cluster_info[const.ADAPTER_ID] - - adapter_info = util.ActionHelper.get_adapter_info( - adapter_id, cluster_id, user) - - cluster_hosts = cluster_db.list_cluster_hosts(cluster_id, user) - hosts_id_list = [host['id'] for host in cluster_hosts] - - hosts_info = util.ActionHelper.get_hosts_info( - cluster_id, hosts_id_list, user) - - deploy_successful = True - try: - deploy_manager = DeployManager( - adapter_info, cluster_info, hosts_info) - # deploy_manager.prepare_for_deploy() - deploy_manager.redeploy() - except Exception as error: - logging.exception(error) - deploy_successful = False - if deploy_successful: - util.ActionHelper.update_state( - cluster_id, hosts_id_list, user, state='INSTALLING', - ) - else: - util.ActionHelper.update_state( - cluster_id, hosts_id_list, user, state='ERROR', - message='failed to start redeployment', severity='ERROR' - ) - - -def health_check(cluster_id, report_uri, username): - with util.lock('cluster_health_check') as lock: - if not lock: - raise Exception('failed to acquire lock to check health') - - user = user_db.get_user_object(username) - cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) - adapter_id = cluster_info[const.ADAPTER_ID] - - adapter_info = util.ActionHelper.get_adapter_info( - adapter_id, cluster_id, user - ) - - deploy_manager = DeployManager(adapter_info, cluster_info, None) - try: - deploy_manager.check_cluster_health(report_uri) - except Exception as exc: - logging.error("health_check exception: ============= %s" % exc) - data = {'state': 'error', 'error_message': str(exc), 'report': {}} - reports = health_check_db.list_health_reports( - cluster_id, user=user) - if not reports: - # Exception before executing command remotely for health check. - # No reports names sending back yet. Create a report - name = 'pre_remote_health_check' - health_check_db.add_report_record( - cluster_id, name, user=user, **data - ) - - health_check_db.update_multi_reports(cluster_id, user=user, **data) - - -class ServerPowerMgmt(object): - """Power management for bare-metal machines by IPMI command.""" - @staticmethod - def poweron(machine_id, user): - """Power on the specified machine.""" - pass - - @staticmethod - def poweroff(machine_id, user): - pass - - @staticmethod - def reset(machine_id, user): - pass - - -class HostPowerMgmt(object): - """Power management for hosts installed OS by OS installer. OS installer - - will poweron/poweroff/reset host. - - """ - @staticmethod - def poweron(host_id, user): - """Power on the specified host.""" - pass - - @staticmethod - def poweroff(host_id, user): - pass - - @staticmethod - def reset(host_id, user): - pass diff --git a/compass-tasks/actions/health_check/__init__.py b/compass-tasks/actions/health_check/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/actions/health_check/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/actions/health_check/base.py b/compass-tasks/actions/health_check/base.py deleted file mode 100644 index 22b6fae..0000000 --- a/compass-tasks/actions/health_check/base.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Base class for Compass Health Check.""" -from compass.actions.health_check import utils as health_check_utils -from compass.db.api import adapter as adapter_api -from compass.utils import setting_wrapper as setting - - -class BaseCheck(object): - """health check base class.""" - - def __init__(self): - self.config = setting - self.code = 1 - self.messages = [] - self.dist, self.version, self.release = health_check_utils.get_dist() - adapter_api.load_adapters_internal() - self.os_installer = self._get_os_installer() - self.package_installer = self._get_package_installer() - - def _get_os_installer(self): - installer = adapter_api.OS_INSTALLERS.values()[0] - os_installer = {} - os_installer['name'] = health_check_utils.strip_name( - installer['name']) - os_installer.update(installer['settings']) - return os_installer - - def _get_package_installer(self): - package_installer = {} - installer = adapter_api.PACKAGE_INSTALLERS.values()[0] - package_installer = {} - package_installer['name'] = health_check_utils.strip_name( - installer['name']) - package_installer.update(installer['settings']) - return package_installer - - def _set_status(self, code, message): - """set status.""" - self.code = code - self.messages.append(message) - - def get_status(self): - """get status.""" - return (self.code, self.messages) diff --git a/compass-tasks/actions/health_check/check.py b/compass-tasks/actions/health_check/check.py deleted file mode 100644 index c1adbc6..0000000 --- a/compass-tasks/actions/health_check/check.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Main Entry Point of Compass Health Check.""" -from compass.actions.health_check import base -from compass.actions.health_check import check_apache -from compass.actions.health_check import check_celery -from compass.actions.health_check import check_dhcp -from compass.actions.health_check import check_dns -from compass.actions.health_check import check_hds -from compass.actions.health_check import check_misc -from compass.actions.health_check import check_os_installer -from compass.actions.health_check import check_package_installer -from compass.actions.health_check import check_squid -from compass.actions.health_check import check_tftp - - -class BootCheck(base.BaseCheck): - """health check for all components.""" - - def run(self): - """do health check.""" - status = {} - status['apache'] = self._check_apache() - status['celery'] = self._check_celery() - status['dhcp'] = self._check_dhcp() - status['dns'] = self._check_dns() - status['hds'] = self._check_hds() - status['os_installer'] = self._check_os_installer() - status['package_installer'] = self._check_package_installer() - status['squid'] = self._check_squid() - status['tftp'] = self._check_tftp() - status['other'] = self._check_misc() - - return status - - def _check_apache(self): - """do apache health check.""" - checker = check_apache.ApacheCheck() - return checker.run() - - def _check_celery(self): - """do celery health check.""" - checker = check_celery.CeleryCheck() - return checker.run() - - def _check_dhcp(self): - """do dhcp health check.""" - checker = check_dhcp.DhcpCheck() - return checker.run() - - def _check_dns(self): - """do dns health check.""" - checker = check_dns.DnsCheck() - return checker.run() - - def _check_hds(self): - """do hds health check.""" - checker = check_hds.HdsCheck() - return checker.run() - - def _check_os_installer(self): - """do os installer health check.""" - checker = check_os_installer.OsInstallerCheck() - return checker.run() - - def _check_package_installer(self): - """do package installer health check.""" - checker = check_package_installer.PackageInstallerCheck() - return checker.run() - - def _check_squid(self): - """do squid health check.""" - checker = check_squid.SquidCheck() - return checker.run() - - def _check_tftp(self): - """do tftp health check.""" - checker = check_tftp.TftpCheck() - return checker.run() - - def _check_misc(self): - """do misc health check.""" - checker = check_misc.MiscCheck() - return checker.run() diff --git a/compass-tasks/actions/health_check/check_apache.py b/compass-tasks/actions/health_check/check_apache.py deleted file mode 100644 index 294d6f9..0000000 --- a/compass-tasks/actions/health_check/check_apache.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Health Check module for Apache service.""" - -import socket -import urllib2 - -from compass.actions.health_check import base -from compass.actions.health_check import utils as health_check_utils - - -class ApacheCheck(base.BaseCheck): - """apache server health check class.""" - NAME = "Apache Check" - - def run(self): - """do the healthcheck.""" - if self.dist in ("centos", "redhat", "fedora", "scientific linux"): - apache_service = 'httpd' - else: - apache_service = 'apache2' - self.check_apache_conf(apache_service) - print "[Done]" - self.check_apache_running(apache_service) - print "[Done]" - if self.code == 1: - self.messages.append( - "[%s]Info: Apache health check has completed. " - "No problems found, all systems go." % self.NAME) - return (self.code, self.messages) - - def check_apache_conf(self, apache_service): - """Validates if Apache settings. - - :param apache_service : service type of apache, os dependent. - e.g. httpd or apache2 - :type apache_service : string - - """ - print "Checking Apache Config......", - conf_err_msg = health_check_utils.check_path( - self.NAME, - "/etc/%s/conf.d/ods-server.conf" % apache_service) - if not conf_err_msg == "": - self._set_status(0, conf_err_msg) - - wsgi_err_msg = health_check_utils.check_path( - self.NAME, - '/var/www/compass/compass.wsgi') - if not wsgi_err_msg == "": - self._set_status(0, wsgi_err_msg) - - return True - - def check_apache_running(self, apache_service): - """Checks if Apache service is running on port 80.""" - - print "Checking Apache service......", - serv_err_msg = health_check_utils.check_service_running(self.NAME, - apache_service) - if not serv_err_msg == "": - self._set_status(0, serv_err_msg) - if 'http' != socket.getservbyport(80): - self._set_status( - 0, - "[%s]Error: Apache is not listening on port 80." - % self.NAME) - try: - html = urllib2.urlopen('http://localhost') - html.geturl() - except Exception: - self._set_status( - 0, - "[%s]Error: Apache is not listening on port 80." - % self.NAME) - - return True diff --git a/compass-tasks/actions/health_check/check_celery.py b/compass-tasks/actions/health_check/check_celery.py deleted file mode 100644 index 2d8d27c..0000000 --- a/compass-tasks/actions/health_check/check_celery.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Health Check module for Celery.""" -import commands -import os - -from celery.task.control import inspect - -from compass.actions.health_check import base -from compass.actions.health_check import utils as health_check_utils - - -class CeleryCheck(base.BaseCheck): - """celery health check class.""" - NAME = "Celery Check." - - def run(self): - """do health check.""" - self.check_compass_celery_setting() - print "[Done]" - self.check_celery_backend() - print "[Done]" - if self.code == 1: - self.messages.append("[%s]Info: Celery health check " - "has completed. No problems found, " - "all systems go." % self.NAME) - return (self.code, self.messages) - - def check_compass_celery_setting(self): - """Validates Celery settings.""" - - print "Checking Celery setting......", - setting_map = { - 'logfile': 'CELERY_LOGFILE', - 'configdir': 'CELERYCONFIG_DIR', - 'configfile': 'CELERYCONFIG_FILE', - } - unset = [] - res = health_check_utils.validate_setting('Celery', - self.config, - 'CELERY_LOGFILE') - if res is False: - unset.append(setting_map["logfile"]) - self._set_status(0, res) - - res = health_check_utils.validate_setting('Celery', - self.config, - 'CELERYCONFIG_DIR') - if res is False: - unset.append(setting_map["configdir"]) - self._set_status(0, res) - - res = health_check_utils.validate_setting('Celery', - self.config, - 'CELERYCONFIG_FILE') - if res is False: - unset.append(setting_map["configdir"]) - self._set_status(0, res) - - if len(unset) != 0: - self._set_status(0, - "[%s]Error: Unset celery settings: %s" - " in /etc/compass/setting" - % (self.NAME, ', '.join(item for item in unset))) - return True - - def check_celery_backend(self): - """Checks if Celery backend is running and configured properly.""" - - print "Checking Celery Backend......", - if 'celery worker' not in commands.getoutput('ps -ef'): - self._set_status(0, "[%s]Error: celery is not running" % self.NAME) - return True - - if not os.path.exists('/etc/compass/celeryconfig'): - self._set_status( - 0, - "[%s]Error: No celery config file found for Compass" - % self.NAME) - return True - - try: - insp = inspect() - celery_stats = inspect.stats(insp) - if not celery_stats: - self._set_status( - 0, - "[%s]Error: No running Celery workers were found." - % self.NAME) - except IOError as error: - self._set_status( - 0, - "[%s]Error: Failed to connect to the backend: %s" - % (self.NAME, str(error))) - from errno import errorcode - if ( - len(error.args) > 0 and - errorcode.get(error.args[0]) == 'ECONNREFUSED' - ): - self.messages.append( - "[%s]Error: RabbitMQ server isn't running" - % self.NAME) - return True diff --git a/compass-tasks/actions/health_check/check_dhcp.py b/compass-tasks/actions/health_check/check_dhcp.py deleted file mode 100644 index e3bae1e..0000000 --- a/compass-tasks/actions/health_check/check_dhcp.py +++ /dev/null @@ -1,184 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Health Check module for DHCP service.""" -import commands -import os -import re -import socket -import xmlrpclib - -from compass.actions.health_check import base - - -class DhcpCheck(base.BaseCheck): - """dhcp health check class.""" - - NAME = "DHCP Check" - - def run(self): - """do health check.""" - method_name = "self.check_" + self.os_installer['name'] + "_dhcp()" - return eval(method_name) - - def check_cobbler_dhcp(self): - """Checks if Cobbler has taken over DHCP service.""" - - try: - remote = xmlrpclib.Server( - self.os_installer['cobbler_url'], - allow_none=True) - credentials = self.os_installer['credentials'] - remote.login( - credentials['username'], credentials['password']) - except Exception: - self._set_status( - 0, - "[%s]Error: Cannot login to Cobbler with " - "the tokens provided in the config file" % self.NAME) - return (self.code, self.messages) - - cobbler_settings = remote.get_settings() - if cobbler_settings['manage_dhcp'] == 0: - self.messages.append( - "[%s]Info: DHCP service is " - "not managed by Compass" % self.NAME) - self.code = 0 - return (self.code, self.messages) - - self.check_cobbler_dhcp_template() - print "[Done]" - self.check_dhcp_service() - self.check_dhcp_netmask() - print "[Done]" - if self.code == 1: - self.messages.append( - "[%s]Info: DHCP health check has completed. " - "No problems found, all systems go." % self.NAME) - - return (self.code, self.messages) - - def check_cobbler_dhcp_template(self): - """Validates Cobbler's DHCP template file.""" - print "Checking DHCP template......", - if os.path.exists("/etc/cobbler/dhcp.template"): - var_map = { - "match_next_server": False, - "match_subnet": False, - "match_filename": False, - "match_range": False, - } - - ip_regex = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') - - dhcp_template = open("/etc/cobbler/dhcp.template") - for line in dhcp_template.readlines(): - if line.find("next_server") != -1: - elmlist = line.split(" ") - for elm in elmlist: - if ";" in elm: - elm = elm[:-2] - - if "$next_server" in elm or ip_regex.match(elm): - var_map["match_next_server"] = True - - elif line.find("subnet") != -1 and line.find("{") != -1: - elmlist = line.split(" ") - for elm in elmlist: - if ip_regex.match(elm): - if elm[-1] == "0" and "255" not in elm: - var_map["match_subnet"] = True - elif elm[-1] != "0": - self.messages.append( - "[%s]Error: Subnet should be set " - "in the form of 192.168.0.0 in" - "/etc/cobbler/dhcp.template" % self.NAME) - - elif line.find("filename") != -1: - var_map["match_filename"] = True - elif line.find("range dynamic-bootp") != -1: - elmlist = line.split(" ") - ip_count = 0 - for elm in elmlist: - if ";" in elm and "\n" in elm: - elm = elm[:-2] - - if ip_regex.match(elm): - ip_count += 1 - - if ip_count != 2: - self.messages.append( - "[%s]Error: DHCP range should be set " - "between two IP addresses in " - "/etc/cobbler/dhcp.template" % self.NAME) - else: - var_map["match_range"] = True - - dhcp_template.close() - fails = [] - for var in var_map.keys(): - if var_map[var] is False: - fails.append(var) - - if len(fails) != 0: - self._set_status( - 0, - "[%s]Info: DHCP template file " - "failed components: %s" % ( - self.NAME, ' '.join(failed for failed in fails))) - - else: - self._set_status( - 0, - "[%s]Error: DHCP template file doesn't exist, " - "health check failed." % self.NAME) - - return True - - def check_dhcp_netmask(self): - with open('/etc/dhcp/dhcpd.conf') as conf_reader: - lines = conf_reader.readlines() - for line in lines: - if re.search('^subnet', line): - elm_list = line.split(' ') - break - subnet_ip = elm_list[1] - netmask = elm_list[-2] - subnet_ip_elm = subnet_ip.split('.') - netmask_elm = netmask.split('.') - for index, digit in enumerate(subnet_ip_elm): - if int(digit) & int(netmask_elm[index]) != int(digit): - self._set_status( - 0, - "[%s]Info: DHCP subnet IP and " - "netmask do not match" % self.NAME) - break - return True - - def check_dhcp_service(self): - """Checks if DHCP is running on port 67.""" - print "Checking DHCP service......", - if not commands.getoutput('pgrep dhcp'): - self._set_status( - 0, - "[%s]Error: dhcp service does not " - "seem to be running" % self.NAME) - - if socket.getservbyport(67) != 'bootps': - self._set_status( - 0, - "[%s]Error: bootps is not listening " - "on port 67" % self.NAME) - - return True diff --git a/compass-tasks/actions/health_check/check_dns.py b/compass-tasks/actions/health_check/check_dns.py deleted file mode 100644 index 843d7e2..0000000 --- a/compass-tasks/actions/health_check/check_dns.py +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Health Check module for DNS service.""" - -import commands -import os -import socket -import xmlrpclib - -from compass.actions.health_check import base - - -class DnsCheck(base.BaseCheck): - """dns health check class.""" - NAME = "DNS Check" - - def run(self): - """do health check.""" - method_name = "self.check_" + self.os_installer['name'] + "_dns()" - return eval(method_name) - - def check_cobbler_dns(self): - """Checks if Cobbler has taken over DNS service.""" - try: - remote = xmlrpclib.Server( - self.os_installer['cobbler_url'], - allow_none=True) - credentials = self.os_installer['credentials'] - remote.login( - credentials['username'], credentials['password']) - except Exception: - self._set_status(0, - "[%s]Error: Cannot login to Cobbler " - "with the tokens provided in the config file" - % self.NAME) - return (self.code, self.messages) - - cobbler_settings = remote.get_settings() - if cobbler_settings['manage_dns'] == 0: - self.messages.append('[DNS]Info: DNS is not managed by Compass') - return (0, self.messages) - self.check_cobbler_dns_template() - print "[Done]" - self.check_dns_service() - print "[Done]" - if self.code == 1: - self.messages.append( - "[%s]Info: DNS health check has complated. " - "No problems found, all systems go." % self.NAME) - return (self.code, self.messages) - - def check_cobbler_dns_template(self): - """Validates Cobbler's DNS template file.""" - - print "Checking DNS template......", - if os.path.exists("/etc/cobbler/named.template"): - var_map = { - "match_port": False, - "match_allow_query": False, - } - named_template = open("/etc/cobbler/named.template") - host_ip = socket.gethostbyname(socket.gethostname()) - missing_query = [] - for line in named_template.readlines(): - if "listen-on port 53" in line and host_ip in line: - var_map["match_port"] = True - - if "allow-query" in line: - for subnet in ["127.0.0.0/8"]: - if subnet not in line: - missing_query.append(subnet) - - named_template.close() - - if var_map["match_port"] is False: - self.messages.append( - "[%s]Error: named service port " - "and/or IP is misconfigured in " - "/etc/cobbler/named.template" % self.NAME) - - if len(missing_query) != 0: - self.messages.append( - "[%s]Error: Missing allow_query values in " - "/etc/cobbler/named.template:%s" % ( - self.NAME, - ', '.join(subnet for subnet in missing_query))) - else: - var_map["match_allow_query"] = True - - fails = [] - for var in var_map.keys(): - if var_map[var] is False: - fails.append(var) - - if len(fails) != 0: - self._set_status( - 0, - "[%s]Info: DNS template failed components: " - "%s" % ( - self.NAME, - ' '.join(failed for failed in fails))) - - else: - self._set_status( - 0, - "[%s]Error: named template file doesn't exist, " - "health check failed." % self.NAME) - - return True - - def check_dns_service(self): - """Checks if DNS is running on port 53.""" - - print "Checking DNS service......", - if 'named' not in commands.getoutput('ps -ef'): - self._set_status( - 0, - "[%s]Error: named service does not seem to be " - "running" % self.NAME) - - if socket.getservbyport(53) != 'domain': - self._set_status( - 0, - "[%s]Error: domain service is not listening on port " - "53" % self.NAME) - - return None diff --git a/compass-tasks/actions/health_check/check_hds.py b/compass-tasks/actions/health_check/check_hds.py deleted file mode 100644 index d176f1f..0000000 --- a/compass-tasks/actions/health_check/check_hds.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Health Check module for Hardware Discovery.""" -import logging - -from compass.actions.health_check import base -from compass.actions.health_check import utils as health_check_utils - - -class HdsCheck(base.BaseCheck): - """hds health check class.""" - NAME = "HDS Check" - - def run(self): - """do health check.""" - if self.dist in ("centos", "redhat", "fedora", "scientific linux"): - pkg_type = "yum" - else: - pkg_type = "apt" - - try: - pkg_module = __import__(pkg_type) - except Exception: - self._set_status( - 0, "[%s]Error: No module named %s please install it first." - % (self.NAME, pkg_type) - ) - return (self.code, self.messages) - - logging.info('import %s: %s', pkg_type, pkg_module) - method_name = 'self.check_' + pkg_type + '_snmp(pkg_module)' - eval(method_name) - print "[Done]" - self.check_snmp_mibs() - print "[Done]" - if self.code == 1: - self.messages.append("[%s]Info: hds health check has complated. " - "No problems found, all systems go." - % self.NAME) - - return (self.code, self.messages) - - def check_yum_snmp(self, pkg_module): - """Check if SNMP yum dependencies are installed - - :param pkg_module : python yum library - :type pkg_module : python module - - """ - print "Checking SNMP Packages......", - yum_base = pkg_module.YumBase() - uninstalled = [] - for package in ['net-snmp-utils', 'net-snmp', 'net-snmp-python']: - if len(yum_base.rpmdb.searchNevra(name=package)) == 0: - self.messages.append("[%s]Error: %s package is required " - "for HDS" % (self.NAME, package)) - uninstalled.append(package) - - if len(uninstalled) != 0: - self._set_status(0, "[%s]Info: Uninstalled packages: %s" - % (self.NAME, - ', '.join(item for item in uninstalled))) - - return True - - def check_apt_snmp(self, pkg_module): - """do apt health check.""" - return None - - def check_snmp_mibs(self): - """Checks if SNMP MIB files are properly placed.""" - - print "Checking SNMP MIBs......", - conf_err_msg = health_check_utils.check_path(self.NAME, - '/etc/snmp/snmp.conf') - if not conf_err_msg == "": - self._set_status(0, conf_err_msg) - - mibs_err_msg = health_check_utils.check_path( - self.NAME, - '/usr/local/share/snmp/mibs') - if not mibs_err_msg == "": - self._set_status(0, mibs_err_msg) - - return True diff --git a/compass-tasks/actions/health_check/check_misc.py b/compass-tasks/actions/health_check/check_misc.py deleted file mode 100644 index b8beb1b..0000000 --- a/compass-tasks/actions/health_check/check_misc.py +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Miscellaneous Health Check for Compass.""" -import logging - -from compass.actions.health_check import base -from compass.actions.health_check import utils as health_check_utils - - -class MiscCheck(base.BaseCheck): - """health check for misc.""" - NAME = "Miscellaneous Check" - - MISC_MAPPING = { - "yum": "rsyslog ntp iproute openssh-clients python git wget " - "python-setuptools " - "amqp mod_wsgi httpd squid " - "dhcp bind rsync yum-utils xinetd tftp-server gcc " - "net-snmp-utils net-snmp".split(" "), - "pip": "netaddr flask flask_script flask_restful amqplib " - "flask_sqlalchemy paramiko mock celery six discover daemon " - "unittest2 chef".split(" "), - "disable": "iptables ip6tables".split(" "), - "enable": "httpd squid xinetd dhcpd named sshd rsyslog cobblerd " - "ntpd compass-celeryd compass-progress-updated".split(" "), - } - - def run(self): - """do health check.""" - self.check_linux_dependencies() - print "[Done]" - self.check_pip_dependencies() - print "[Done]" - self.check_ntp() - print "[Done]" - self.check_rsyslogd() - print "[Done]" - self.check_chkconfig() - print "[Done]" - self.check_selinux() - print "[Done]" - - if self.code == 1: - self.messages.append( - "[%s]Info: Miscellaneous check has completed " - "No problems found, all systems go." % self.NAME) - return (self.code, self.messages) - - def check_linux_dependencies(self): - """Checks if dependencies are installed.""" - print "Checking Linux dependencies....", - if self.dist in ("centos", "redhat", "fedora", "scientific linux"): - pkg_type = "yum" - else: - pkg_type = "apt" - - try: - pkg_module = __import__(pkg_type) - except Exception: - self._set_status( - 0, - "[%s]Error: No module named %s, " - "please install it first." % (self.NAME, pkg_type)) - return True - - logging.info('import %s: %s', pkg_type, pkg_module) - method_name = 'self.check_' + pkg_type + '_dependencies(pkg_module)' - eval(method_name) - - def check_yum_dependencies(self, pkg_module): - """Checks if yum dependencies are installed. - - :param pkg_module : python yum library - :type pkg_module : python module - - """ - print "Checking Yum dependencies......", - yum_base = pkg_module.YumBase() - uninstalled = [] - for package in self.MISC_MAPPING["yum"]: - if len(yum_base.rpmdb.searchNevra(name=package)) == 0: - self._set_status( - 0, - "[%s]Error: %s package is required" - % (self.NAME, package)) - uninstalled.append(package) - - if len(uninstalled) != 0: - self._set_status( - 0, - "[%s]Info: Uninstalled yum packages: %s" - % (self.NAME, ', '.join(item for item in uninstalled))) - - return True - - def check_pip_dependencies(self): - """Checks if required pip packages are installed.""" - print "Checking pip dependencies......", - uninstalled = [] - for module in self.MISC_MAPPING['pip']: - try: - __import__(module) - except Exception: - self._set_status( - 0, - "[%s]Error: pip package %s is requred" - % (self.NAME, module)) - uninstalled.append(module) - - if len(uninstalled) != 0: - self._set_status( - 0, - "[%s]Info: Uninstalled pip packages: %s" - % (self.NAME, ', '.join(item for item in uninstalled))) - - return True - - def check_ntp(self): - """Validates ntp configuration and service.""" - - print "Checking NTP......", - conf_err_msg = health_check_utils.check_path(self.NAME, - '/etc/ntp.conf') - if not conf_err_msg == "": - self._set_status(0, conf_err_msg) - - serv_err_msg = health_check_utils.check_service_running(self.NAME, - 'ntpd') - if not serv_err_msg == "": - self._set_status(0, serv_err_msg) - - return True - - def check_rsyslogd(self): - """Validates rsyslogd configuration and service.""" - - print "Checking rsyslog......", - conf_err_msg = health_check_utils.check_path(self.NAME, - '/etc/rsyslog.conf') - if not conf_err_msg == "": - self._set_status(0, conf_err_msg) - - dir_err_msg = health_check_utils.check_path(self.NAME, - '/etc/rsyslog.d/') - if not dir_err_msg == "": - self._set_status(0, dir_err_msg) - - serv_err_msg = health_check_utils.check_service_running(self.NAME, - 'rsyslogd') - if not serv_err_msg == "": - self._set_status(0, serv_err_msg) - - return True - - def check_chkconfig(self): - """Check if required services are enabled on the start up.""" - - print "Checking chkconfig......", - serv_to_disable = [] - for serv in self.MISC_MAPPING["disable"]: - if health_check_utils.check_chkconfig(serv) is True: - self._set_status( - 0, - "[%s]Error: %s is not disabled" - % (self.NAME, serv)) - serv_to_disable.append(serv) - - if len(serv_to_disable) != 0: - self._set_status( - 0, - "[%s]Info: You need to disable these services " - "on system start-up: %s" - % (self.NAME, - ", ".join(item for item in serv_to_disable))) - - serv_to_enable = [] - for serv in self.MISC_MAPPING["enable"]: - if health_check_utils.check_chkconfig(serv) is False: - self._set_status( - 0, "[%s]Error: %s is disabled" % (self.NAME, serv)) - serv_to_enable.append(serv) - - if len(serv_to_enable) != 0: - self._set_status(0, "[%s]Info: You need to enable these " - "services on system start-up: %s" - % (self.NAME, - ", ".join(item for item in serv_to_enable))) - - return True - - def check_selinux(self): - """Check if SELinux is disabled.""" - print "Checking Selinux......", - disabled = False - with open("/etc/selinux/config") as selinux: - for line in selinux: - if "SELINUX=disabled" in line: - disabled = True - break - - if disabled is False: - self._set_status( - 0, - "[%s]Selinux is not disabled, " - "please disable it in /etc/selinux/config." % self.NAME) - - return True diff --git a/compass-tasks/actions/health_check/check_os_installer.py b/compass-tasks/actions/health_check/check_os_installer.py deleted file mode 100644 index 6ef9818..0000000 --- a/compass-tasks/actions/health_check/check_os_installer.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Compass Health Check module for OS Installer.""" - -import os -import xmlrpclib - -from compass.actions.health_check import base - - -class OsInstallerCheck(base.BaseCheck): - """os installer health check.""" - NAME = "OS Installer Check" - - def run(self): - """do health check.""" - method_name = 'self.' + self.os_installer['name'] + '_check()' - return eval(method_name) - - def cobbler_check(self): - """Runs cobbler check from xmlrpc client.""" - try: - remote = xmlrpclib.Server( - self.os_installer['cobbler_url'], - allow_none=True) - credentials = self.os_installer['credentials'] - token = remote.login( - credentials['username'], credentials['password']) - except Exception: - self.code = 0 - self.messages.append( - "[%s]Error: Cannot login to Cobbler with " - "the tokens provided in the config file" - % self.NAME) - self.messages.append( - "[%s]Error: Failed to connect to Cobbler " - "API, please check if /etc/cobbler/setting " - "is properly configured" % self.NAME) - return (self.code, self.messages) - - check_result = remote.check(token) - - for index, message in enumerate(check_result): - if "SELinux" in message: - check_result.pop(index) - - if len(check_result) != 0: - self.code = 0 - for error_msg in check_result: - self.messages.append("[%s]Error: " % self.NAME + error_msg) - - if len(remote.get_distros()) == 0: - self._set_status(0, - "[%s]Error: No Cobbler distros found" % self.NAME) - - if len(remote.get_profiles()) == 0: - self._set_status(0, - "[%s]Error: No Cobbler profiles found" - % self.NAME) - - found_ppa = False - if len(remote.get_repos()) != 0: - for repo in remote.get_repos(): - if 'ppa_repo' in repo['mirror']: - found_ppa = True - break - - if found_ppa is False: - self._set_status(0, - "[%s]Error: No repository ppa_repo found" - % self.NAME) - - path_map = { - 'match_kickstart': ( - '/var/lib/cobbler/kickstarts/', - ['default.ks', 'default.seed'] - ), - 'match_snippets': ( - '/var/lib/cobbler/snippets/', - [ - 'kickstart_done', - 'kickstart_start', - 'kickstart_pre_partition_disks', - 'kickstart_partition_disks', - 'kickstart_pre_anamon', - 'kickstart_post_anamon', - 'kickstart_pre_install_network_config', - 'kickstart_network_config', - 'kickstart_post_install_network_config', - 'kickstart_chef', - 'kickstart_ntp', - 'kickstart_yum_repo_config', - 'preseed_pre_partition_disks', - 'preseed_partition_disks', - 'preseed_pre_anamon', - 'preseed_post_anamon', - 'preseed_pre_install_network_config', - 'preseed_network_config', - 'preseed_post_install_network_config', - 'preseed_chef', - 'preseed_ntp', - 'preseed_apt_repo_config', - ] - ), - 'match_ks_mirror': ( - '/var/www/cobbler/', - ['ks_mirror'] - ), - 'match_repo_mirror': ( - '/var/www/cobbler/', - ['repo_mirror'] - ), - 'match_iso': ( - '/var/lib/cobbler/', - ['iso'] - ), - } - not_exists = [] - for key in path_map.keys(): - for path in path_map[key][1]: - if not os.path.exists(path_map[key][0] + path): - not_exists.append(path_map[key][0] + path) - - if len(not_exists) != 0: - self._set_status( - 0, - "[%s]Error: These locations do not exist: " - "%s" % ( - self.NAME, - ', '.join(item for item in not_exists) - ) - ) - - if self.code == 1: - self.messages.append( - "[%s]Info: OS Installer health check has completed." - " No problems found, all systems go." % self.NAME) - - return (self.code, self.messages) diff --git a/compass-tasks/actions/health_check/check_package_installer.py b/compass-tasks/actions/health_check/check_package_installer.py deleted file mode 100644 index efcd8e8..0000000 --- a/compass-tasks/actions/health_check/check_package_installer.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Health Check module for Package Installer.""" -import logging -import os -import requests - -from compass.actions.health_check import base -from compass.actions.health_check import utils as health_check_utils - - -class PackageInstallerCheck(base.BaseCheck): - """package installer health check class.""" - NAME = "Package Installer Check" - - def run(self): - """do health check.""" - method_name = "self." + self.package_installer['name'] + "_check()" - return eval(method_name) - - def chef_check(self): - """Checks chef setting, cookbooks and roles.""" - self.check_chef_config_dir() - print "[Done]" - if self.code == 1: - self.messages.append( - "[%s]Info: Package installer health check " - "has completed. No problems found, all systems " - "go." % self.NAME) - - return (self.code, self.messages) - - def check_chef_config_dir(self): - """Validates chef configuration directories.""" - - print "Checking Chef configurations......", - message = health_check_utils.check_path(self.NAME, '/etc/chef-server/') - if not message == "": - self._set_status(0, message) - - message = health_check_utils.check_path(self.NAME, '/opt/chef-server/') - if not message == "": - self._set_status(0, message) - - return None - - def ansible_check(self): - """Placeholder for ansible check.""" - print "Checking ansible......" - print ("[Done]") - self.code == 1 - self.messages.append( - "[%s]Info: Package installer health check " - "has completed. No problems found, all systems " - "go." % self.NAME) - return (self.code, self.messages) diff --git a/compass-tasks/actions/health_check/check_squid.py b/compass-tasks/actions/health_check/check_squid.py deleted file mode 100644 index 5628a63..0000000 --- a/compass-tasks/actions/health_check/check_squid.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Health Check module for Squid service.""" -import commands -import os -import pwd -import socket - -from compass.actions.health_check import base -from compass.actions.health_check import utils as health_check_utils - - -class SquidCheck(base.BaseCheck): - """Squid health check class.""" - NAME = "Squid Check" - - def run(self): - """do health check.""" - self.check_squid_files() - print "[Done]" - self.check_squid_service() - print "[Done]" - if self.code == 1: - self.messages.append( - "[%s]Info: Squid health check has completed. " - "No problems found, all systems go." % self.NAME) - return (self.code, self.messages) - - def check_squid_files(self): - """Validates squid config, cache directory and ownership.""" - print "Checking Squid Files......", - var_map = { - 'match_squid_conf': False, - 'match_squid_cache': False, - 'match_squid_ownership': False, - } - - conf_err_msg = health_check_utils.check_path( - self.NAME, - "/etc/squid/squid.conf") - if not conf_err_msg == "": - self._set_status(0, conf_err_msg) - elif int(oct(os.stat('/etc/squid/squid.conf').st_mode)) < 100644: - self._set_status( - 0, - "[%s]Error: squid.conf has incorrect " - "file permissions" % self.NAME) - else: - var_map['match_squid_conf'] = True - - squid_path_err_msg = health_check_utils.check_path( - self.NAME, '/var/squid/') - if not squid_path_err_msg == "": - self._set_status(0, squid_path_err_msg) - elif health_check_utils.check_path( - self.NAME, - '/var/squid/cache' - ) != "": - self._set_status( - 0, - health_check_utils.check_path( - self.NAME, - '/var/squid/cache' - ) - ) - else: - var_map['match_squid_cache'] = True - uid = os.stat('/var/squid/').st_uid - gid = os.stat('/var/squid/').st_gid - if uid != gid or pwd.getpwuid(23).pw_name != 'squid': - self._set_status( - 0, - "[%s]Error: /var/squid directory ownership " - "misconfigured" % self.NAME) - else: - var_map['match_squid_ownership'] = True - - fails = [] - for key in var_map.keys(): - if var_map[key] is False: - fails.append(key) - - if len(fails) != 0: - self.messages.append( - "[%s]Info: Failed components for squid config: " - "%s" % ( - self.NAME, - ', '.join(item for item in fails) - ) - ) - return True - - def check_squid_service(self): - """Checks if squid is running on port 3128.""" - - print "Checking Squid service......", - if 'squid' not in commands.getoutput('ps -ef'): - self._set_status( - 0, - "[%s]Error: squid service does not seem " - "running" % self.NAME) - - try: - if 'squid' != socket.getservbyport(3128): - self._set_status( - 0, - "[%s]Error: squid is not listening on " - "3128" % self.NAME) - - except Exception: - self._set_status( - 0, - "[%s]Error: No service is listening on 3128, " - "squid failed" % self.NAME) - - return True diff --git a/compass-tasks/actions/health_check/check_tftp.py b/compass-tasks/actions/health_check/check_tftp.py deleted file mode 100644 index 7ca6405..0000000 --- a/compass-tasks/actions/health_check/check_tftp.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Health Check module for TFTP service.""" -import os -import socket -import xmlrpclib - -from compass.actions.health_check import base -from compass.actions.health_check import utils as health_check_utils - - -class TftpCheck(base.BaseCheck): - """tftp health check class.""" - NAME = "TFTP Check" - - def run(self): - """do health check.""" - method_name = "self.check_" + self.os_installer['name'] + "_tftp()" - return eval(method_name) - - def check_cobbler_tftp(self): - """Checks if Cobbler manages TFTP service. - - :note: we assume TFTP service is running at the - same machine where this health check runs at - """ - - try: - remote = xmlrpclib.Server( - self.os_installer['cobbler_url'], - allow_none=True) - credentials = self.os_installer['credentials'] - remote.login( - credentials['username'], credentials['password']) - except Exception: - self._set_status( - 0, - "[%s]Error: Cannot login to Cobbler with the tokens " - " provided in the config file" % self.NAME) - return (self.code, self.messages) - - cobbler_settings = remote.get_settings() - if cobbler_settings['manage_tftp'] == 0: - self.messages.append( - '[TFTP]Info: tftp service is not managed by Compass') - return (0, self.messages) - self.check_tftp_dir() - print "[Done]" - self.check_tftp_service() - print "[Done]" - if self.code == 1: - self.messages.append( - "[%s]Info: tftp service health check has completed. " - "No problems found, all systems go." % self.NAME) - - return (self.code, self.messages) - - def check_tftp_dir(self): - """Validates TFTP directories and configurations.""" - print "Checking TFTP directories......", - if not os.path.exists('/var/lib/tftpboot/'): - self._set_status( - 0, - "[%s]Error: No tftp-boot libraries found, " - "please check if tftp server is properly " - "installed/managed" % self.NAME) - - return True - - def check_tftp_service(self): - """Checks if TFTP is running on port 69.""" - print "Checking TFTP services......", - serv_err_msg = health_check_utils.check_service_running(self.NAME, - 'xinetd') - if not serv_err_msg == "": - self._set_status(0, serv_err_msg) - - if 'tftp' != socket.getservbyport(69): - self._set_status( - 0, - "[%s]Error: tftp doesn't seem to be listening " - "on Port 60." % self.NAME) - - return True diff --git a/compass-tasks/actions/health_check/utils.py b/compass-tasks/actions/health_check/utils.py deleted file mode 100644 index 369c5b6..0000000 --- a/compass-tasks/actions/health_check/utils.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Compass Health Check heavy-lifting utilities""" -import commands -import os -import platform -import re - - -def validate_setting(module, setting, param): - """Checks if a Compass setting exists in the config file. - - :param module : module name to be checked - :type module : string - :param setting : compass setting wrapper - :type setting : python module - :param param : settings defined in compass config file - :type param : string - - """ - if hasattr(setting, param): - return True - else: - err_msg = "[%s]Error: no %s defined" % (module, param) - return err_msg - - -def get_dist(): - """Returns the operating system related information.""" - - os_version, version, release = platform.linux_distribution() - return (os_version.lower().strip(), version, release.lower().strip()) - - -def check_path(module_name, path): - """Checks if a directory or file exisits. - - :param module_name : module name to be checked - :type module_name : string - :param path : path of the directory of file - :type path : string - - """ - err_msg = "" - if not os.path.exists(path): - err_msg = ( - "[%s]Error: %s does not exist, " - "please check your configurations.") % (module_name, path) - return err_msg - - -def check_service_running(module_name, service_name): - """Checks if a certain service is running. - - :param module_name : module name to be checked - :type module_name : string - :param service_name : service name to be checked - :type service_name : string - - """ - err_msg = "" - if service_name not in commands.getoutput('ps -ef'): - err_msg = "[%s]Error: %s is not running." % ( - module_name, service_name) - - return err_msg - - -def check_chkconfig(service_name): - """Checks if a service is enabled at the start up. - - :param service_name : service name to be checked - :type service_name : string - - """ - chk_on = False - for service in os.listdir('/etc/rc3.d/'): - if service_name in service and 'S' in service: - chk_on = True - break - - return chk_on - - -def strip_name(name): - """Reformats names.""" - if not any([s in name for s in "(,),-,_".split(',')]): - return name - - paren_regex = re.compile("(.*?)\s*\(") - dash_regex = re.compile("(.*?)\s*\-") - under_dash_regex = re.compile("(.*?)\s*\_") - - r1 = paren_regex.match(name) - r2 = dash_regex.match(name) - r3 = under_dash_regex.match(name) - shortest = 'AVeryLongStringForDefualt' - for r in [r1, r2, r3]: - if r and len(r.group(1)) < len(shortest): - shortest = r.group(1) - - return shortest diff --git a/compass-tasks/actions/install_callback.py b/compass-tasks/actions/install_callback.py deleted file mode 100644 index 14d2639..0000000 --- a/compass-tasks/actions/install_callback.py +++ /dev/null @@ -1,181 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to receive installation callback. - - .. moduleauthor:: Xiaodong Wang -""" -import logging - -from compass.actions import util -from compass.db.api import cluster as cluster_api -from compass.db.api import host as host_api -from compass.db.api import user as user_db -from compass.deployment.deploy_manager import DeployManager -from compass.deployment.utils import constants as const - - -def os_installed( - host_id, clusterhosts_ready, clusters_os_ready, - username=None -): - """Callback when os is installed. - - :param host_id: host that os is installed. - :type host_id: integer - :param clusterhosts_ready: the clusterhosts that should trigger ready. - :param clusters_os_ready: the cluster that should trigger os ready. - - .. note:: - The function should be called out of database session. - """ - with util.lock('serialized_action') as lock: - if not lock: - raise Exception( - 'failed to acquire lock to ' - 'do the post action after os installation' - ) - logging.info( - 'os installed on host %s ' - 'with cluster host ready %s cluster os ready %s', - host_id, clusterhosts_ready, clusters_os_ready - ) - if username: - user = user_db.get_user_object(username) - else: - user = None - os_installed_triggered = False - for cluster_id, clusterhost_ready in clusterhosts_ready.items(): - if not clusterhost_ready and os_installed_triggered: - continue - cluster_id = int(cluster_id) - cluster_info = util.ActionHelper.get_cluster_info( - cluster_id, user) - adapter_id = cluster_info[const.ADAPTER_ID] - - adapter_info = util.ActionHelper.get_adapter_info( - adapter_id, cluster_id, user) - hosts_info = util.ActionHelper.get_hosts_info( - cluster_id, [host_id], user) - - deploy_manager = DeployManager( - adapter_info, cluster_info, hosts_info) - - if not os_installed_triggered: - deploy_manager.os_installed() - util.ActionHelper.host_ready(host_id, True, user) - os_installed_triggered = True - - if clusterhost_ready: - # deploy_manager.cluster_os_installed() - util.ActionHelper.cluster_host_ready( - cluster_id, host_id, False, user - ) - - if util.ActionHelper.is_cluster_os_ready(cluster_id, user): - logging.info("deploy_manager begin cluster_os_installed") - deploy_manager.cluster_os_installed() - - -def package_installed( - cluster_id, host_id, cluster_ready, - host_ready, username=None -): - """Callback when package is installed. - - :param cluster_id: cluster id. - :param host_id: host id. - :param cluster_ready: if the cluster should trigger ready. - :param host_ready: if the host should trigger ready. - - .. note:: - The function should be called out of database session. - """ - with util.lock('serialized_action') as lock: - if not lock: - raise Exception( - 'failed to acquire lock to ' - 'do the post action after package installation' - ) - logging.info( - 'package installed on cluster %s host %s ' - 'with cluster ready %s host ready %s', - cluster_id, host_id, cluster_ready, host_ready - ) - - if username: - user = user_db.get_user_object(username) - else: - user = None - cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) - adapter_id = cluster_info[const.ADAPTER_ID] - - adapter_info = util.ActionHelper.get_adapter_info( - adapter_id, cluster_id, user) - hosts_info = util.ActionHelper.get_hosts_info( - cluster_id, [host_id], user) - - deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) - - deploy_manager.package_installed() - util.ActionHelper.cluster_host_ready(cluster_id, host_id, True, user) - if cluster_ready: - util.ActionHelper.cluster_ready(cluster_id, False, user) - if host_ready: - util.ActionHelper.host_ready(host_id, False, user) - - -def cluster_installed( - cluster_id, clusterhosts_ready, - username=None -): - """Callback when cluster is installed. - - :param cluster_id: cluster id - :param clusterhosts_ready: clusterhosts that should trigger ready. - - .. note:: - The function should be called out of database session. - """ - with util.lock('serialized_action') as lock: - if not lock: - raise Exception( - 'failed to acquire lock to ' - 'do the post action after cluster installation' - ) - logging.info( - 'package installed on cluster %s with clusterhosts ready %s', - cluster_id, clusterhosts_ready - ) - if username: - user = user_db.get_user_object(username) - else: - user = None - cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) - adapter_id = cluster_info[const.ADAPTER_ID] - - adapter_info = util.ActionHelper.get_adapter_info( - adapter_id, cluster_id, user) - hosts_info = util.ActionHelper.get_hosts_info( - cluster_id, clusterhosts_ready.keys(), user) - - deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) - - deploy_manager.cluster_installed() - util.ActionHelper.cluster_ready(cluster_id, True, user) - for host_id, clusterhost_ready in clusterhosts_ready.items(): - if clusterhost_ready: - util.ActionHelper.cluster_host_ready( - cluster_id, host_id, False, user - ) diff --git a/compass-tasks/actions/patch.py b/compass-tasks/actions/patch.py deleted file mode 100644 index 6d29be6..0000000 --- a/compass-tasks/actions/patch.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to patch an existing cluster -""" -import logging -import simplejson as json - -from compass.actions import util -from compass.db.api import cluster as cluster_db -from compass.db.api import user as user_db -from compass.deployment.deploy_manager import Patcher -from compass.deployment.utils import constants as const - - -def patch(cluster_id, username=None): - """Patch cluster. - - :param cluster_id: id of the cluster - :type cluster_id: int - - .. note:: - The function should be called out of database session. - """ - with util.lock('serialized_action', timeout=1000) as lock: - if not lock: - raise Exception('failed to acquire lock to deploy') - - user = user_db.get_user_object(username) - cluster_hosts = cluster_db.list_cluster_hosts(cluster_id, user) - hosts_id_list = [host['id'] for host in cluster_hosts] - cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) - adapter_id = cluster_info[const.ADAPTER_ID] - - adapter_info = util.ActionHelper.get_adapter_info( - adapter_id, cluster_id, user) - hosts_info = util.ActionHelper.get_hosts_info( - cluster_id, hosts_id_list, user) - patch_successful = True - try: - patcher = Patcher( - adapter_info, cluster_info, hosts_info, cluster_hosts) - patched_config = patcher.patch() - except Exception as error: - logging.exception(error) - patch_successful = False - - if patch_successful: - clean_payload = '{"patched_roles": []}' - clean_payload = json.loads(clean_payload) - for cluster_host in cluster_hosts: - cluster_db.update_cluster_host( - cluster_id, cluster_host['id'], user, **clean_payload) - logging.info( - "cleaning up patched roles for host id: %s", - cluster_host['id'] - ) - logging.info("Patch successful: %s", patched_config) diff --git a/compass-tasks/actions/poll_switch.py b/compass-tasks/actions/poll_switch.py deleted file mode 100644 index 5c29b01..0000000 --- a/compass-tasks/actions/poll_switch.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to provider function to poll switch.""" -import logging -import netaddr - -from compass.actions import util -from compass.db.api import database -from compass.db.api import switch as switch_api -from compass.db.api import user as user_api -from compass.hdsdiscovery.hdmanager import HDManager - - -def _poll_switch(ip_addr, credentials, req_obj='mac', oper="SCAN"): - """Poll switch by ip addr. - - - Args: - ip_addr: ip addr of the switch. - credentials: credentials of the switch. - - Returns: switch attributes dict and list of machine attributes dict. - """ - under_monitoring = 'under_monitoring' - unreachable = 'unreachable' - polling_error = 'error' - hdmanager = HDManager() - vendor, state, err_msg = hdmanager.get_vendor(ip_addr, credentials) - if not vendor: - logging.info("*****error_msg: %s****", err_msg) - logging.error('no vendor found or match switch %s', ip_addr) - return ( - { - 'vendor': vendor, 'state': state, 'err_msg': err_msg - }, { - } - ) - - logging.debug( - 'hdmanager learn switch from %s', ip_addr - ) - results = [] - try: - results = hdmanager.learn( - ip_addr, credentials, vendor, req_obj, oper - ) - except Exception as error: - logging.exception(error) - state = unreachable - err_msg = ( - 'SNMP walk for querying MAC addresses timedout' - ) - return ( - { - 'vendor': vendor, 'state': state, 'err_msg': err_msg - }, { - } - ) - - logging.info("pollswitch %s result: %s", ip_addr, results) - if not results: - logging.error( - 'no result learned from %s', ip_addr - ) - state = polling_error - err_msg = 'No result learned from SNMP walk' - return ( - {'vendor': vendor, 'state': state, 'err_msg': err_msg}, - {} - ) - - logging.info('poll switch result: %s' % str(results)) - machine_dicts = {} - for machine in results: - mac = machine['mac'] - port = machine['port'] - vlan = int(machine['vlan']) - if vlan: - vlans = [vlan] - else: - vlans = [] - if mac not in machine_dicts: - machine_dicts[mac] = {'mac': mac, 'port': port, 'vlans': vlans} - else: - machine_dicts[mac]['port'] = port - machine_dicts[mac]['vlans'].extend(vlans) - - logging.debug('update switch %s state to under monitoring', ip_addr) - state = under_monitoring - return ( - {'vendor': vendor, 'state': state, 'err_msg': err_msg}, - machine_dicts.values() - ) - - -def poll_switch(poller_email, ip_addr, credentials, - req_obj='mac', oper="SCAN"): - """Query switch and update switch machines. - - .. note:: - When polling switch succeeds, for each mac it got from polling switch, - A Machine record associated with the switch is added to the database. - - :param ip_addr: switch ip address. - :type ip_addr: str - :param credentials: switch crednetials. - :type credentials: dict - :param req_obj: the object requested to query from switch. - :type req_obj: str - :param oper: the operation to query the switch. - :type oper: str, should be one of ['SCAN', 'GET', 'SET'] - - .. note:: - The function should be called out of database session scope. - """ - poller = user_api.get_user_object(poller_email) - ip_int = long(netaddr.IPAddress(ip_addr)) - with util.lock('poll switch %s' % ip_addr, timeout=120) as lock: - if not lock: - raise Exception( - 'failed to acquire lock to poll switch %s' % ip_addr - ) - - # TODO(grace): before repoll the switch, set the state to repolling. - # and when the poll switch is timeout, set the state to error. - # the frontend should only consider some main state like INTIALIZED, - # ERROR and SUCCESSFUL, REPOLLING is as an intermediate state to - # indicate the switch is in learning the mac of the machines connected - # to it. - logging.debug('poll switch: %s', ip_addr) - switch_dict, machine_dicts = _poll_switch( - ip_addr, credentials, req_obj=req_obj, oper=oper - ) - switches = switch_api.list_switches(ip_int=ip_int, user=poller) - if not switches: - logging.error('no switch found for %s', ip_addr) - return - - for switch in switches: - for machine_dict in machine_dicts: - logging.info('add machine: %s', machine_dict) - machine_dict['owner_id'] = poller.id - switch_api.add_switch_machine( - switch['id'], False, user=poller, **machine_dict - ) - switch_api.update_switch( - switch['id'], - user=poller, - **switch_dict - ) diff --git a/compass-tasks/actions/reinstall.py b/compass-tasks/actions/reinstall.py deleted file mode 100644 index 62d1bcb..0000000 --- a/compass-tasks/actions/reinstall.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to reinstall a given cluster - - .. moduleauthor:: Xiaodong Wang -""" -import logging - -from compass.actions import util -from compass.db.api import database - - -def reinstall(cluster_hosts): - """Reinstall clusters. - - :param cluster_hosts: clusters and hosts in each cluster to reinstall. - :type cluster_hosts: dict of int or str to list of int or str - - .. note:: - The function should be called out of database session. - """ - with util.lock('serialized_action') as lock: - if not lock: - raise Exception( - 'failed to acquire lock to reinstall') - logging.debug('reinstall cluster_hosts: %s', cluster_hosts) diff --git a/compass-tasks/actions/search.py b/compass-tasks/actions/search.py deleted file mode 100644 index 73ce1d9..0000000 --- a/compass-tasks/actions/search.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to search configs of given clusters - - .. moduleauthor:: Xiaodong Wang -""" -import logging - -from compass.actions import util -from compass.config_management.utils.config_manager import ConfigManager -from compass.db.api import database - - -def search(cluster_hosts, cluster_propreties_match, - cluster_properties_name, host_properties_match, - host_properties_name): - """search clusters. - - :param cluster_hosts: clusters and hosts in each cluster to search. - :type cluster_hosts: dict of int or str to list of int or str - - .. note:: - The function should be called out of database session. - """ - logging.debug('search cluster_hosts: %s', cluster_hosts) - with database.session(): - cluster_hosts, os_versions, target_systems = ( - util.update_cluster_hosts(cluster_hosts)) - manager = ConfigManager() - return manager.filter_cluster_and_hosts( - cluster_hosts, os_versions, - target_systems, cluster_propreties_match, - cluster_properties_name, host_properties_match, - host_properties_name) diff --git a/compass-tasks/actions/update_progress.py b/compass-tasks/actions/update_progress.py deleted file mode 100644 index 67a9963..0000000 --- a/compass-tasks/actions/update_progress.py +++ /dev/null @@ -1,298 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to update status and installing progress of the given cluster. - - .. moduleauthor:: Xiaodong Wang -""" -import logging - -from compass.actions import util -from compass.db.api import adapter_holder as adapter_api -from compass.db.api import cluster as cluster_api -from compass.db.api import host as host_api -from compass.db.api import user as user_api -from compass.log_analyzor import progress_calculator -from compass.utils import setting_wrapper as setting - - -def update_progress(): - """Update status and installing progress of the given cluster. - - :param cluster_hosts: clusters and hosts in each cluster to update. - :type cluster_hosts: dict of int or str to list of int or str - - .. note:: - The function should be called out of the database session scope. - In the function, it will update the database cluster_state and - host_state table for the deploying cluster and hosts. - - The function will also query log_progressing_history table to get - the lastest installing progress and the position of log it has - processed in the last run. The function uses these information to - avoid recalculate the progress from the beginning of the log file. - After the progress got updated, these information will be stored back - to the log_progressing_history for next time run. - """ - with util.lock('log_progressing', timeout=60, blocking=False) as lock: - if not lock: - logging.error( - 'failed to acquire lock to calculate installation progress' - ) - return - - logging.info('update installing progress') - - user = user_api.get_user_object(setting.COMPASS_ADMIN_EMAIL) - hosts = host_api.list_hosts(user=user) - host_mapping = {} - for host in hosts: - if 'id' not in host: - logging.error('id is not in host %s', host) - continue - host_id = host['id'] - if 'os_name' not in host: - logging.error('os_name is not in host %s', host) - continue - if 'os_installer' not in host: - logging.error('os_installer is not in host %s', host) - continue - host_dirname = setting.HOST_INSTALLATION_LOGDIR_NAME - if host_dirname not in host: - logging.error( - '%s is not in host %s', host_dirname, host - ) - continue - host_state = host_api.get_host_state(host_id, user=user) - if 'state' not in host_state: - logging.error('state is not in host state %s', host_state) - continue - if host_state['state'] == 'INSTALLING': - host_log_histories = host_api.get_host_log_histories( - host_id, user=user - ) - host_log_history_mapping = {} - for host_log_history in host_log_histories: - if 'filename' not in host_log_history: - logging.error( - 'filename is not in host log history %s', - host_log_history - ) - continue - host_log_history_mapping[ - host_log_history['filename'] - ] = host_log_history - host_mapping[host_id] = ( - host, host_state, host_log_history_mapping - ) - else: - logging.info( - 'ignore host state %s since it is not in installing', - host_state - ) - adapters = adapter_api.list_adapters(user=user) - adapter_mapping = {} - for adapter in adapters: - if 'id' not in adapter: - logging.error( - 'id not in adapter %s', adapter - ) - continue - if 'package_installer' not in adapter: - logging.info( - 'package_installer not in adapter %s', adapter - ) - continue - adapter_id = adapter['id'] - adapter_mapping[adapter_id] = adapter - clusters = cluster_api.list_clusters(user=user) - cluster_mapping = {} - for cluster in clusters: - if 'id' not in cluster: - logging.error('id not in cluster %s', cluster) - continue - cluster_id = cluster['id'] - if 'adapter_id' not in cluster: - logging.error( - 'adapter_id not in cluster %s', - cluster - ) - continue - cluster_state = cluster_api.get_cluster_state( - cluster_id, - user=user - ) - if 'state' not in cluster_state: - logging.error('state not in cluster state %s', cluster_state) - continue - cluster_mapping[cluster_id] = (cluster, cluster_state) - clusterhosts = cluster_api.list_clusterhosts(user=user) - clusterhost_mapping = {} - for clusterhost in clusterhosts: - if 'clusterhost_id' not in clusterhost: - logging.error( - 'clusterhost_id not in clusterhost %s', - clusterhost - ) - continue - clusterhost_id = clusterhost['clusterhost_id'] - if 'cluster_id' not in clusterhost: - logging.error( - 'cluster_id not in clusterhost %s', - clusterhost - ) - continue - cluster_id = clusterhost['cluster_id'] - if cluster_id not in cluster_mapping: - logging.info( - 'ignore clusterhost %s ' - 'since the cluster_id ' - 'is not in cluster_mapping %s', - clusterhost, cluster_mapping - ) - continue - cluster, _ = cluster_mapping[cluster_id] - if 'flavor_name' not in cluster: - logging.error( - 'flavor_name is not in clusterhost %s related cluster', - clusterhost - ) - continue - clusterhost_dirname = setting.CLUSTERHOST_INATALLATION_LOGDIR_NAME - if clusterhost_dirname not in clusterhost: - logging.error( - '%s is not in clusterhost %s', - clusterhost_dirname, clusterhost - ) - continue - adapter_id = cluster['adapter_id'] - if adapter_id not in adapter_mapping: - logging.info( - 'ignore clusterhost %s ' - 'since the adapter_id %s ' - 'is not in adaper_mapping %s', - clusterhost, adapter_id, adapter_mapping - ) - continue - adapter = adapter_mapping[adapter_id] - if 'package_installer' not in adapter: - logging.info( - 'ignore clusterhost %s ' - 'since the package_installer is not define ' - 'in adapter %s', - clusterhost, adapter - ) - continue - package_installer = adapter['package_installer'] - clusterhost['package_installer'] = package_installer - clusterhost['adapter_name'] = adapter['name'] - clusterhost_state = cluster_api.get_clusterhost_self_state( - clusterhost_id, user=user - ) - if 'state' not in clusterhost_state: - logging.error( - 'state not in clusterhost_state %s', - clusterhost_state - ) - continue - if clusterhost_state['state'] == 'INSTALLING': - clusterhost_log_histories = ( - cluster_api.get_clusterhost_log_histories( - clusterhost_id, user=user - ) - ) - clusterhost_log_history_mapping = {} - for clusterhost_log_history in clusterhost_log_histories: - if 'filename' not in clusterhost_log_history: - logging.error( - 'filename not in clusterhost_log_history %s', - clusterhost_log_history - ) - continue - clusterhost_log_history_mapping[ - clusterhost_log_history['filename'] - ] = clusterhost_log_history - clusterhost_mapping[clusterhost_id] = ( - clusterhost, clusterhost_state, - clusterhost_log_history_mapping - ) - else: - logging.info( - 'ignore clusterhost state %s ' - 'since it is not in installing', - clusterhost_state - ) - - progress_calculator.update_host_progress( - host_mapping) - for host_id, (host, host_state, host_log_history_mapping) in ( - host_mapping.items() - ): - host_api.update_host_state( - host_id, user=user, - percentage=host_state.get('percentage', 0), - message=host_state.get('message', ''), - severity=host_state.get('severity', 'INFO') - ) - for filename, host_log_history in ( - host_log_history_mapping.items() - ): - host_api.add_host_log_history( - host_id, filename=filename, user=user, - position=host_log_history.get('position', 0), - percentage=host_log_history.get('percentage', 0), - partial_line=host_log_history.get('partial_line', ''), - message=host_log_history.get('message', ''), - severity=host_log_history.get('severity', 'INFO'), - line_matcher_name=host_log_history.get( - 'line_matcher_name', 'start' - ) - ) - progress_calculator.update_clusterhost_progress( - clusterhost_mapping) - for ( - clusterhost_id, - (clusterhost, clusterhost_state, clusterhost_log_history_mapping) - ) in ( - clusterhost_mapping.items() - ): - cluster_api.update_clusterhost_state( - clusterhost_id, user=user, - percentage=clusterhost_state.get('percentage', 0), - message=clusterhost_state.get('message', ''), - severity=clusterhost_state.get('severity', 'INFO') - ) - for filename, clusterhost_log_history in ( - clusterhost_log_history_mapping.items() - ): - cluster_api.add_clusterhost_log_history( - clusterhost_id, user=user, filename=filename, - position=clusterhost_log_history.get('position', 0), - percentage=clusterhost_log_history.get('percentage', 0), - partial_line=clusterhost_log_history.get( - 'partial_line', ''), - message=clusterhost_log_history.get('message', ''), - severity=clusterhost_log_history.get('severity', 'INFO'), - line_matcher_name=( - clusterhost_log_history.get( - 'line_matcher_name', 'start' - ) - ) - ) - progress_calculator.update_cluster_progress( - cluster_mapping) - for cluster_id, (cluster, cluster_state) in cluster_mapping.items(): - cluster_api.update_cluster_state( - cluster_id, user=user - ) diff --git a/compass-tasks/actions/util.py b/compass-tasks/actions/util.py deleted file mode 100644 index 4d9f855..0000000 --- a/compass-tasks/actions/util.py +++ /dev/null @@ -1,342 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to provide util for actions - - .. moduleauthor:: Xiaodong Wang ,xiaodongwang@huawei.com> -""" -import logging -import redis - -from contextlib import contextmanager - -from compass.db.api import adapter_holder as adapter_db -from compass.db.api import cluster as cluster_db -from compass.db.api import host as host_db -from compass.db.api import machine as machine_db -from compass.deployment.utils import constants as const - - -@contextmanager -def lock(lock_name, blocking=True, timeout=10): - """acquire a lock to do some actions. - - The lock is acquired by lock_name among the whole distributed - systems. - """ - # TODO(xicheng): in future we should explicitly told which redis - # server we want to talk to make the lock works on distributed - # systems. - redis_instance = redis.Redis() - instance_lock = redis_instance.lock(lock_name, timeout=timeout) - owned = False - try: - locked = instance_lock.acquire(blocking=blocking) - if locked: - owned = True - logging.debug('acquired lock %s', lock_name) - yield instance_lock - else: - logging.info('lock %s is already hold', lock_name) - yield None - - except Exception as error: - logging.info( - 'redis fails to acquire the lock %s', lock_name) - logging.exception(error) - yield None - - finally: - if owned: - instance_lock.acquired_until = 0 - instance_lock.release() - logging.debug('released lock %s', lock_name) - else: - logging.debug('nothing to release %s', lock_name) - - -class ActionHelper(object): - - @staticmethod - def get_adapter_info(adapter_id, cluster_id, user): - """Get adapter information. Return a dictionary as below, - - { - "id": 1, - "name": "xxx", - "flavors": [ - { - "flavor_name": "xxx", - "roles": ['xxx', 'yyy', ...], - "template": "xxx.tmpl" - }, - ... - ], - "metadata": { - "os_config": { - ... - }, - "package_config": { - ... - } - }, - "os_installer": { - "name": "cobbler", - "settings": {....} - }, - "pk_installer": { - "name": "chef", - "settings": {....} - }, - ... - } - To view a complete output, please refer to backend doc. - """ - - adapter_info = adapter_db.get_adapter(adapter_id, user=user) - metadata = cluster_db.get_cluster_metadata(cluster_id, user=user) - adapter_info.update({const.METADATA: metadata}) - - for flavor_info in adapter_info[const.FLAVORS]: - roles = flavor_info[const.ROLES] - flavor_info[const.ROLES] = ActionHelper._get_role_names(roles) - - return adapter_info - - @staticmethod - def _get_role_names(roles): - return [role[const.NAME] for role in roles] - - @staticmethod - def get_cluster_info(cluster_id, user): - """Get cluster information.Return a dictionary as below, - - { - "id": 1, - "adapter_id": 1, - "os_version": "CentOS-6.5-x86_64", - "name": "cluster_01", - "flavor": { - "flavor_name": "zzz", - "template": "xx.tmpl", - "roles": [...] - } - "os_config": {..}, - "package_config": {...}, - "deployed_os_config": {}, - "deployed_package_config": {}, - "owner": "xxx" - } - """ - - cluster_info = cluster_db.get_cluster(cluster_id, user=user) - - # convert roles retrieved from db into a list of role names - roles_info = cluster_info.setdefault( - const.FLAVOR, {}).setdefault(const.ROLES, []) - cluster_info[const.FLAVOR][const.ROLES] = \ - ActionHelper._get_role_names(roles_info) - - # get cluster config info - cluster_config = cluster_db.get_cluster_config(cluster_id, user=user) - cluster_info.update(cluster_config) - - deploy_config = cluster_db.get_cluster_deployed_config(cluster_id, - user=user) - cluster_info.update(deploy_config) - - return cluster_info - - @staticmethod - def get_hosts_info(cluster_id, hosts_id_list, user): - """Get hosts information. Return a dictionary as below, - - { - "hosts": { - 1($host_id): { - "reinstall_os": True, - "mac": "xxx", - "name": "xxx", - "roles": [xxx, yyy] - }, - "networks": { - "eth0": { - "ip": "192.168.1.1", - "netmask": "255.255.255.0", - "is_mgmt": True, - "is_promiscuous": False, - "subnet": "192.168.1.0/24" - }, - "eth1": {...} - }, - "os_config": {}, - "package_config": {}, - "deployed_os_config": {}, - "deployed_package_config": {} - }, - 2: {...}, - .... - } - } - """ - - hosts_info = {} - for host_id in hosts_id_list: - info = cluster_db.get_cluster_host(cluster_id, host_id, user=user) - logging.debug("checking on info %r %r" % (host_id, info)) - - info[const.ROLES] = ActionHelper._get_role_names(info[const.ROLES]) - - # TODO(grace): Is following line necessary?? - info.setdefault(const.ROLES, []) - - config = cluster_db.get_cluster_host_config(cluster_id, - host_id, - user=user) - info.update(config) - - networks = info[const.NETWORKS] - networks_dict = {} - # Convert networks from list to dictionary format - for entry in networks: - nic_info = {} - nic_info = { - entry[const.NIC]: { - const.IP_ADDR: entry[const.IP_ADDR], - const.NETMASK: entry[const.NETMASK], - const.MGMT_NIC_FLAG: entry[const.MGMT_NIC_FLAG], - const.PROMISCUOUS_FLAG: entry[const.PROMISCUOUS_FLAG], - const.SUBNET: entry[const.SUBNET] - } - } - networks_dict.update(nic_info) - - info[const.NETWORKS] = networks_dict - - hosts_info[host_id] = info - - return hosts_info - - @staticmethod - def save_deployed_config(deployed_config, user): - """Save deployed config.""" - cluster_config = deployed_config[const.CLUSTER] - cluster_id = cluster_config[const.ID] - del cluster_config[const.ID] - - cluster_db.update_cluster_deployed_config(cluster_id, user=user, - **cluster_config) - - hosts_id_list = deployed_config[const.HOSTS].keys() - for host_id in hosts_id_list: - config = deployed_config[const.HOSTS][host_id] - cluster_db.update_cluster_host_deployed_config(cluster_id, - host_id, - user=user, - **config) - - @staticmethod - def update_state( - cluster_id, host_id_list, user, **kwargs - ): - # update all clusterhosts state - for host_id in host_id_list: - cluster_db.update_cluster_host_state( - cluster_id, - host_id, - user=user, - **kwargs - ) - - # update cluster state - cluster_db.update_cluster_state( - cluster_id, - user=user, - **kwargs - ) - - @staticmethod - def delete_cluster( - cluster_id, host_id_list, user, delete_underlying_host=False - ): - """Delete cluster. - - If delete_underlying_host is set, underlying hosts will also - be deleted. - """ - if delete_underlying_host: - for host_id in host_id_list: - host_db.del_host( - host_id, True, True, user=user - ) - cluster_db.del_cluster( - cluster_id, True, True, user=user - ) - - @staticmethod - def delete_cluster_host( - cluster_id, host_id, user, delete_underlying_host=False - ): - """Delete clusterhost. - - If delete_underlying_host set, also delete underlying host. - """ - if delete_underlying_host: - host_db.del_host( - host_id, True, True, user=user - ) - cluster_db.del_cluster_host( - cluster_id, host_id, True, True, user=user - ) - - @staticmethod - def delete_host(host_id, user): - host_db.del_host( - host_id, True, True, user=user - ) - - @staticmethod - def host_ready(host_id, from_database_only, user): - """Trigger host ready.""" - host_db.update_host_state_internal( - host_id, from_database_only=from_database_only, - user=user, ready=True - ) - - @staticmethod - def cluster_host_ready( - cluster_id, host_id, from_database_only, user - ): - """Trigger clusterhost ready.""" - cluster_db.update_cluster_host_state_internal( - cluster_id, host_id, from_database_only=from_database_only, - user=user, ready=True - ) - - @staticmethod - def is_cluster_os_ready(cluster_id, user=None): - return cluster_db.is_cluster_os_ready(cluster_id, user=user) - - @staticmethod - def cluster_ready(cluster_id, from_database_only, user): - """Trigger cluster ready.""" - cluster_db.update_cluster_state_internal( - cluster_id, from_database_only=from_database_only, - user=user, ready=True - ) - - @staticmethod - def get_machine_IPMI(machine_id, user): - machine_info = machine_db.get_machine(machine_id, user=user) - return machine_info[const.IPMI_CREDS] diff --git a/compass-tasks/apiclient/__init__.py b/compass-tasks/apiclient/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compass-tasks/apiclient/example.py b/compass-tasks/apiclient/example.py deleted file mode 100755 index 4c01b98..0000000 --- a/compass-tasks/apiclient/example.py +++ /dev/null @@ -1,463 +0,0 @@ -#!/usr/bin/python -# copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Example code to deploy a cluster by compass client api.""" -import os -import re -import sys -import time - -# from compass.apiclient.restful import Client -from restful import Client - -COMPASS_SERVER_URL = 'http://localhost/api' -COMPASS_LOGIN_EMAIL = 'admin@huawei.com' -COMPASS_LOGIN_PASSWORD = 'admin' -SWITCH_IP = '172.29.8.40' -SWITCH_SNMP_VERSION = '2c' -SWITCH_SNMP_COMMUNITY = 'public' -CLUSTER_NAME = 'test_cluster' -HOST_NAME_PREFIX = 'host' -SERVICE_USERNAME = 'service' -SERVICE_PASSWORD = 'service' -CONSOLE_USERNAME = 'console' -CONSOLE_PASSWORD = 'console' -HA_VIP = '' - -MANAGEMENT_IP_START = '10.145.88.130' -MANAGEMENT_IP_END = '10.145.88.254' -MANAGEMENT_IP_GATEWAY = '10.145.88.1' -MANAGEMENT_NETMASK = '255.255.255.0' -MANAGEMENT_NIC = 'eth0' -MANAGEMENT_PROMISC = 0 -TENANT_IP_START = '192.168.10.130' -TENANT_IP_END = '192.168.10.255' -TENANT_IP_GATEWAY = '192.168.10.1' -TENANT_NETMASK = '255.255.255.0' -TENANT_NIC = 'eth0' -TENANT_PROMISC = 0 -PUBLIC_IP_START = '12.234.32.130' -PUBLIC_IP_END = '12.234.32.255' -PUBLIC_IP_GATEWAY = '12.234.32.1' -PUBLIC_NETMASK = '255.255.255.0' -PUBLIC_NIC = 'eth1' -PUBLIC_PROMISC = 1 -STORAGE_IP_START = '172.16.100.130' -STORAGE_IP_END = '172.16.100.255' -STORAGE_NETMASK = '255.255.255.0' -STORAGE_IP_GATEWAY = '172.16.100.1' -STORAGE_NIC = 'eth0' -STORAGE_PROMISC = 0 -HOME_PERCENTAGE = 5 -TMP_PERCENTAGE = 5 -VAR_PERCENTAGE = 10 -HOST_OS = 'CentOS-6.5-x86_64' - - -PRESET_VALUES = { - 'LANGUAGE': 'EN', - 'TIMEZONE': 'GMT', - 'HTTPS_PROXY': 'http://10.145.89.100:3128', - 'NO_PROXY': ['127.0.0.1'], - 'DOMAIN': 'ods.com', - 'NAMESERVERS': ['10.145.89.100'], - 'NTP_SERVER': '10.145.89.100', - 'GATEWAY': '10.145.88.1', - 'PROXY': 'http://10.145.89.100:3128', - 'OS_NAME_PATTERN': 'CentOS.*', - 'ADAPTER_NAME': 'openstack_icehouse', - 'FLAVOR_PATTERN': 'allinone.*', - 'ROLES_LIST': ['allinone-compute'], - 'MACHINES_TO_ADD': ['00:0c:29:a7:ea:4b'], - 'BUILD_TIMEOUT': 60, - 'SEARCH_PATH': ['ods.com'], - 'SERVER_USERNAME': 'root', - 'SERVER_PASSWORD': 'root' -} -for v in PRESET_VALUES: - if v in os.environ.keys(): - PRESET_VALUES[v] = os.environ.get(v) - print (v + PRESET_VALUES[v] + " is set by env variables") - else: - print (PRESET_VALUES[v]) - -# instantiate a client -client = Client(COMPASS_SERVER_URL) - -# login -status, response = client.login(COMPASS_LOGIN_EMAIL, COMPASS_LOGIN_PASSWORD) -print '============================================================' -print 'login status: %s response: %s' % (status, response) -if status >= 400: - sys.exit(1) - -# list all switches -status, response = client.list_switches() -print '=============================================================' -print 'get all switches status: %s response: %s' % (status, response) - -# add a switch -status, response = client.add_switch( - SWITCH_IP, - SWITCH_SNMP_VERSION, - SWITCH_SNMP_COMMUNITY -) -print '============================================' -print 'adding a switch..status: %s, response: %s' % (status, response) - -# if switch already exists, get one from all switches -switch = None -if status < 400: - switch = response -else: - status, response = client.list_switches() - print '=========================================' - print 'list switches status %s response %s' % (status, response) - if status >= 400: - sys.exit(1) - for switch_ in response: - if switch_['ip'] == SWITCH_IP: - switch = switch_ - break - -switch_id = switch['id'] -switch_ip = switch['ip'] -print '======================' -print 'switch has been set as %s' % switch_ip - -# wait till switch state becomes under_monitoring -while switch['state'] != 'under_monitoring': - print 'waiting for state to become under_monitoring' - client.poll_switch(switch_id) - status, resp = client.get_switch(switch_id) - print '=====================================' - print 'poll switch status %s response %s' % (status, resp) - switch = resp - print 'switch is in state: %s' % switch['state'] - time.sleep(5) - -print '=========================================' -print 'switch state now is %s' % (switch['state']) - -# create a machine list -machine_macs = {} -machines = {} -for machine in PRESET_VALUES['MACHINES_TO_ADD']: - status, response = client.list_machines(mac=machine) - print '============================================' - print 'list machines status %s response %s' % (status, response) - if status >= 400: - sys.exit(1) - if status == 200 and response != []: - machine_id = response[0]['id'] - machine_macs[machine_id] = response[0]['mac'] - machines = response - -print '=================================' -print 'found machines are : %s' % machines - -machines_to_add = PRESET_VALUES['MACHINES_TO_ADD'] -if set(machine_macs.values()) != set(machines_to_add): - print 'only found macs %s while expected are %s' % ( - machine_macs.values(), machines_to_add) - sys.exit(1) - -# list all adapters -status, response = client.list_adapters() -print '===============================' -print 'all adapters are: %s' % response -if status >= 400: - sys.exit(1) - -adapters = response -adapter_id = None -os_id = None -flavor_id = None -adapter_name = PRESET_VALUES['ADPATER_NAME'] -os_pattern = re.compile(PRESET_VALUES['OS_NAME_PATTERN']) -flavor_pattern = re.compile(PRESET_VALUES['FLAVOR_PATTERN']) -for adapter in adapters: - if adapter_name == adapter['name']: - adapter_id = adapter['id'] - for supported_os in adapter['supported_oses']: - if os_pattern.match(supported_os['name']): - os_id = supported_os['id'] - break - for flavor in adapter['flavors']: - if flavor_pattern.match(flavor['name']): - flavor_id = flavor['id'] - if adapter_id and os_id and flavor_id: - break - -print '=======================================================' -print 'using adapter %s os %s flavor %s to deploy cluster' % ( - adapter_id, os_id, flavor_id -) - -# add a cluster -status, response = client.add_cluster( - CLUSTER_NAME, - adapter_id, - os_id, - flavor_id -) -print '===============================================================' -print 'add cluster %s status %s: %s' % (CLUSTER_NAME, status, response) -if status >= 400: - sys.exit(1) - -status, response = client.list_clusters(name=CLUSTER_NAME) -print '================================================================' -print 'list clusters %s status %s: %s' % (CLUSTER_NAME, status, response) -if status >= 400: - sys.exit(1) - -cluster = response[0] -cluster_id = cluster['id'] - -print '==================' -print 'cluster is %s' % cluster - -# Add hosts to the cluster -machines_dict = {} -machine_id_list = [] -for machine in machines: - id_mapping = {} - id_mapping['machine_id'] = machine['id'] - machine_id_list.append(id_mapping) - -machines_dict['machines'] = machine_id_list - -status, response = client.add_hosts_to_cluster( - cluster_id, machines_dict -) -print '===================================' -print 'add hosts %s to cluster status %s response %s' % ( - machines_dict, status, response) -if status >= 400: - sys.exit(1) - -# Add two subnets -subnet_1 = '10.145.89.0/24' -subnet_2 = '192.168.100.0/24' - -status, response = client.add_subnet(subnet_1) -print '==================' -print 'add subnet %s status %s: %s' % (subnet_1, status, response) -if status >= 400: - sys.exit(1) - -status, response = client.add_subnet(subnet_2) -print '==================' -print 'add subnet %s status %s: %s' % (subnet_2, status, response) -if status >= 400: - sys.exit(1) - -status, subnet1 = client.list_subnets(subnet=subnet_1) -print '===========================================================' -print 'list subnet %s status %s: %s' % (subnet_1, status, subnet1) -if status >= 400: - sys.exit(1) - -status, subnet2 = client.list_subnets(subnet=subnet_2) -print '===========================================================' -print 'list subnet %s status %s: %s' % (subnet_2, status, subnet2) -if status >= 400: - sys.exit(1) - -subnet1_id = subnet1[0]['id'] -subnet2_id = subnet2[0]['id'] -print '========================' -print 'subnet1 has id: %s, subnet is %s' % (subnet1_id, subnet1) -print 'subnet2 has id: %s, subnet is %s' % (subnet2_id, subnet2) - -# Add host network -status, response = client.list_cluster_hosts(cluster_id) -print '================================================' -print 'list cluster hosts status %s: %s' % (status, response) -if status >= 400: - sys.exit(1) - -host = response[0] -host_id = host['id'] -print '==================' -print 'host is: %s' % host - -status, response = client.add_host_network( - host_id, - 'eth0', - '10.145.89.200', - subnet1_id, - is_mgmt=True -) -print '=======================' -print 'add eth0 network status %s: %s' % (status, response) -if status >= 400: - sys.exit(1) - -status, response = client.add_host_network( - host_id, - 'eth1', - '192.168.100.200', - subnet2_id, - is_promiscuous=True -) -print '=======================' -print 'add eth1 network status %s: %s' % (status, response) -if status >= 400: - sys.exit(1) - -# Update os config to cluster -cluster_os_config = { - 'general': { - 'language': PRESET_VALUES['LANGUAGE'], - 'timezone': PRESET_VALUES['TIMEZONE'], - 'http_proxy': PRESET_VALUES['PROXY'], - 'https_proxy': PRESET_VALUES['HTTPS_PROXY'], - 'no_proxy': PRESET_VALUES['NO_PROXY'], - 'ntp_server': PRESET_VALUES['NTP_SERVER'], - 'dns_servers': PRESET_VALUES['NAMESERVERS'], - 'domain': PRESET_VALUES['DOMAIN'], - 'search_path': PRESET_VALUES['SEARCH_PATH'], - 'default_gateway': PRESET_VALUES['GATEWAY'] - }, - 'server_credentials': { - 'username': PRESET_VALUES['SERVER_USERNAME'], - 'password': PRESET_VALUES['SERVER_PASSWORD'] - }, - 'partition': { - '/var': { - 'percentage': VAR_PERCENTAGE, - }, - '/home': { - 'percentage': HOME_PERCENTAGE, - } - } -} - - -cluster_package_config = { - 'security': { - 'service_credentials': { - 'image': { - 'username': SERVICE_USERNAME, - 'password': SERVICE_PASSWORD - }, - 'compute': { - 'username': SERVICE_USERNAME, - 'password': SERVICE_PASSWORD - }, - 'dashboard': { - 'username': SERVICE_USERNAME, - 'password': SERVICE_PASSWORD - }, - 'identity': { - 'username': SERVICE_USERNAME, - 'password': SERVICE_PASSWORD - }, - 'metering': { - 'username': SERVICE_USERNAME, - 'password': SERVICE_PASSWORD - }, - 'rabbitmq': { - 'username': SERVICE_USERNAME, - 'password': SERVICE_PASSWORD - }, - 'volume': { - 'username': SERVICE_USERNAME, - 'password': SERVICE_PASSWORD - }, - 'mysql': { - 'username': SERVICE_USERNAME, - 'password': SERVICE_PASSWORD - } - }, - 'console_credentials': { - 'admin': { - 'username': CONSOLE_USERNAME, - 'password': CONSOLE_PASSWORD - }, - 'compute': { - 'username': CONSOLE_USERNAME, - 'password': CONSOLE_PASSWORD - }, - 'dashboard': { - 'username': CONSOLE_USERNAME, - 'password': CONSOLE_PASSWORD - }, - 'image': { - 'username': CONSOLE_USERNAME, - 'password': CONSOLE_PASSWORD - }, - 'metering': { - 'username': CONSOLE_USERNAME, - 'password': CONSOLE_PASSWORD - }, - 'network': { - 'username': CONSOLE_USERNAME, - 'password': CONSOLE_PASSWORD - }, - 'object-store': { - 'username': CONSOLE_USERNAME, - 'password': CONSOLE_PASSWORD - }, - 'volume': { - 'username': CONSOLE_USERNAME, - 'password': CONSOLE_PASSWORD - } - } - }, - 'network_mapping': { - 'management': MANAGEMENT_NIC, - 'tenant': TENANT_NIC, - 'storage': STORAGE_NIC, - 'public': PUBLIC_NIC - } -} - -status, response = client.update_cluster_config( - cluster_id, - cluster_os_config, - cluster_package_config -) - -print '=======================================' -print 'cluster %s update status %s: %s' % ( - cluster_id, status, response) -if status >= 400: - sys.exit(1) - -status, response = client.update_cluster_host( - cluster_id, host_id, roles=PRESET_VALUES['ROLES_LIST']) -print '=================================================' -print 'update cluster host %s/%s status %s: %s' % ( - cluster_id, host_id, status, response) -if status >= 400: - sys.exit(1) - -# Review and deploy -status, response = client.review_cluster( - cluster_id, review={'hosts': [host_id]}) -print '=======================================' -print 'reviewing cluster status %s: %s' % (status, response) -if status >= 400: - sys.exit(1) - -status, response = client.deploy_cluster( - cluster_id, deploy={'hosts': [host_id]}) -print '=======================================' -print 'deploy cluster status %s: %s' % (status, response) -if status >= 400: - sys.exit(1) diff --git a/compass-tasks/apiclient/restful.py b/compass-tasks/apiclient/restful.py deleted file mode 100644 index bb82922..0000000 --- a/compass-tasks/apiclient/restful.py +++ /dev/null @@ -1,1102 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Compass api client library. -""" - -import json -import logging -import requests - - -class Client(object): - """compass restful api wrapper""" - - def __init__(self, url, headers=None, proxies=None, stream=None): - logging.info('create api client %s', url) - self.url_ = url - self.session_ = requests.Session() - - if headers: - self.session_.headers.update(headers) - self.session_.headers.update({ - 'Accept': 'application/json' - }) - - if proxies is not None: - self.session_.proxies = proxies - - if stream is not None: - self.session_.stream = stream - - def __del__(self): - self.session_.close() - - @classmethod - def _get_response(cls, resp): - response_object = {} - try: - response_object = resp.json() - except Exception as error: - logging.error('failed to load object from %s: %s', - resp.url, resp.content) - logging.exception(error) - response_object['status'] = 'Json Parsing Failed' - response_object['message'] = resp.content - - return resp.status_code, response_object - - def _get(self, req_url, data=None): - url = '%s%s' % (self.url_, req_url) - logging.debug('get %s with data %s', url, data) - if data: - resp = self.session_.get(url, params=data) - else: - resp = self.session_.get(url) - - return self._get_response(resp) - - def _post(self, req_url, data=None): - url = '%s%s' % (self.url_, req_url) - logging.debug('post %s with data %s', url, data) - if data: - resp = self.session_.post(url, json.dumps(data)) - else: - resp = self.session_.post(url) - - return self._get_response(resp) - - def _put(self, req_url, data=None): - """encapsulate put method.""" - url = '%s%s' % (self.url_, req_url) - logging.debug('put %s with data %s', url, data) - if data: - resp = self.session_.put(url, json.dumps(data)) - else: - resp = self.session_.put(url) - - return self._get_response(resp) - - def _patch(self, req_url, data=None): - url = '%s%s' % (self.url_, req_url) - logging.debug('patch %s with data %s', url, data) - if data: - resp = self.session_.patch(url, json.dumps(data)) - else: - resp = self.session_.patch(url) - - return self._get_response(resp) - - def _delete(self, req_url): - url = '%s%s' % (self.url_, req_url) - logging.debug('delete %s', url) - return self._get_response(self.session_.delete(url)) - - def login(self, email, password): - credential = {} - credential['email'] = email - credential['password'] = password - return self._post('/users/login', data=credential) - - def get_token(self, email, password): - credential = {} - credential['email'] = email - credential['password'] = password - status, resp = self._post('/users/token', data=credential) - if status < 400: - self.session_.headers.update({'X-Auth-Token': resp['token']}) - return status, resp - - def get_users(self): - users = self._get('/users') - return users - - def list_switches( - self, - switch_ips=None, - switch_ip_networks=None): - """list switches.""" - params = {} - if switch_ips: - params['switchIp'] = switch_ips - - if switch_ip_networks: - params['switchIpNetwork'] = switch_ip_networks - - switchlist = self._get('/switches', data=params) - return switchlist - - def get_switch(self, switch_id): - return self._get('/switches/%s' % switch_id) - - def add_switch( - self, - switch_ip, - version=None, - community=None, - raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - data['ip'] = switch_ip - data['credentials'] = {} - if version: - data['credentials']['version'] = version - - if community: - data['credentials']['community'] = community - - return self._post('/switches', data=data) - - def update_switch(self, switch_id, state='initialized', - version='2c', community='public', raw_data={}): - data = {} - if raw_data: - data = raw_data - - else: - data['credentials'] = {} - if version: - data['credentials']['version'] = version - - if community: - data['credentials']['community'] = community - - if state: - data['state'] = state - - return self._put('/switches/%s' % switch_id, data=data) - - def delete_switch(self, switch_id): - return self._delete('/switches/%s' % switch_id) - - def list_switch_machines(self, switch_id, port=None, vlans=None, - tag=None, location=None): - data = {} - if port: - data['port'] = port - - if vlans: - data['vlans'] = vlans - - if tag: - data['tag'] = tag - - if location: - data['location'] = location - - return self._get('/switches/%s/machines' % switch_id, data=data) - - def get_switch_machine(self, switch_id, machine_id): - return self._get('/switches/%s/machines/%s' % (switch_id, machine_id)) - - def list_switch_machines_hosts(self, switch_id, port=None, vlans=None, - mac=None, tag=None, location=None, - os_name=None, os_id=None): - - data = {} - if port: - data['port'] = port - - if vlans: - data['vlans'] = vlans - - if mac: - data['mac'] = mac - - if tag: - data['tag'] = tag - - if location: - data['location'] = location - - if os_name: - data['os_name'] = os_name - - if os_id: - data['os_id'] = os_id - - return self._get('/switches/%s/machines-hosts' % switch_id, data=data) - - def add_switch_machine(self, switch_id, mac=None, port=None, - vlans=None, ipmi_credentials=None, - tag=None, location=None, raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if mac: - data['mac'] = mac - - if port: - data['port'] = port - - if vlans: - data['vlans'] = vlans - - if ipmi_credentials: - data['ipmi_credentials'] = ipmi_credentials - - if tag: - data['tag'] = tag - - if location: - data['location'] = location - - return self._post('/switches/%s/machines' % switch_id, data=data) - - def update_switch_machine(self, switch_id, machine_id, port=None, - vlans=None, ipmi_credentials=None, tag=None, - location=None, raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if port: - data['port'] = port - - if vlans: - data['vlans'] = vlans - - if ipmi_credentials: - data['ipmi_credentials'] = ipmi_credentials - - if tag: - data['tag'] = tag - - if location: - data['location'] = location - - return self._put('/switches/%s/machines/%s' % - (switch_id, machine_id), data=data) - - def delete_switch_machine(self, switch_id, machine_id): - return self._delete('/switches/%s/machines/%s' % - (switch_id, machine_id)) - - # test these - def poll_switch(self, switch_id): - data = {} - data['find_machines'] = None - return self._post('/switches/%s/action' % switch_id, data=data) - - def add_group_switch_machines(self, switch_id, group_machine_ids): - data = {} - data['add_machines'] = group_machine_ids - return self._post('/switches/%s/action' % switch_id, data=data) - - def remove_group_switch_machines(self, switch_id, group_machine_ids): - data = {} - data['remove_machines'] = group_machine_ids - return self._post('/switches/%s/action' % switch_id, data=data) - - def update_group_switch_machines(self, switch_id, group_machines): - data = {} - data['set_machines'] = group_machines - return self._post('/switches/%s/action' % switch_id, data=data) - # end - - def list_switchmachines(self, switch_ip_int=None, port=None, vlans=None, - mac=None, tag=None, location=None): - data = {} - if switch_ip_int: - data['switch_ip_int'] = switch_ip_int - - if port: - data['port'] = port - - if vlans: - data['vlans'] = vlans - - if mac: - data['mac'] = mac - - if tag: - data['tag'] = tag - - if location: - data['location'] = location - - return self._get('/switch-machines', data=data) - - def list_switchmachines_hosts(self, switch_ip_int=None, port=None, - vlans=None, mac=None, tag=None, - location=None, os_name=None, os_id=None): - - data = {} - if switch_ip_int: - data['switch_ip_int'] = switch_ip_int - - if port: - data['port'] = port - - if vlans: - data['vlans'] = vlans - - if mac: - data['mac'] = mac - - if tag: - data['tag'] = tag - - if location: - data['location'] = location - - if os_name: - data['os_name'] = os_name - - if os_id: - data['os_id'] = os_id - - return self._get('/switches-machines-hosts', data=data) - - def show_switchmachine(self, switchmachine_id): - return self._get('/switch-machines/%s' % switchmachine_id) - - def update_switchmachine(self, switchmachine_id, - port=None, vlans=None, raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if port: - data['port'] = port - - if vlans: - data['vlans'] = vlans - - return self._put('/switch-machines/%s' % switchmachine_id, data=data) - - def patch_switchmachine(self, switchmachine_id, - vlans=None, raw_data=None): - data = {} - if raw_data: - data = raw_data - elif vlans: - data['vlans'] = vlans - - return self._patch('/switch-machines/%s' % switchmachine_id, data=data) - - def delete_switchmachine(self, switchmachine_id): - return self._delete('/switch-machines/%s' % switchmachine_id) - - def list_machines(self, mac=None, tag=None, location=None): - data = {} - if mac: - data['mac'] = mac - - if tag: - data['tag'] = tag - - if location: - data['location'] = location - - return self._get('/machines', data=data) - - def get_machine(self, machine_id): - data = {} - if id: - data['id'] = id - - return self._get('/machines/%s' % machine_id, data=data) - - def update_machine(self, machine_id, ipmi_credentials=None, tag=None, - location=None, raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if ipmi_credentials: - data['ipmi_credentials'] = ipmi_credentials - - if tag: - data['tag'] = tag - - if location: - data['location'] = location - - return self._put('/machines/%s' % machine_id, data=data) - - def patch_machine(self, machine_id, ipmi_credentials=None, - tag=None, location=None, - raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if ipmi_credentials: - data['ipmi_credentials'] = ipmi_credentials - - if tag: - data['tag'] = tag - - if location: - data['location'] = location - - return self._patch('/machines/%s' % machine_id, data=data) - - def delete_machine(self, machine_id): - return self._delete('machines/%s' % machine_id) - - def list_subnets(self, subnet=None, name=None): - data = {} - if subnet: - data['subnet'] = subnet - - if name: - data['name'] = name - - return self._get('/subnets', data=data) - - def get_subnet(self, subnet_id): - return self._get('/subnets/%s' % subnet_id) - - def add_subnet(self, subnet, name=None, raw_data=None): - data = {} - data['subnet'] = subnet - if raw_data: - data.update(raw_data) - else: - if name: - data['name'] = name - - return self._post('/subnets', data=data) - - def update_subnet(self, subnet_id, subnet=None, - name=None, raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if subnet: - data['subnet'] = subnet - - if name: - data['name'] = name - return self._put('/subnets/%s' % subnet_id, data=data) - - def delete_subnet(self, subnet_id): - return self._delete('/subnets/%s' % subnet_id) - - def list_adapters(self, name=None): - data = {} - if name: - data['name'] = name - - return self._get('/adapters', data=data) - - def get_adapter(self, adapter_id): - return self._get('/adapters/%s' % adapter_id) - - def get_adapter_roles(self, adapter_id): - return self._get('/adapters/%s/roles' % adapter_id) - - def get_adapter_metadata(self, adapter_id): - return self._get('/adapters/%s/metadata' % adapter_id) - - def get_os_metadata(self, os_id): - return self._get('/oses/%s/metadata' % os_id) - - def list_clusters(self, name=None, os_name=None, - owner=None, - adapter_id=None): - data = {} - if name: - data['name'] = name - - if os_name: - data['os_name'] = os_name - - if owner: - data['owner'] = owner - - if adapter_id: - data['adapter_id'] = adapter_id - - return self._get('/clusters', data=data) - - def get_cluster(self, cluster_id): - return self._get('/clusters/%s' % cluster_id) - - def add_cluster(self, name, adapter_id, os_id, - flavor_id=None, raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if flavor_id: - data['flavor_id'] = flavor_id - data['name'] = name - data['adapter_id'] = adapter_id - data['os_id'] = os_id - - return self._post('/clusters', data=data) - - def update_cluster(self, cluster_id, name=None, - reinstall_distributed_system=None, - raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if name: - data['name'] = name - - if reinstall_distributed_system: - data['reinstall_distributed_system'] = ( - reinstall_distributed_system - ) - return self._put('/clusters/%s' % cluster_id, data=data) - - def delete_cluster(self, cluster_id): - return self._delete('/clusters/%s' % cluster_id) - - def get_cluster_config(self, cluster_id): - return self._get('/clusters/%s/config' % cluster_id) - - def get_cluster_metadata(self, cluster_id): - return self._get('/clusters/%s/metadata' % cluster_id) - - def update_cluster_config(self, cluster_id, os_config=None, - package_config=None, config_step=None, - raw_data=None): - data = {} - if raw_data: - data = raw_data - - if os_config: - data['os_config'] = os_config - - if package_config: - data['package_config'] = package_config - - if config_step: - data['config_step'] = config_step - - return self._put('/clusters/%s/config' % cluster_id, data=data) - - def patch_cluster_config(self, cluster_id, os_config=None, - package_config=None, config_step=None, - raw_data=None): - data = {} - if raw_data: - data = raw_data - - if os_config: - data['os_config'] = os_config - - if package_config: - data['package_config'] = package_config - - if config_step: - data['config_step'] = config_step - - return self._patch('/clusters/%s/config' % cluster_id, data=data) - - def delete_cluster_config(self, cluster_id): - return self._delete('/clusters/%s/config' % cluster_id) - - # test these - def add_hosts_to_cluster(self, cluster_id, hosts): - data = {} - data['add_hosts'] = hosts - return self._post('/clusters/%s/action' % cluster_id, data=data) - - def set_hosts_in_cluster(self, cluster_id, hosts): - data = {} - data['set_hosts'] = hosts - return self._post('/clusters/%s/action' % cluster_id, data=data) - - def remove_hosts_from_cluster(self, cluster_id, hosts): - data = {} - data['remove_hosts'] = hosts - return self._post('/clusters/%s/action' % cluster_id, data=data) - - def review_cluster(self, cluster_id, review={}): - data = {} - data['review'] = review - return self._post('/clusters/%s/action' % cluster_id, data=data) - - def deploy_cluster(self, cluster_id, deploy={}): - data = {} - data['deploy'] = deploy - return self._post('/clusters/%s/action' % cluster_id, data=data) - - def redeploy_cluster(self, cluster_id, deploy={}): - data = {} - data['redeploy'] = deploy - return self._post('/clusters/%s/action' % cluster_id, data=data) - - def get_cluster_state(self, cluster_id): - return self._get('/clusters/%s/state' % cluster_id) - - def list_cluster_hosts(self, cluster_id): - return self._get('/clusters/%s/hosts' % cluster_id) - - def list_clusterhosts(self): - return self._get('/clusterhosts') - - def get_cluster_host(self, cluster_id, host_id): - return self._get('/clusters/%s/hosts/%s' % (cluster_id, host_id)) - - def get_clusterhost(self, clusterhost_id): - return self._get('/clusterhosts/%s' % clusterhost_id) - - def add_cluster_host(self, cluster_id, machine_id=None, name=None, - reinstall_os=None, raw_data=None): - data = {} - data['machine_id'] = machine_id - if raw_data: - data.update(raw_data) - else: - if name: - data['name'] = name - - if reinstall_os: - data['reinstall_os'] = reinstall_os - - return self._post('/clusters/%s/hosts' % cluster_id, data=data) - - def delete_cluster_host(self, cluster_id, host_id): - return self._delete('/clusters/%s/hosts/%s' % - (cluster_id, host_id)) - - def delete_clusterhost(self, clusterhost_id): - return self._delete('/clusterhosts/%s' % clusterhost_id) - - def get_cluster_host_config(self, cluster_id, host_id): - return self._get('/clusters/%s/hosts/%s/config' % - (cluster_id, host_id)) - - def get_clusterhost_config(self, clusterhost_id): - return self._get('/clusterhosts/%s/config' % clusterhost_id) - - def update_cluster_host_config(self, cluster_id, host_id, - os_config=None, - package_config=None, - raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if os_config: - data['os_config'] = os_config - - if package_config: - data['package_config'] = package_config - - return self._put('/clusters/%s/hosts/%s/config' % - (cluster_id, host_id), data=data) - - def update_clusterhost_config(self, clusterhost_id, os_config=None, - package_config=None, raw_data=None): - data = {} - if raw_data: - data = raw_data - - else: - if os_config: - data['os_config'] = os_config - - if package_config: - data['package_config'] = package_config - - return self._put('/clusterhosts/%s/config' % clusterhost_id, - data=data) - - def patch_cluster_host_config(self, cluster_id, host_id, - os_config=None, - package_config=None, - raw_data=None): - data = {} - if raw_data: - data = raw_data - - else: - if os_config: - data['os_config'] = os_config - - if package_config: - data['package_config'] = package_config - - return self._patch('/clusters/%s/hosts/%s/config' % - (cluster_id, host_id), data=data) - - def patch_clusterhost_config(self, clusterhost_id, os_config=None, - package_config=None, raw_data=None): - data = {} - if raw_data: - data = raw_data - - else: - if os_config: - data['os_config'] = os_config - - if package_config: - data['package_config'] = package_config - - return self._patch('/clusterhosts/%s' % clusterhost_id, data=data) - - def delete_cluster_host_config(self, cluster_id, host_id): - return self._delete('/clusters/%s/hosts/%s/config' % - (cluster_id, host_id)) - - def delete_clusterhost_config(self, clusterhost_id): - return self._delete('/clusterhosts/%s/config' % clusterhost_id) - - def get_cluster_host_state(self, cluster_id, host_id): - return self._get('/clusters/%s/hosts/%s/state' % - (cluster_id, host_id)) - - def update_cluster_host(self, cluster_id, host_id, - roles=None, raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if roles: - data['roles'] = roles - - return self._put('/clusters/%s/hosts/%s' % - (cluster_id, host_id), data=data) - - def update_clusterhost(self, clusterhost_id, - roles=None, raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if roles: - data['roles'] = roles - - return self._put('/clusterhosts/%s' % clusterhost_id, data=data) - - def patch_cluster_host(self, cluster_id, host_id, - roles=None, raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if roles: - data['roles'] = roles - - return self._patch('/clusters/%s/hosts/%s' % - (cluster_id, host_id), data=data) - - def patch_clusterhost(self, clusterhost_id, - roles=None, raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if roles: - data['roles'] = roles - - return self._patch('/clusterhosts/%s' % clusterhost_id, data=data) - - def get_clusterhost_state(self, clusterhost_id): - return self._get('/clusterhosts/%s/state' % clusterhost_id) - - def update_cluster_host_state(self, cluster_id, host_id, state=None, - percentage=None, message=None, - raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if state: - data['state'] = state - - if percentage: - data['percentage'] = percentage - - if message: - data['message'] = message - - return self._put('/clusters/%s/hosts/%s/state' % (cluster_id, host_id), - data=data) - - def update_clusterhost_state(self, clusterhost_id, state=None, - percentage=None, message=None, - raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if state: - data['state'] = state - - if percentage: - data['percentage'] = percentage - - if message: - data['message'] = message - - return self._put('/clusterhosts/%s/state' % clusterhost_id, data=data) - - def list_hosts(self, name=None, os_name=None, owner=None, mac=None): - data = {} - if name: - data['name'] = name - - if os_name: - data['os_name'] = os_name - - if owner: - data['owner'] = owner - - if mac: - data['mac'] = mac - - return self._get('/hosts', data=data) - - def get_host(self, host_id): - return self._get('/hosts/%s' % host_id) - - def list_machines_or_hosts(self, mac=None, tag=None, - location=None, os_name=None, - os_id=None): - data = {} - if mac: - data['mac'] = mac - - if tag: - data['tag'] = tag - - if location: - data['location'] = location - - if os_name: - data['os_name'] = os_name - - if os_id: - data['os_id'] = os_id - - return self._get('/machines-hosts', data=data) - - def get_machine_or_host(self, host_id): - return self._get('/machines-hosts/%s' % host_id) - - def update_host(self, host_id, name=None, - reinstall_os=None, raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if name: - data['name'] = name - - if reinstall_os: - data['reinstall_os'] = reinstall_os - - return self._put('/hosts/%s' % host_id, data=data) - - def delete_host(self, host_id): - return self._delete('/hosts/%s' % host_id) - - def get_host_clusters(self, host_id): - return self._get('/hosts/%s/clusters' % host_id) - - def get_host_config(self, host_id): - return self._get('/hosts/%s/config' % host_id) - - def update_host_config(self, host_id, os_config, raw_data=None): - data = {} - data['os_config'] = os_config - if raw_data: - data.update(raw_data) - - return self._put('/hosts/%s/config' % host_id, data=data) - - def patch_host_config(self, host_id, os_config, raw_data=None): - data = {} - data['os_config'] = os_config - if raw_data: - data.update(raw_data) - - return self._patch('/hosts/%s/config' % host_id, data=data) - - def delete_host_config(self, host_id): - return self._delete('/hosts/%s/config' % host_id) - - def list_host_networks(self, host_id, interface=None, ip=None, - subnet=None, is_mgmt=None, is_promiscuous=None): - data = {} - if interface: - data['interface'] = interface - - if ip: - data['ip'] = ip - - if subnet: - data['subnet'] = subnet - - if is_mgmt: - data['is_mgmt'] = is_mgmt - - if is_promiscuous: - data['is_promiscuous'] = is_promiscuous - - return self._get('/hosts/%s/networks' % host_id, data=data) - - def list_all_host_networks(self, interface=None, ip=None, subnet=None, - is_mgmt=None, is_promiscuous=None): - data = {} - if interface: - data['interface'] = interface - - if ip: - data['ip'] = ip - - if subnet: - data['subnet'] = subnet - - if is_mgmt: - data['is_mgmt'] = is_mgmt - - if is_promiscuous: - data['is_promiscuous'] = is_promiscuous - - return self._get('/host-networks', data=data) - - def get_host_network(self, host_id, host_network_id): - return self._get('/hosts/%s/networks/%s' % - (host_id, host_network_id)) - - def get_network_for_all_hosts(self, host_network_id): - return self._get('/host-networks/%s' % host_network_id) - - def add_host_network(self, host_id, interface, ip, subnet_id, - is_mgmt=None, is_promiscuous=None, - raw_data=None): - data = {} - data['interface'] = interface - data['ip'] = ip - data['subnet_id'] = subnet_id - if raw_data: - data.update(raw_data) - else: - if is_mgmt: - data['is_mgmt'] = is_mgmt - - if is_promiscuous: - data['is_promiscuous'] = is_promiscuous - - return self._post('/hosts/%s/networks' % host_id, data=data) - - def update_host_network(self, host_id, host_network_id, - ip=None, subnet_id=None, subnet=None, - is_mgmt=None, is_promiscuous=None, - raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if ip: - data['ip'] = ip - - if subnet_id: - data['subnet_id'] = subnet_id - - if subnet: - data['subnet'] = subnet - - if is_mgmt: - data['is_mgmt'] = is_mgmt - - if is_promiscuous: - data['is_promiscuous'] = is_promiscuous - - return self._put('/hosts/%s/networks/%s' % - (host_id, host_network_id), data=data) - - def update_hostnetwork(self, host_network_id, ip=None, - subnet_id=None, subnet=None, - is_mgmt=None, is_promiscuous=None, - raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if ip: - data['ip'] = ip - - if subnet_id: - data['subnet_id'] = subnet_id - - if subnet: - data['subnet'] = subnet - - if is_mgmt: - data['is_mgmt'] = is_mgmt - - if is_promiscuous: - data['is_promiscuous'] = is_promiscuous - - return self._put('/host-networks/%s' % host_network_id, - data=data) - - def delete_host_network(self, host_id, host_network_id): - return self._delete('/hosts/%s/networks/%s', - (host_id, host_network_id)) - - def delete_hostnetwork(self, host_network_id): - return self._delete('/host-networks/%s' % host_network_id) - - def get_host_state(self, host_id): - return self._get('/hosts/%s/state' % host_id) - - def update_host_state(self, host_id, state=None, - percentage=None, message=None, - raw_data=None): - data = {} - if raw_data: - data = raw_data - else: - if state: - data['state'] = state - - if percentage: - data['percentage'] = percentage - - if message: - data['message'] = message - - return self._put('/hosts/%s/state' % host_id, date=data) - - def poweron_host(self, host_id): - data = {} - data['poweron'] = True - - return self._post('/hosts/%s/action' % host_id, data=data) - - def poweroff_host(self, host_id): - data = {} - data['poweroff'] = True - - return self._post('/hosts/%s/action' % host_id, data=data) - - def reset_host(self, host_id): - data = {} - data['reset'] = True - - return self._post('/hosts/%s/action' % host_id, data=data) - - def clusterhost_ready(self, clusterhost_name): - data = {} - data['ready'] = True - - return self._post('/clusterhosts/%s/state_internal' % - clusterhost_name, data=data) diff --git a/compass-tasks/apiclient/v1/__init__.py b/compass-tasks/apiclient/v1/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compass-tasks/apiclient/v1/example.py b/compass-tasks/apiclient/v1/example.py deleted file mode 100755 index 6f7a7f7..0000000 --- a/compass-tasks/apiclient/v1/example.py +++ /dev/null @@ -1,305 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Example code to deploy a cluster by compass client api.""" -import os -import re -import requests -import sys -import time - -from compass.apiclient.restful import Client - - -COMPASS_SERVER_URL = 'http://127.0.0.1/api' -SWITCH_IP = '10.145.81.220' -SWITCH_SNMP_VERSION = 'v2c' -SWITCH_SNMP_COMMUNITY = 'public' -# MACHINES_TO_ADD = ['00:11:20:30:40:01'] -CLUSTER_NAME = 'cluster2' -HOST_NAME_PREFIX = 'host' -SERVER_USERNAME = 'root' -SERVER_PASSWORD = 'root' -SERVICE_USERNAME = 'service' -SERVICE_PASSWORD = 'service' -CONSOLE_USERNAME = 'console' -CONSOLE_PASSWORD = 'console' -HA_VIP = '' -# NAMESERVERS = '192.168.10.6' -SEARCH_PATH = 'ods.com' -# GATEWAY = '192.168.10.6' -# PROXY = 'http://192.168.10.6:3128' -# NTP_SERVER = '192.168.10.6' -MANAGEMENT_IP_START = '192.168.10.130' -MANAGEMENT_IP_END = '192.168.10.254' -MANAGEMENT_IP_GATEWAY = '192.168.10.1' -MANAGEMENT_NETMASK = '255.255.255.0' -MANAGEMENT_NIC = 'eth0' -MANAGEMENT_PROMISC = 0 -TENANT_IP_START = '192.168.10.100' -TENANT_IP_END = '192.168.10.255' -TENANT_IP_GATEWAY = '192.168.10.1' -TENANT_NETMASK = '255.255.255.0' -TENANT_NIC = 'eth0' -TENANT_PROMISC = 0 -PUBLIC_IP_START = '12.234.32.100' -PUBLIC_IP_END = '12.234.32.255' -PUBLIC_IP_GATEWAY = '12.234.32.1' -PUBLIC_NETMASK = '255.255.255.0' -PUBLIC_NIC = 'eth1' -PUBLIC_PROMISC = 1 -STORAGE_IP_START = '172.16.100.100' -STORAGE_IP_END = '172.16.100.255' -STORAGE_NETMASK = '255.255.255.0' -STORAGE_IP_GATEWAY = '172.16.100.1' -STORAGE_NIC = 'eth0' -STORAGE_PROMISC = 0 -HOME_PERCENTAGE = 5 -TMP_PERCENTAGE = 5 -VAR_PERCENTAGE = 10 -# ROLES_LIST = [['os-dashboard']] - -PRESET_VALUES = { - 'NAMESERVERS': '192.168.10.1', - 'NTP_SERVER': '192.168.10.1', - 'GATEWAY': '192.168.10.1', - 'PROXY': 'http://192.168.10.1:3128', - 'ROLES_LIST': 'os-dashboard', - 'MACHINES_TO_ADD': '00:11:20:30:40:01', - 'BUILD_TIMEOUT': 60 -} -for v in PRESET_VALUES: - if v in os.environ.keys(): - PRESET_VALUES[v] = os.environ.get(v) - print (v + PRESET_VALUES[v] + " is set by env variables") - else: - print (PRESET_VALUES[v]) - -# get apiclient object. -client = Client(COMPASS_SERVER_URL) - - -# get all switches. -status, resp = client.get_switches() -print 'get all switches status: %s resp: %s' % (status, resp) - -# add a switch. -status, resp = client.add_switch( - SWITCH_IP, version=SWITCH_SNMP_VERSION, - community=SWITCH_SNMP_COMMUNITY) - -print 'add a switch status: %s resp: %s' % (status, resp) - -if status < 400: - switch = resp['switch'] -else: - status, resp = client.get_switches() - print 'get all switches status: %s resp: %s' % (status, resp) - switch = None - for switch in resp['switches']: - if switch['ip'] == SWITCH_IP: - break - -switch_id = switch['id'] -switch_ip = switch['ip'] - - -# if the switch is not in under_monitoring, wait for the poll switch task -# update the swich information and change the switch state. -while switch['state'] != 'under_monitoring': - print 'waiting for the switch into under_monitoring' - status, resp = client.get_switch(switch_id) - print 'get switch %s status: %s, resp: %s' % (switch_id, status, resp) - switch = resp['switch'] - time.sleep(10) - - -# get machines connected to the switch. -status, resp = client.get_machines(switch_id=switch_id) -print 'get all machines under switch %s status: %s, resp: %s' % ( - switch_id, status, resp) -machines = {} -MACHINES_TO_ADD = PRESET_VALUES['MACHINES_TO_ADD'].split() -for machine in resp['machines']: - mac = machine['mac'] - if mac in MACHINES_TO_ADD: - machines[machine['id']] = mac - -print 'machine to add: %s' % machines - -if set(machines.values()) != set(MACHINES_TO_ADD): - print 'only found macs %s while expected are %s' % ( - machines.values(), MACHINES_TO_ADD) - sys.exit(1) - - -# get adapters. -status, resp = client.get_adapters() -print 'get all adapters status: %s, resp: %s' % (status, resp) -adapter_ids = [] -for adapter in resp['adapters']: - adapter_ids.append(adapter['id']) - -adapter_id = adapter_ids[0] -print 'adpater for deploying a cluster: %s' % adapter_id - - -# add a cluster. -status, resp = client.add_cluster( - cluster_name=CLUSTER_NAME, adapter_id=adapter_id) -print 'add cluster %s status: %s, resp: %s' % (CLUSTER_NAME, status, resp) -cluster = resp['cluster'] -cluster_id = cluster['id'] - -# add hosts to the cluster. -status, resp = client.add_hosts( - cluster_id=cluster_id, - machine_ids=machines.keys()) -print 'add hosts to cluster %s status: %s, resp: %s' % ( - cluster_id, status, resp) -host_ids = [] -for host in resp['cluster_hosts']: - host_ids.append(host['id']) - -print 'added hosts: %s' % host_ids - - -# set cluster security -status, resp = client.set_security( - cluster_id, server_username=SERVER_USERNAME, - server_password=SERVER_PASSWORD, - service_username=SERVICE_USERNAME, - service_password=SERVICE_PASSWORD, - console_username=CONSOLE_USERNAME, - console_password=CONSOLE_PASSWORD) -print 'set security config to cluster %s status: %s, resp: %s' % ( - cluster_id, status, resp) - - -# set cluster networking -status, resp = client.set_networking( - cluster_id, - nameservers=PRESET_VALUES["NAMESERVERS"], - search_path=SEARCH_PATH, - gateway=PRESET_VALUES["GATEWAY"], - proxy=PRESET_VALUES["PROXY"], - ntp_server=PRESET_VALUES["NTP_SERVER"], - ha_vip=HA_VIP, - management_ip_start=MANAGEMENT_IP_START, - management_ip_end=MANAGEMENT_IP_END, - management_netmask=MANAGEMENT_NETMASK, - management_nic=MANAGEMENT_NIC, - management_gateway=MANAGEMENT_IP_GATEWAY, - management_promisc=MANAGEMENT_PROMISC, - tenant_ip_start=TENANT_IP_START, - tenant_ip_end=TENANT_IP_END, - tenant_netmask=TENANT_NETMASK, - tenant_nic=TENANT_NIC, - tenant_gateway=TENANT_IP_GATEWAY, - tenant_promisc=TENANT_PROMISC, - public_ip_start=PUBLIC_IP_START, - public_ip_end=PUBLIC_IP_END, - public_netmask=PUBLIC_NETMASK, - public_nic=PUBLIC_NIC, - public_gateway=PUBLIC_IP_GATEWAY, - public_promisc=PUBLIC_PROMISC, - storage_ip_start=STORAGE_IP_START, - storage_ip_end=STORAGE_IP_END, - storage_netmask=STORAGE_NETMASK, - storage_nic=STORAGE_NIC, - storage_gateway=STORAGE_IP_GATEWAY, - storage_promisc=STORAGE_PROMISC) -print 'set networking config to cluster %s status: %s, resp: %s' % ( - cluster_id, status, resp) - - -# set partiton of each host in cluster -status, resp = client.set_partition( - cluster_id, - home_percentage=HOME_PERCENTAGE, - tmp_percentage=TMP_PERCENTAGE, - var_percentage=VAR_PERCENTAGE) -print 'set partition config to cluster %s status: %s, resp: %s' % ( - cluster_id, status, resp) - - -# set each host config in cluster. -ROLES_LIST = [PRESET_VALUES['ROLES_LIST'].split()] -for host_id in host_ids: - if ROLES_LIST: - roles = ROLES_LIST.pop(0) - else: - roles = [] - status, resp = client.update_host_config( - host_id, hostname='%s%s' % (HOST_NAME_PREFIX, host_id), - roles=roles) - print 'set roles to host %s status: %s, resp: %s' % ( - host_id, status, resp) - - -# deploy cluster. -status, resp = client.deploy_hosts(cluster_id) -print 'deploy cluster %s status: %s, resp: %s' % (cluster_id, status, resp) - - -# get intalling progress. -BUILD_TIMEOUT = float(PRESET_VALUES['BUILD_TIMEOUT']) -timeout = time.time() + BUILD_TIMEOUT * 60 -while True: - status, resp = client.get_cluster_installing_progress(cluster_id) - print 'get cluster %s installing progress status: %s, resp: %s' % ( - cluster_id, status, resp) - progress = resp['progress'] - if ( - progress['state'] not in ['UNINITIALIZED', 'INSTALLING'] or - progress['percentage'] >= 1.0 - ): - break - if ( - time.time() > timeout - ): - raise Exception("Timeout! The system is not ready in time.") - - for host_id in host_ids: - status, resp = client.get_host_installing_progress(host_id) - print 'get host %s installing progress status: %s, resp: %s' % ( - host_id, status, resp) - - time.sleep(60) - - -status, resp = client.get_dashboard_links(cluster_id) -print 'get cluster %s dashboardlinks status: %s, resp: %s' % ( - cluster_id, status, resp) -dashboardlinks = resp['dashboardlinks'] -if not dashboardlinks.keys(): - raise Exception("Dashboard link is not found!") -for x in dashboardlinks.keys(): - if x in ("os-dashboard", "os-controller"): - dashboardurl = dashboardlinks.get(x) - if dashboardurl is None: - raise Exception("No dashboard link is found") - r = requests.get(dashboardurl, verify=False) - r.raise_for_status() - match = re.search( - r'(?m)(http://\d+\.\d+\.\d+\.\d+:5000/v2\.0)', r.text) - if match: - print 'dashboard login page can be downloaded' - break - print ( - 'dashboard login page failed to be downloaded\n' - 'the context is:\n%s\n') % r.text - raise Exception("os-dashboard is not properly installed!") diff --git a/compass-tasks/apiclient/v1/restful.py b/compass-tasks/apiclient/v1/restful.py deleted file mode 100644 index 3fb235c..0000000 --- a/compass-tasks/apiclient/v1/restful.py +++ /dev/null @@ -1,655 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Compass api client library. - - .. moduleauthor:: Xiaodong Wang -""" -import json -import logging -import requests - - -class Client(object): - """wrapper for compass restful api. - - .. note:: - Every api client method returns (status as int, resp as dict). - If the api succeeds, the status is 2xx, the resp includes - {'status': 'OK'} and other keys depend on method. - If the api fails, the status is 4xx, the resp includes { - 'status': '...', 'message': '...'} - """ - - def __init__(self, url, headers=None, proxies=None, stream=None): - """Restful api client initialization. - - :param url: url to the compass web service. - :type url: str. - :param headers: http header sent in each restful request. - :type headers: dict of header name (str) to heade value (str). - :param proxies: the proxy address for each protocol. - :type proxies: dict of protocol (str) to proxy url (str). - :param stream: wether the restful response should be streamed. - :type stream: bool. - """ - self.url_ = url - self.session_ = requests.Session() - if headers: - self.session_.headers = headers - - if proxies is not None: - self.session_.proxies = proxies - - if stream is not None: - self.session_.stream = stream - - def __del__(self): - self.session_.close() - - @classmethod - def _get_response(cls, resp): - """decapsulate the resp to status code and python formatted data.""" - resp_obj = {} - try: - resp_obj = resp.json() - except Exception as error: - logging.error('failed to load object from %s: %s', - resp.url, resp.content) - logging.exception(error) - resp_obj['status'] = 'Json Parsing Failure' - resp_obj['message'] = resp.content - - return resp.status_code, resp_obj - - def _get(self, relative_url, params=None): - """encapsulate get method.""" - url = '%s%s' % (self.url_, relative_url) - if params: - resp = self.session_.get(url, params=params) - else: - resp = self.session_.get(url) - - return self._get_response(resp) - - def _post(self, relative_url, data=None): - """encapsulate post method.""" - url = '%s%s' % (self.url_, relative_url) - if data: - resp = self.session_.post(url, json.dumps(data)) - else: - resp = self.session_.post(url) - - return self._get_response(resp) - - def _put(self, relative_url, data=None): - """encapsulate put method.""" - url = '%s%s' % (self.url_, relative_url) - if data: - resp = self.session_.put(url, json.dumps(data)) - else: - resp = self.session_.put(url) - - return self._get_response(resp) - - def _delete(self, relative_url): - """encapsulate delete method.""" - url = '%s%s' % (self.url_, relative_url) - return self._get_response(self.session_.delete(url)) - - def get_switches(self, switch_ips=None, switch_networks=None, limit=None): - """List details for switches. - - .. note:: - The switches can be filtered by switch_ips, siwtch_networks and - limit. These params can be None or missing. If the param is None - or missing, that filter will be ignored. - - :param switch_ips: Filter switch(es) with IP(s). - :type switch_ips: list of str. Each is as 'xxx.xxx.xxx.xxx'. - :param switch_networks: Filter switche(es) with network(s). - :type switch_networks: list of str. Each is as 'xxx.xxx.xxx.xxx/xx'. - :param limit: int, The maximum number of switches to return. - :type limit: int. 0 means unlimited. - """ - params = {} - if switch_ips: - params['switchIp'] = switch_ips - - if switch_networks: - params['switchIpNetwork'] = switch_networks - - if limit: - params['limit'] = limit - return self._get('/switches', params=params) - - def get_switch(self, switch_id): - """Lists details for a specified switch. - - :param switch_id: switch id. - :type switch_id: int. - """ - return self._get('/switches/%s' % switch_id) - - def add_switch(self, switch_ip, version=None, community=None, - username=None, password=None, raw_data=None): - """Create a switch with specified details. - - .. note:: - It will trigger switch polling if successful. During - the polling, MAC address of the devices connected to the - switch will be learned by SNMP or SSH. - - :param switch_ip: the switch IP address. - :type switch_ip: str, as xxx.xxx.xxx.xxx. - :param version: SNMP version when using SNMP to poll switch. - :type version: str, one in ['v1', 'v2c', 'v3'] - :param community: SNMP community when using SNMP to poll switch. - :type community: str, usually 'public'. - :param username: SSH username when using SSH to poll switch. - :type username: str. - :param password: SSH password when using SSH to poll switch. - :type password: str. - """ - data = {} - if raw_data: - data = raw_data - else: - data['switch'] = {} - data['switch']['ip'] = switch_ip - data['switch']['credential'] = {} - if version: - data['switch']['credential']['version'] = version - - if community: - data['switch']['credential']['community'] = community - - if username: - data['switch']['credential']['username'] = username - - if password: - data['switch']['credential']['password'] = password - - return self._post('/switches', data=data) - - def update_switch(self, switch_id, ip_addr=None, - version=None, community=None, - username=None, password=None, - raw_data=None): - """Updates a switch with specified details. - - .. note:: - It will trigger switch polling if successful. During - the polling, MAC address of the devices connected to the - switch will be learned by SNMP or SSH. - - :param switch_id: switch id - :type switch_id: int. - :param ip_addr: the switch ip address. - :type ip_addr: str, as 'xxx.xxx.xxx.xxx' format. - :param version: SNMP version when using SNMP to poll switch. - :type version: str, one in ['v1', 'v2c', 'v3']. - :param community: SNMP community when using SNMP to poll switch. - :type community: str, usually be 'public'. - :param username: username when using SSH to poll switch. - :type username: str. - :param password: password when using SSH to poll switch. - """ - data = {} - if raw_data: - data = raw_data - else: - data['switch'] = {} - if ip_addr: - data['switch']['ip'] = ip_addr - - data['switch']['credential'] = {} - if version: - data['switch']['credential']['version'] = version - - if community: - data['switch']['credential']['community'] = community - - if username: - data['switch']['credential']['username'] = username - - if password: - data['switch']['credential']['password'] = password - - return self._put('/switches/%s' % switch_id, data=data) - - def delete_switch(self, switch_id): - """Not implemented in api.""" - return self._delete('/switches/%s' % switch_id) - - def get_machines(self, switch_id=None, vlan_id=None, - port=None, limit=None): - """Get the details of machines. - - .. note:: - The machines can be filtered by switch_id, vlan_id, port - and limit. These params can be None or missing. If the param - is None or missing, the filter will be ignored. - - :param switch_id: Return machine(s) connected to the switch. - :type switch_id: int. - :param vlan_id: Return machine(s) belonging to the vlan. - :type vlan_id: int. - :param port: Return machine(s) connect to the port. - :type port: int. - :param limit: the maximum number of machines will be returned. - :type limit: int. 0 means no limit. - """ - params = {} - if switch_id: - params['switchId'] = switch_id - - if vlan_id: - params['vlanId'] = vlan_id - - if port: - params['port'] = port - - if limit: - params['limit'] = limit - - return self._get('/machines', params=params) - - def get_machine(self, machine_id): - """Lists the details for a specified machine. - - :param machine_id: Return machine with the id. - :type machine_id: int. - """ - return self._get('/machines/%s' % machine_id) - - def get_clusters(self): - """Lists the details for all clusters.""" - return self._get('/clusters') - - def get_cluster(self, cluster_id): - """Lists the details of the specified cluster. - - :param cluster_id: cluster id. - :type cluster_id: int. - """ - return self._get('/clusters/%d' % cluster_id) - - def add_cluster(self, cluster_name, adapter_id, raw_data=None): - """Creates a cluster by specified name and given adapter id. - - :param cluster_name: cluster name. - :type cluster_name: str. - :param adapter_id: adapter id. - :type adapter_id: int. - """ - data = {} - if raw_data: - data = raw_data - else: - data['cluster'] = {} - data['cluster']['name'] = cluster_name - data['cluster']['adapter_id'] = adapter_id - return self._post('/clusters', data=data) - - def add_hosts(self, cluster_id, machine_ids, raw_data=None): - """add the specified machine(s) as the host(s) to the cluster. - - :param cluster_id: cluster id. - :type cluster_id: int. - :param machine_ids: machine ids to add to cluster. - :type machine_ids: list of int, each is the id of one machine. - """ - data = {} - if raw_data: - data = raw_data - else: - data['addHosts'] = machine_ids - return self._post('/clusters/%d/action' % cluster_id, data=data) - - def remove_hosts(self, cluster_id, host_ids, raw_data=None): - """remove the specified host(s) from the cluster. - - :param cluster_id: cluster id. - :type cluster_id: int. - :param host_ids: host ids to remove from cluster. - :type host_ids: list of int, each is the id of one host. - """ - data = {} - if raw_data: - data = raw_data - else: - data['removeHosts'] = host_ids - return self._post('/clusters/%s/action' % cluster_id, data=data) - - def replace_hosts(self, cluster_id, machine_ids, raw_data=None): - """replace the cluster hosts with the specified machine(s). - - :param cluster_id: int, The unique identifier of the cluster. - :type cluster_id: int. - :param machine_ids: the machine ids to replace the hosts in cluster. - :type machine_ids: list of int, each is the id of one machine. - """ - data = {} - if raw_data: - data = raw_data - else: - data['replaceAllHosts'] = machine_ids - return self._post('/clusters/%s/action' % cluster_id, data=data) - - def deploy_hosts(self, cluster_id, raw_data=None): - """Deploy the cluster. - - :param cluster_id: The unique identifier of the cluster - :type cluster_id: int. - """ - data = {} - if raw_data: - data = raw_data - else: - data['deploy'] = [] - return self._post('/clusters/%d/action' % cluster_id, data=data) - - @classmethod - def parse_security(cls, kwargs): - """parse the arguments to security data.""" - data = {} - for key, value in kwargs.items(): - if '_' not in key: - continue - key_name, key_value = key.split('_', 1) - data.setdefault( - '%s_credentials' % key_name, {})[key_value] = value - - return data - - def set_security(self, cluster_id, **kwargs): - """Update the cluster security configuration. - - :param cluster_id: cluster id. - :type cluster_id: int. - :param _username: username of the security name. - :type _username: str. - :param _password: passowrd of the security name. - :type _password: str. - - .. note:: - security_name should be one of ['server', 'service', 'console']. - """ - data = {} - data['security'] = self.parse_security(kwargs) - return self._put('/clusters/%d/security' % cluster_id, data=data) - - @classmethod - def parse_networking(cls, kwargs): - """parse arguments to network data.""" - data = {} - global_keys = [ - 'nameservers', 'search_path', 'gateway', - 'proxy', 'ntp_server', 'ha_vip'] - for key, value in kwargs.items(): - if key in global_keys: - data.setdefault('global', {})[key] = value - else: - if '_' not in key: - continue - - key_name, key_value = key.split('_', 1) - data.setdefault( - 'interfaces', {} - ).setdefault( - key_name, {} - )[key_value] = value - - return data - - def set_networking(self, cluster_id, **kwargs): - """Update the cluster network configuration. - - :param cluster_id: cluster id. - :type cluster_id: int. - :param nameservers: comma seperated nameserver ip address. - :type nameservers: str. - :param search_path: comma seperated dns name search path. - :type search_path: str. - :param gateway: gateway ip address for routing to outside. - :type gateway: str. - :param proxy: proxy url for downloading packages. - :type proxy: str. - :param ntp_server: ntp server ip address to sync timestamp. - :type ntp_server: str. - :param ha_vip: ha vip address to run ha proxy. - :type ha_vip: str. - :param _ip_start: start ip address to host's interface. - :type _ip_start: str. - :param _ip_end: end ip address to host's interface. - :type _ip_end: str. - :param _netmask: netmask to host's interface. - :type _netmask: str. - :param _nic: host physical interface name. - :type _nic: str. - :param _promisc: if the interface in promiscous mode. - :type _promisc: int, 0 or 1. - - .. note:: - interface should be one of ['management', 'tenant', - 'public', 'storage']. - """ - data = {} - data['networking'] = self.parse_networking(kwargs) - return self._put('/clusters/%d/networking' % cluster_id, data=data) - - @classmethod - def parse_partition(cls, kwargs): - """parse arguments to partition data.""" - data = {} - for key, value in kwargs.items(): - if key.endswith('_percentage'): - key_name = key[:-len('_percentage')] - data[key_name] = '%s%%' % value - elif key.endswitch('_mbytes'): - key_name = key[:-len('_mbytes')] - data[key_name] = str(value) - - return ';'.join([ - '/%s %s' % (key, value) for key, value in data.items() - ]) - - def set_partition(self, cluster_id, **kwargs): - """Update the cluster partition configuration. - - :param cluster_id: cluster id. - :type cluster_id: int. - :param _percentage: the partiton percentage. - :type _percentage: float between 0 to 100. - :param _mbytes: the partition mbytes. - :type _mbytes: int. - - .. note:: - partition should be one of ['home', 'var', 'tmp']. - """ - data = {} - data['partition'] = self.parse_partition(kwargs) - return self._put('/clusters/%s/partition' % cluster_id, data=data) - - def get_hosts(self, hostname=None, clustername=None): - """Lists the details of hosts. - - .. note:: - The hosts can be filtered by hostname, clustername. - These params can be None or missing. If the param - is None or missing, the filter will be ignored. - - :param hostname: The name of a host. - :type hostname: str. - :param clustername: The name of a cluster. - :type clustername: str. - """ - params = {} - if hostname: - params['hostname'] = hostname - - if clustername: - params['clustername'] = clustername - - return self._get('/clusterhosts', params=params) - - def get_host(self, host_id): - """Lists the details for the specified host. - - :param host_id: host id. - :type host_id: int. - """ - return self._get('/clusterhosts/%s' % host_id) - - def get_host_config(self, host_id): - """Lists the details of the config for the specified host. - - :param host_id: host id. - :type host_id: int. - """ - return self._get('/clusterhosts/%s/config' % host_id) - - def update_host_config(self, host_id, hostname=None, - roles=None, raw_data=None, **kwargs): - """Updates config for the host. - - :param host_id: host id. - :type host_id: int. - :param hostname: host name. - :type hostname: str. - :param security__username: username of the security name. - :type security__username: str. - :param security__password: passowrd of the security name. - :type security__password: str. - :param networking_nameservers: comma seperated nameserver ip address. - :type networking_nameservers: str. - :param networking_search_path: comma seperated dns name search path. - :type networking_search_path: str. - :param networking_gateway: gateway ip address for routing to outside. - :type networking_gateway: str. - :param networking_proxy: proxy url for downloading packages. - :type networking_proxy: str. - :param networking_ntp_server: ntp server ip address to sync timestamp. - :type networking_ntp_server: str. - :param networking__ip: ip address to host interface. - :type networking__ip: str. - :param networking__netmask: netmask to host's interface. - :type networking__netmask: str. - :param networking__nic: host physical interface name. - :type networking__nic: str. - :param networking__promisc: if the interface is promiscous. - :type networking__promisc: int, 0 or 1. - :param partition__percentage: the partiton percentage. - :type partition__percentage: float between 0 to 100. - :param partition__mbytes: the partition mbytes. - :type partition__mbytes: int. - :param roles: host assigned roles in the cluster. - :type roles: list of str. - """ - data = {} - if raw_data: - data = raw_data - else: - if hostname: - data['hostname'] = hostname - - sub_kwargs = {} - for key, value in kwargs.items(): - key_name, key_value = key.split('_', 1) - sub_kwargs.setdefault(key_name, {})[key_value] = value - - if 'security' in sub_kwargs: - data['security'] = self.parse_security(sub_kwargs['security']) - - if 'networking' in sub_kwargs: - data['networking'] = self.parse_networking( - sub_kwargs['networking']) - if 'partition' in sub_kwargs: - data['partition'] = self.parse_partition( - sub_kwargs['partition']) - - if roles: - data['roles'] = roles - - return self._put('/clusterhosts/%s/config' % host_id, data) - - def delete_from_host_config(self, host_id, delete_key): - """Deletes one key in config for the host. - - :param host_id: host id. - :type host_id: int. - :param delete_key: the key in host config to be deleted. - :type delete_key: str. - """ - return self._delete('/clusterhosts/%s/config/%s' % ( - host_id, delete_key)) - - def get_adapters(self, name=None): - """Lists details of adapters. - - .. note:: - the adapter can be filtered by name of name is given and not None. - - :param name: adapter name. - :type name: str. - """ - params = {} - if name: - params['name'] = name - - return self._get('/adapters', params=params) - - def get_adapter(self, adapter_id): - """Lists details for the specified adapter. - - :param adapter_id: adapter id. - :type adapter_id: int. - """ - return self._get('/adapters/%s' % adapter_id) - - def get_adapter_roles(self, adapter_id): - """Lists roles to assign to hosts for the specified adapter. - - :param adapter_id: adapter id. - :type adapter_id: int. - """ - return self._get('/adapters/%s/roles' % adapter_id) - - def get_host_installing_progress(self, host_id): - """Lists progress details for the specified host. - - :param host_id: host id. - :type host_id: int. - """ - return self._get('/clusterhosts/%s/progress' % host_id) - - def get_cluster_installing_progress(self, cluster_id): - """Lists progress details for the specified cluster. - - :param cluster_id: cluster id. - :param cluster_id: int. - """ - - return self._get('/clusters/%s/progress' % cluster_id) - - def get_dashboard_links(self, cluster_id): - """Lists links for dashboards of deployed cluster. - - :param cluster_id: cluster id. - :type cluster_id: int. - """ - params = {} - params['cluster_id'] = cluster_id - return self._get('/dashboardlinks', params) diff --git a/compass-tasks/build.sh b/compass-tasks/build.sh deleted file mode 100755 index e0dceea..0000000 --- a/compass-tasks/build.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -############################################################################## -# Copyright (c) 2016-2017 HUAWEI TECHNOLOGIES CO.,LTD and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -set -x -COMPASS_DIR=${BASH_SOURCE[0]%/*} - -rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -sed -i 's/^mirrorlist=https/mirrorlist=http/g' /etc/yum.repos.d/epel.repo -yum update -y - -yum --nogpgcheck install -y python python-devel git amqp python-pip libffi-devel openssl-devel gcc python-setuptools MySQL-python supervisor redis sshpass python-keyczar vim ansible-2.2.1.0 libyaml-devel make - -mkdir -p $COMPASS_DIR/compass -touch $COMPASS_DIR/compass/__init__.py -mv $COMPASS_DIR/actions $COMPASS_DIR/compass/ -mv $COMPASS_DIR/apiclient $COMPASS_DIR/compass/ -mv $COMPASS_DIR/tasks $COMPASS_DIR/compass/ -mv $COMPASS_DIR/utils $COMPASS_DIR/compass/ -mv $COMPASS_DIR/deployment $COMPASS_DIR/compass/ -mv $COMPASS_DIR/db $COMPASS_DIR/compass/ -mv $COMPASS_DIR/hdsdiscovery $COMPASS_DIR/compass/ -mv $COMPASS_DIR/log_analyzor $COMPASS_DIR/compass/ - -easy_install --upgrade pip -pip install --upgrade pip -pip install --upgrade setuptools -pip install --upgrade Flask - -mkdir -p /etc/compass/ -mkdir -p /etc/compass/machine_list -mkdir -p /etc/compass/switch_list -mkdir -p /var/log/compass -mkdir -p /opt/ansible_callbacks -mkdir -p /root/.ssh; -echo "UserKnownHostsFile /dev/null" >> /root/.ssh/config; -echo "StrictHostKeyChecking no" >> /root/.ssh/config - -cd $COMPASS_DIR -python setup.py install -cp supervisord.conf /etc/supervisord.conf -cp start.sh /usr/local/bin/start.sh - -yum clean all - -set -x diff --git a/compass-tasks/db/__init__.py b/compass-tasks/db/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/db/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/db/api/__init__.py b/compass-tasks/db/api/__init__.py deleted file mode 100644 index 5e42ae9..0000000 --- a/compass-tasks/db/api/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/db/api/adapter.py b/compass-tasks/db/api/adapter.py deleted file mode 100644 index c3ad48d..0000000 --- a/compass-tasks/db/api/adapter.py +++ /dev/null @@ -1,313 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Adapter related database operations.""" -import logging -import re - -from compass.db.api import database -from compass.db.api import utils -from compass.db import exception -from compass.db import models - -from compass.utils import setting_wrapper as setting -from compass.utils import util - - -OSES = None -OS_INSTALLERS = None -PACKAGE_INSTALLERS = None -ADAPTERS = None -ADAPTERS_FLAVORS = None -ADAPTERS_ROLES = None - - -def _get_oses_from_configuration(): - """Get all os configs from os configuration dir. - - Example: { - : { - 'name': , - 'id': , - 'os_id': , - 'deployable': True - } - } - """ - configs = util.load_configs(setting.OS_DIR) - systems = {} - for config in configs: - logging.info('get config %s', config) - system_name = config['NAME'] - parent_name = config.get('PARENT', None) - system = { - 'name': system_name, - 'id': system_name, - 'os_id': system_name, - 'parent': parent_name, - 'parent_id': parent_name, - 'deployable': config.get('DEPLOYABLE', False) - } - systems[system_name] = system - parents = {} - for name, system in systems.items(): - parent = system.get('parent', None) - parents[name] = parent - for name, system in systems.items(): - util.recursive_merge_dict(name, systems, parents) - return systems - - -def _get_installers_from_configuration(configs): - """Get installers from configurations. - - Example: { - : { - 'alias': , - 'id': , - 'name': , - 'settings': - } - } - """ - installers = {} - for config in configs: - name = config['NAME'] - instance_name = config.get('INSTANCE_NAME', name) - installers[instance_name] = { - 'alias': instance_name, - 'id': instance_name, - 'name': name, - 'settings': config.get('SETTINGS', {}) - } - return installers - - -def _get_os_installers_from_configuration(): - """Get os installers from os installer config dir.""" - configs = util.load_configs(setting.OS_INSTALLER_DIR) - return _get_installers_from_configuration(configs) - - -def _get_package_installers_from_configuration(): - """Get package installers from package installer config dir.""" - configs = util.load_configs(setting.PACKAGE_INSTALLER_DIR) - return _get_installers_from_configuration(configs) - - -def _get_adapters_from_configuration(): - """Get adapters from adapter config dir.""" - configs = util.load_configs(setting.ADAPTER_DIR) - adapters = {} - for config in configs: - logging.info('add config %s to adapter', config) - if 'OS_INSTALLER' in config: - os_installer = OS_INSTALLERS[config['OS_INSTALLER']] - else: - os_installer = None - - if 'PACKAGE_INSTALLER' in config: - package_installer = PACKAGE_INSTALLERS[ - config['PACKAGE_INSTALLER'] - ] - else: - package_installer = None - - adapter_name = config['NAME'] - parent_name = config.get('PARENT', None) - adapter = { - 'name': adapter_name, - 'id': adapter_name, - 'parent': parent_name, - 'parent_id': parent_name, - 'display_name': config.get('DISPLAY_NAME', adapter_name), - 'os_installer': os_installer, - 'package_installer': package_installer, - 'deployable': config.get('DEPLOYABLE', False), - 'health_check_cmd': config.get('HEALTH_CHECK_COMMAND', None), - 'supported_oses': [], - 'roles': [], - 'flavors': [] - } - supported_os_patterns = [ - re.compile(supported_os_pattern) - for supported_os_pattern in config.get('SUPPORTED_OS_PATTERNS', []) - ] - for os_name, os in OSES.items(): - if not os.get('deployable', False): - continue - for supported_os_pattern in supported_os_patterns: - if supported_os_pattern.match(os_name): - adapter['supported_oses'].append(os) - break - adapters[adapter_name] = adapter - - parents = {} - for name, adapter in adapters.items(): - parent = adapter.get('parent', None) - parents[name] = parent - for name, adapter in adapters.items(): - util.recursive_merge_dict(name, adapters, parents) - return adapters - - -def _add_roles_from_configuration(): - """Get roles from roles config dir and update to adapters.""" - configs = util.load_configs(setting.ADAPTER_ROLE_DIR) - for config in configs: - logging.info( - 'add config %s to role', config - ) - adapter_name = config['ADAPTER_NAME'] - adapter = ADAPTERS[adapter_name] - adapter_roles = ADAPTERS_ROLES.setdefault(adapter_name, {}) - for role_dict in config['ROLES']: - role_name = role_dict['role'] - display_name = role_dict.get('display_name', role_name) - adapter_roles[role_name] = { - 'name': role_name, - 'id': '%s:%s' % (adapter_name, role_name), - 'adapter_id': adapter_name, - 'adapter_name': adapter_name, - 'display_name': display_name, - 'description': role_dict.get('description', display_name), - 'optional': role_dict.get('optional', False) - } - parents = {} - for name, adapter in ADAPTERS.items(): - parent = adapter.get('parent', None) - parents[name] = parent - for adapter_name, adapter_roles in ADAPTERS_ROLES.items(): - util.recursive_merge_dict(adapter_name, ADAPTERS_ROLES, parents) - for adapter_name, adapter_roles in ADAPTERS_ROLES.items(): - adapter = ADAPTERS[adapter_name] - adapter['roles'] = adapter_roles.values() - - -def _add_flavors_from_configuration(): - """Get flavors from flavor config dir and update to adapters.""" - configs = util.load_configs(setting.ADAPTER_FLAVOR_DIR) - for config in configs: - logging.info('add config %s to flavor', config) - adapter_name = config['ADAPTER_NAME'] - adapter = ADAPTERS[adapter_name] - adapter_flavors = ADAPTERS_FLAVORS.setdefault(adapter_name, {}) - adapter_roles = ADAPTERS_ROLES[adapter_name] - for flavor_dict in config['FLAVORS']: - flavor_name = flavor_dict['flavor'] - flavor_id = '%s:%s' % (adapter_name, flavor_name) - flavor = { - 'name': flavor_name, - 'id': flavor_id, - 'adapter_id': adapter_name, - 'adapter_name': adapter_name, - 'display_name': flavor_dict.get('display_name', flavor_name), - 'template': flavor_dict.get('template', None) - } - flavor_roles = flavor_dict.get('roles', []) - roles_in_flavor = [] - for flavor_role in flavor_roles: - if isinstance(flavor_role, basestring): - role_name = flavor_role - role_in_flavor = { - 'name': role_name, - 'flavor_id': flavor_id - } - else: - role_in_flavor = flavor_role - role_in_flavor['flavor_id'] = flavor_id - if 'role' in role_in_flavor: - role_in_flavor['name'] = role_in_flavor['role'] - del role_in_flavor['role'] - role_name = role_in_flavor['name'] - role = adapter_roles[role_name] - util.merge_dict(role_in_flavor, role, override=False) - roles_in_flavor.append(role_in_flavor) - flavor['roles'] = roles_in_flavor - adapter_flavors[flavor_name] = flavor - parents = {} - for name, adapter in ADAPTERS.items(): - parent = adapter.get('parent', None) - parents[name] = parent - for adapter_name, adapter_roles in ADAPTERS_FLAVORS.items(): - util.recursive_merge_dict(adapter_name, ADAPTERS_FLAVORS, parents) - for adapter_name, adapter_flavors in ADAPTERS_FLAVORS.items(): - adapter = ADAPTERS[adapter_name] - adapter['flavors'] = adapter_flavors.values() - - -def load_adapters_internal(force_reload=False): - """Load adapter related configurations into memory. - - If force_reload, reload all configurations even it is loaded already. - """ - global OSES - if force_reload or OSES is None: - OSES = _get_oses_from_configuration() - global OS_INSTALLERS - if force_reload or OS_INSTALLERS is None: - OS_INSTALLERS = _get_os_installers_from_configuration() - global PACKAGE_INSTALLERS - if force_reload or PACKAGE_INSTALLERS is None: - PACKAGE_INSTALLERS = _get_package_installers_from_configuration() - global ADAPTERS - if force_reload or ADAPTERS is None: - ADAPTERS = _get_adapters_from_configuration() - global ADAPTERS_ROLES - if force_reload or ADAPTERS_ROLES is None: - ADAPTERS_ROLES = {} - _add_roles_from_configuration() - global ADAPTERS_FLAVORS - if force_reload or ADAPTERS_FLAVORS is None: - ADAPTERS_FLAVORS = {} - _add_flavors_from_configuration() - - -def get_adapters_internal(force_reload=False): - """Get all deployable adapters.""" - load_adapters_internal(force_reload=force_reload) - adapter_mapping = {} - for adapter_name, adapter in ADAPTERS.items(): - if adapter.get('deployable'): - # TODO(xicheng): adapter should be filtered before - # return to caller. - adapter_mapping[adapter_name] = adapter - else: - logging.info( - 'ignore adapter %s since it is not deployable', - adapter_name - ) - return adapter_mapping - - -def get_flavors_internal(force_reload=False): - """Get all deployable flavors.""" - load_adapters_internal(force_reload=force_reload) - adapter_flavor_mapping = {} - for adapter_name, adapter_flavors in ADAPTERS_FLAVORS.items(): - adapter = ADAPTERS.get(adapter_name, {}) - for flavor_name, flavor in adapter_flavors.items(): - if adapter.get('deployable'): - # TODO(xicheng): flavor dict should be filtered before - # return to caller. - adapter_flavor_mapping.setdefault( - adapter_name, {} - )[flavor_name] = flavor - else: - logging.info( - 'ignore adapter %s since it is not deployable', - adapter_name - ) - - return adapter_flavor_mapping diff --git a/compass-tasks/db/api/adapter_holder.py b/compass-tasks/db/api/adapter_holder.py deleted file mode 100644 index 91c65c4..0000000 --- a/compass-tasks/db/api/adapter_holder.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Adapter related object holder.""" -import logging - -from compass.db.api import adapter as adapter_api -from compass.db.api import database -from compass.db.api import permission -from compass.db.api import user as user_api -from compass.db.api import utils -from compass.db import exception - - -SUPPORTED_FIELDS = [ - 'name', -] -RESP_FIELDS = [ - 'id', 'name', 'roles', 'flavors', - 'os_installer', 'package_installer', - 'supported_oses', 'display_name', 'health_check_cmd' -] -RESP_OS_FIELDS = [ - 'id', 'name', 'os_id' -] -RESP_ROLES_FIELDS = [ - 'id', 'name', 'display_name', 'description', 'optional' -] -RESP_FLAVORS_FIELDS = [ - 'id', 'adapter_id', 'adapter_name', 'name', 'display_name', - 'template', 'roles' -] - - -ADAPTER_MAPPING = None -FLAVOR_MAPPING = None - - -def load_adapters(force_reload=False): - global ADAPTER_MAPPING - if force_reload or ADAPTER_MAPPING is None: - logging.info('load adapters into memory') - ADAPTER_MAPPING = adapter_api.get_adapters_internal( - force_reload=force_reload - ) - - -def load_flavors(force_reload=False): - global FLAVOR_MAPPING - if force_reload or FLAVOR_MAPPING is None: - logging.info('load flavors into memory') - FLAVOR_MAPPING = {} - adapters_flavors = adapter_api.get_flavors_internal( - force_reload=force_reload - ) - for adapter_name, adapter_flavors in adapters_flavors.items(): - for flavor_name, flavor in adapter_flavors.items(): - FLAVOR_MAPPING['%s:%s' % (adapter_name, flavor_name)] = flavor - - -def _filter_adapters(adapter_config, filter_name, filter_value): - if filter_name not in adapter_config: - return False - if isinstance(filter_value, list): - return bool( - adapter_config[filter_name] in filter_value - ) - elif isinstance(filter_value, dict): - return all([ - _filter_adapters( - adapter_config[filter_name], - sub_filter_key, sub_filter_value - ) - for sub_filter_key, sub_filter_value in filter_value.items() - ]) - else: - return adapter_config[filter_name] == filter_value - - -@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_ADAPTERS -) -@utils.output_filters(name=utils.general_filter_callback) -@utils.wrap_to_dict( - RESP_FIELDS, - supported_oses=RESP_OS_FIELDS, - roles=RESP_ROLES_FIELDS, - flavors=RESP_FLAVORS_FIELDS -) -def list_adapters(user=None, session=None, **filters): - """list adapters.""" - load_adapters() - return ADAPTER_MAPPING.values() - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_ADAPTERS -) -@utils.wrap_to_dict( - RESP_FIELDS, - supported_oses=RESP_OS_FIELDS, - roles=RESP_ROLES_FIELDS, - flavors=RESP_FLAVORS_FIELDS -) -def get_adapter(adapter_id, user=None, session=None, **kwargs): - """get adapter.""" - load_adapters() - if adapter_id not in ADAPTER_MAPPING: - raise exception.RecordNotExists( - 'adpater %s does not exist' % adapter_id - ) - return ADAPTER_MAPPING[adapter_id] - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_FLAVORS_FIELDS) -def list_flavors(user=None, session=None, **filters): - """List flavors.""" - load_flavors() - return FLAVOR_MAPPING.values() - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_FLAVORS_FIELDS) -def get_flavor(flavor_id, user=None, session=None, **kwargs): - """Get flavor.""" - load_flavors() - if flavor_id not in FLAVOR_MAPPING: - raise exception.RecordNotExists( - 'flavor %s does not exist' % flavor_id - ) - return FLAVOR_MAPPING[flavor_id] diff --git a/compass-tasks/db/api/cluster.py b/compass-tasks/db/api/cluster.py deleted file mode 100644 index 7a7022c..0000000 --- a/compass-tasks/db/api/cluster.py +++ /dev/null @@ -1,2444 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, - -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Cluster database operations.""" -import copy -import functools -import logging -import re - -from compass.db.api import adapter_holder as adapter_api -from compass.db.api import database -from compass.db.api import metadata_holder as metadata_api -from compass.db.api import permission -from compass.db.api import user as user_api -from compass.db.api import utils -from compass.db import exception -from compass.db import models -from compass.utils import util - - -SUPPORTED_FIELDS = [ - 'name', 'os_name', 'owner', - 'adapter_name', 'flavor_name' -] -SUPPORTED_CLUSTERHOST_FIELDS = [] -RESP_FIELDS = [ - 'id', 'name', 'os_name', 'os_id', 'adapter_id', 'flavor_id', - 'reinstall_distributed_system', 'flavor', - 'distributed_system_installed', - 'owner', 'adapter_name', 'flavor_name', - 'created_at', 'updated_at' -] -RESP_CLUSTERHOST_FIELDS = [ - 'id', 'host_id', 'clusterhost_id', 'machine_id', - 'name', 'hostname', 'roles', 'os_installer', - 'cluster_id', 'clustername', 'location', 'tag', - 'networks', 'mac', 'switch_ip', 'port', 'switches', - 'os_installed', 'distributed_system_installed', - 'os_name', 'os_id', 'ip', - 'reinstall_os', 'reinstall_distributed_system', - 'owner', 'cluster_id', - 'created_at', 'updated_at', - 'patched_roles' -] -RESP_CONFIG_FIELDS = [ - 'os_config', - 'package_config', - 'config_step', - 'config_validated', - 'created_at', - 'updated_at' -] -RESP_DEPLOYED_CONFIG_FIELDS = [ - 'deployed_os_config', - 'deployed_package_config', - 'created_at', - 'updated_at' -] -RESP_METADATA_FIELDS = [ - 'os_config', 'package_config' -] -RESP_CLUSTERHOST_CONFIG_FIELDS = [ - 'package_config', - 'os_config', - 'config_step', - 'config_validated', - 'networks', - 'created_at', - 'updated_at' -] -RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS = [ - 'deployed_os_config', - 'deployed_package_config', - 'created_at', - 'updated_at' -] -RESP_STATE_FIELDS = [ - 'id', 'state', 'percentage', 'message', 'severity', - 'status', 'ready', - 'created_at', 'updated_at' -] -RESP_CLUSTERHOST_STATE_FIELDS = [ - 'id', 'state', 'percentage', 'message', 'severity', - 'ready', 'created_at', 'updated_at' -] -RESP_REVIEW_FIELDS = [ - 'cluster', 'hosts' -] -RESP_DEPLOY_FIELDS = [ - 'status', 'cluster', 'hosts' -] -IGNORE_FIELDS = ['id', 'created_at', 'updated_at'] -ADDED_FIELDS = ['name', 'adapter_id', 'os_id'] -OPTIONAL_ADDED_FIELDS = ['flavor_id'] -UPDATED_FIELDS = ['name', 'reinstall_distributed_system'] -ADDED_HOST_FIELDS = ['machine_id'] -UPDATED_HOST_FIELDS = ['name', 'reinstall_os'] -UPDATED_CLUSTERHOST_FIELDS = ['roles', 'patched_roles'] -PATCHED_CLUSTERHOST_FIELDS = ['patched_roles'] -UPDATED_CONFIG_FIELDS = [ - 'put_os_config', 'put_package_config', 'config_step' -] -UPDATED_DEPLOYED_CONFIG_FIELDS = [ - 'deployed_os_config', 'deployed_package_config' -] -PATCHED_CONFIG_FIELDS = [ - 'patched_os_config', 'patched_package_config', 'config_step' -] -UPDATED_CLUSTERHOST_CONFIG_FIELDS = [ - 'put_os_config', - 'put_package_config' -] -PATCHED_CLUSTERHOST_CONFIG_FIELDS = [ - 'patched_os_config', - 'patched_package_config' -] -UPDATED_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS = [ - 'deployed_os_config', - 'deployed_package_config' -] -UPDATED_CLUSTERHOST_STATE_FIELDS = [ - 'state', 'percentage', 'message', 'severity' -] -UPDATED_CLUSTERHOST_STATE_INTERNAL_FIELDS = [ - 'ready' -] -UPDATED_CLUSTER_STATE_FIELDS = ['state'] -IGNORE_UPDATED_CLUSTER_STATE_FIELDS = ['percentage', 'message', 'severity'] -UPDATED_CLUSTER_STATE_INTERNAL_FIELDS = ['ready'] -RESP_CLUSTERHOST_LOG_FIELDS = [ - 'clusterhost_id', 'id', 'host_id', 'cluster_id', - 'filename', 'position', 'partial_line', - 'percentage', - 'message', 'severity', 'line_matcher_name' -] -ADDED_CLUSTERHOST_LOG_FIELDS = [ - 'filename' -] -UPDATED_CLUSTERHOST_LOG_FIELDS = [ - 'position', 'partial_line', 'percentage', - 'message', 'severity', 'line_matcher_name' -] - - -@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_CLUSTERS -) -@utils.wrap_to_dict(RESP_FIELDS) -def list_clusters(user=None, session=None, **filters): - """List clusters.""" - clusters = utils.list_db_objects( - session, models.Cluster, **filters - ) - logging.info('user is %s', user.email) - if not user.is_admin and len(clusters): - clusters = [c for c in clusters if c.owner == user.email] - return clusters - - -def _get_cluster(cluster_id, session=None, **kwargs): - """Get cluster by id.""" - if isinstance(cluster_id, (int, long)): - return utils.get_db_object( - session, models.Cluster, id=cluster_id, **kwargs - ) - raise exception.InvalidParameter( - 'cluster id %s type is not int compatible' % cluster_id - ) - - -def get_cluster_internal(cluster_id, session=None, **kwargs): - """Helper function to get cluster. - - Should be only used by other files under db/api. - """ - return _get_cluster(cluster_id, session=session, **kwargs) - - -def _get_cluster_host( - cluster_id, host_id, session=None, **kwargs -): - """Get clusterhost by cluster id and host id.""" - cluster = _get_cluster(cluster_id, session=session, **kwargs) - from compass.db.api import host as host_api - host = host_api.get_host_internal(host_id, session=session, **kwargs) - return utils.get_db_object( - session, models.ClusterHost, - cluster_id=cluster.id, - host_id=host.id, - **kwargs - ) - - -def _get_clusterhost(clusterhost_id, session=None, **kwargs): - """Get clusterhost by clusterhost id.""" - if isinstance(clusterhost_id, (int, long)): - return utils.get_db_object( - session, models.ClusterHost, - clusterhost_id=clusterhost_id, - **kwargs - ) - raise exception.InvalidParameter( - 'clusterhost id %s type is not int compatible' % clusterhost_id - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_CLUSTERS -) -@utils.wrap_to_dict(RESP_FIELDS) -def get_cluster( - cluster_id, exception_when_missing=True, - user=None, session=None, **kwargs -): - """Get cluster info.""" - return _get_cluster( - cluster_id, - session=session, - exception_when_missing=exception_when_missing - ) - - -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_CLUSTERS) -def is_cluster_os_ready( - cluster_id, exception_when_missing=True, - user=None, session=None, **kwargs -): - cluster = utils.get_db_object( - session, models.Cluster, exception_when_missing, id=cluster_id) - - all_states = ([i.host.state.ready for i in cluster.clusterhosts]) - - logging.info("is_cluster_os_ready: all_states %s" % all_states) - - return all(all_states) - - -def check_cluster_validated(cluster): - """Check cluster is validated.""" - if not cluster.config_validated: - raise exception.Forbidden( - 'cluster %s is not validated' % cluster.name - ) - - -def check_clusterhost_validated(clusterhost): - """Check clusterhost is validated.""" - if not clusterhost.config_validated: - raise exception.Forbidden( - 'clusterhost %s is not validated' % clusterhost.name - ) - - -def check_cluster_editable( - cluster, user=None, - check_in_installing=False -): - """Check if cluster is editable. - - If we try to set cluster - reinstall_distributed_system attribute or any - checking to make sure the cluster is not in installing state, - we can set check_in_installing to True. - Otherwise we will make sure the cluster is not in deploying or - deployed. - If user is not admin or not the owner of the cluster, the check - will fail to make sure he can not update the cluster attributes. - """ - if check_in_installing: - if cluster.state.state == 'INSTALLING': - raise exception.Forbidden( - 'cluster %s is not editable ' - 'when state is installing' % cluster.name - ) -# elif ( -# cluster.flavor_name and -# not cluster.reinstall_distributed_system -# ): -# raise exception.Forbidden( -# 'cluster %s is not editable ' -# 'when not to be reinstalled' % cluster.name -# ) - if user and not user.is_admin and cluster.creator_id != user.id: - raise exception.Forbidden( - 'cluster %s is not editable ' - 'when user is not admin or cluster owner' % cluster.name - ) - - -def is_cluster_editable( - cluster, user=None, - check_in_installing=False -): - """Get if cluster is editble.""" - try: - check_cluster_editable( - cluster, user=user, - check_in_installing=check_in_installing - ) - return True - except exception.Forbidden: - return False - - -@utils.supported_filters( - ADDED_FIELDS, - optional_support_keys=OPTIONAL_ADDED_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates(name=utils.check_name) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_CLUSTER -) -@utils.wrap_to_dict(RESP_FIELDS) -def add_cluster( - exception_when_existing=True, - name=None, adapter_id=None, flavor_id=None, - user=None, session=None, **kwargs -): - """Create a cluster.""" - adapter = adapter_api.get_adapter( - adapter_id, user=user, session=session - ) - # if flavor_id is not None, also set flavor field. - # In future maybe we can move the use of flavor from - # models.py to db/api and explictly get flavor when - # needed instead of setting flavor into cluster record. - flavor = {} - if flavor_id: - flavor = adapter_api.get_flavor( - flavor_id, - user=user, session=session - ) - if flavor['adapter_id'] != adapter['id']: - raise exception.InvalidParameter( - 'flavor %s is not of adapter %s' % ( - flavor_id, adapter_id - ) - ) - - cluster = utils.add_db_object( - session, models.Cluster, exception_when_existing, - name, user.id, adapter_id=adapter_id, - flavor_id=flavor_id, flavor=flavor, **kwargs - ) - return cluster - - -@utils.supported_filters( - optional_support_keys=UPDATED_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates(name=utils.check_name) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_CLUSTER -) -@utils.wrap_to_dict(RESP_FIELDS) -def update_cluster(cluster_id, user=None, session=None, **kwargs): - """Update a cluster.""" - cluster = _get_cluster( - cluster_id, session=session - ) - check_cluster_editable( - cluster, user=user, - check_in_installing=( - kwargs.get('reinstall_distributed_system', False) - ) - ) - return utils.update_db_object(session, cluster, **kwargs) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEL_CLUSTER -) -@utils.wrap_to_dict( - RESP_FIELDS + ['status', 'cluster', 'hosts'], - cluster=RESP_FIELDS, - hosts=RESP_CLUSTERHOST_FIELDS -) -def del_cluster( - cluster_id, force=False, from_database_only=False, - delete_underlying_host=False, user=None, session=None, **kwargs -): - """Delete a cluster. - - If force, the cluster will be deleted anyway. It is used by cli to - force clean a cluster in any case. - If from_database_only, the cluster recored will only be removed from - database. Otherwise, a del task is sent to celery to do clean deletion. - If delete_underlying_host, all hosts under this cluster will also be - deleted. - The backend will call del_cluster again with from_database_only set - when it has done the deletion work on os installer/package installer. - """ - cluster = _get_cluster( - cluster_id, session=session - ) - logging.debug( - 'delete cluster %s with force=%s ' - 'from_database_only=%s delete_underlying_host=%s', - cluster.id, force, from_database_only, delete_underlying_host - ) - # force set cluster state to ERROR and the state of any clusterhost - # in the cluster to ERROR when we want to delete the cluster anyway - # even the cluster is in installing or already installed. - # It let the api know the deleting is in doing when backend is doing - # the real deleting. - # In future we may import a new state like INDELETE to indicate - # the deleting is processing. - # We need discuss about if we can delete a cluster when it is already - # installed by api. - for clusterhost in cluster.clusterhosts: - if clusterhost.state.state != 'UNINITIALIZED' and force: - clusterhost.state.state = 'ERROR' - if delete_underlying_host: - host = clusterhost.host - if host.state.state != 'UNINITIALIZED' and force: - host.state.state = 'ERROR' - if cluster.state.state != 'UNINITIALIZED' and force: - cluster.state.state = 'ERROR' - - check_cluster_editable( - cluster, user=user, - check_in_installing=True - ) - - # delete underlying host if delete_underlying_host is set. - if delete_underlying_host: - for clusterhost in cluster.clusterhosts: - # delete underlying host only user has permission. - from compass.db.api import host as host_api - host = clusterhost.host - if host_api.is_host_editable( - host, user=user, check_in_installing=True - ): - # Delete host record directly in database when there is no need - # to do the deletion in backend or from_database_only is set. - if host.state.state == 'UNINITIALIZED' or from_database_only: - utils.del_db_object( - session, host - ) - - # Delete cluster record directly in database when there - # is no need to do the deletion in backend or from_database_only is set. - if cluster.state.state == 'UNINITIALIZED' or from_database_only: - return utils.del_db_object( - session, cluster - ) - else: - from compass.tasks import client as celery_client - logging.info('send del cluster %s task to celery', cluster_id) - celery_client.celery.send_task( - 'compass.tasks.delete_cluster', - ( - user.email, cluster.id, - [ - clusterhost.host_id - for clusterhost in cluster.clusterhosts - ], - delete_underlying_host - ), - queue=user.email, - exchange=user.email, - routing_key=user.email - ) - return { - 'status': 'delete action is sent', - 'cluster': cluster, - 'hosts': cluster.clusterhosts - } - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_CLUSTER_CONFIG -) -@utils.wrap_to_dict(RESP_CONFIG_FIELDS) -def get_cluster_config(cluster_id, user=None, session=None, **kwargs): - """Get cluster config.""" - return _get_cluster(cluster_id, session=session) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_CLUSTER_CONFIG -) -@utils.wrap_to_dict(RESP_DEPLOYED_CONFIG_FIELDS) -def get_cluster_deployed_config(cluster_id, user=None, session=None, **kwargs): - """Get cluster deployed config.""" - return _get_cluster(cluster_id, session=session) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_METADATA_FIELDS) -def get_cluster_metadata(cluster_id, user=None, session=None, **kwargs): - """Get cluster metadata. - - If no flavor in the cluster, it means this is a os only cluster. - We ignore package metadata for os only cluster. - """ - cluster = _get_cluster(cluster_id, session=session) - metadatas = {} - os_name = cluster.os_name - if os_name: - metadatas.update( - metadata_api.get_os_metadata( - os_name, session=session - ) - ) - flavor_id = cluster.flavor_id - if flavor_id: - metadatas.update( - metadata_api.get_flavor_metadata( - flavor_id, - user=user, session=session - ) - ) - - return metadatas - - -def _cluster_os_config_validates( - config, cluster, session=None, user=None, **kwargs -): - """Check cluster os config validation.""" - metadata_api.validate_os_config( - config, cluster.os_id - ) - - -def _cluster_package_config_validates( - config, cluster, session=None, user=None, **kwargs -): - """Check cluster package config validation.""" - metadata_api.validate_flavor_config( - config, cluster.flavor_id - ) - - -@utils.input_validates_with_args( - put_os_config=_cluster_os_config_validates, - put_package_config=_cluster_package_config_validates -) -@utils.output_validates_with_args( - os_config=_cluster_os_config_validates, - package_config=_cluster_package_config_validates -) -@utils.wrap_to_dict(RESP_CONFIG_FIELDS) -def _update_cluster_config(cluster, session=None, user=None, **kwargs): - """Update a cluster config.""" - check_cluster_editable(cluster, user=user) - return utils.update_db_object( - session, cluster, **kwargs - ) - - -# replace os_config to deployed_os_config, -# package_config to deployed_package_config -@utils.replace_filters( - os_config='deployed_os_config', - package_config='deployed_package_config' -) -@utils.supported_filters( - optional_support_keys=UPDATED_DEPLOYED_CONFIG_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_CLUSTER_CONFIG -) -@utils.wrap_to_dict(RESP_DEPLOYED_CONFIG_FIELDS) -def update_cluster_deployed_config( - cluster_id, user=None, session=None, **kwargs -): - """Update cluster deployed config.""" - cluster = _get_cluster(cluster_id, session=session) - check_cluster_editable(cluster, user=user) - check_cluster_validated(cluster) - return utils.update_db_object( - session, cluster, **kwargs - ) - - -# replace os_config to put_os_config, -# package_config to put_package_config in kwargs. -# It tells db these fields will be updated not patched. -@utils.replace_filters( - os_config='put_os_config', - package_config='put_package_config' -) -@utils.supported_filters( - optional_support_keys=UPDATED_CONFIG_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_CLUSTER_CONFIG -) -def update_cluster_config(cluster_id, user=None, session=None, **kwargs): - """Update cluster config.""" - cluster = _get_cluster(cluster_id, session=session) - return _update_cluster_config( - cluster, session=session, user=user, **kwargs - ) - - -# replace os_config to patched_os_config and -# package_config to patched_package_config in kwargs. -# It tells db these fields will be patched not updated. -@utils.replace_filters( - os_config='patched_os_config', - package_config='patched_package_config' -) -@utils.supported_filters( - optional_support_keys=PATCHED_CONFIG_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_CLUSTER_CONFIG -) -def patch_cluster_config(cluster_id, user=None, session=None, **kwargs): - """patch cluster config.""" - cluster = _get_cluster(cluster_id, session=session) - return _update_cluster_config( - cluster, session=session, user=user, **kwargs - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEL_CLUSTER_CONFIG -) -@utils.wrap_to_dict(RESP_CONFIG_FIELDS) -def del_cluster_config(cluster_id, user=None, session=None): - """Delete a cluster config.""" - cluster = _get_cluster( - cluster_id, session=session - ) - check_cluster_editable(cluster, user=user) - return utils.update_db_object( - session, cluster, os_config={}, - package_config={}, config_validated=False - ) - - -def _roles_validates(roles, cluster, session=None, user=None): - """Check roles is validated to a cluster's roles.""" - if roles: - if not cluster.flavor_name: - raise exception.InvalidParameter( - 'not flavor in cluster %s' % cluster.name - ) - cluster_roles = [role['name'] for role in cluster.flavor['roles']] - for role in roles: - if role not in cluster_roles: - raise exception.InvalidParameter( - 'role %s is not in cluster roles %s' % ( - role, cluster_roles - ) - ) - - -def _cluster_host_roles_validates( - value, cluster, host, session=None, user=None, **kwargs -): - """Check clusterhost roles is validated by cluster and host.""" - _roles_validates(value, cluster, session=session, user=user) - - -def _clusterhost_roles_validates( - value, clusterhost, session=None, user=None, **kwargs -): - """Check clusterhost roles is validated by clusterhost.""" - _roles_validates( - value, clusterhost.cluster, session=session, user=user - ) - - -@utils.supported_filters( - optional_support_keys=UPDATED_HOST_FIELDS, - ignore_support_keys=UPDATED_CLUSTERHOST_FIELDS -) -@utils.input_validates(name=utils.check_name) -def _add_host_if_not_exist( - machine_id, cluster, session=None, user=None, **kwargs -): - """Add underlying host if it does not exist.""" - from compass.db.api import host as host_api - host = host_api.get_host_internal( - machine_id, session=session, exception_when_missing=False - ) - if host: - if kwargs: - # ignore update underlying host if host is not editable. - from compass.db.api import host as host_api - if host_api.is_host_editable( - host, user=cluster.creator, - check_in_installing=kwargs.get('reinstall_os', False), - ): - utils.update_db_object( - session, host, - **kwargs - ) - else: - logging.debug( - 'ignore update host host %s ' - 'since it is not editable' % host.name - ) - else: - logging.debug('nothing to update for host %s', host.name) - else: - from compass.db.api import adapter_holder as adapter_api - adapter = adapter_api.get_adapter( - cluster.adapter_name, user=user, session=session - ) - host = utils.add_db_object( - session, models.Host, False, machine_id, - os_name=cluster.os_name, - os_installer=adapter['os_installer'], - creator=cluster.creator, - **kwargs - ) - return host - - -@utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_FIELDS, - ignore_support_keys=UPDATED_HOST_FIELDS -) -@utils.input_validates_with_args( - roles=_cluster_host_roles_validates -) -def _add_clusterhost_only( - cluster, host, - exception_when_existing=False, - session=None, user=None, - **kwargs -): - """Get clusterhost only.""" - if not cluster.state.state == "UNINITIALIZED": - cluster.state.ready = False - cluster.state.state = "UNINITIALIZED" - cluster.state.percentage = 0.0 - utils.update_db_object(session, cluster.state, state="UNINITIALIZED") - - return utils.add_db_object( - session, models.ClusterHost, exception_when_existing, - cluster.id, host.id, **kwargs - ) - - -@utils.supported_filters( - ADDED_HOST_FIELDS, - optional_support_keys=UPDATED_HOST_FIELDS + UPDATED_CLUSTERHOST_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -def _add_clusterhost( - cluster, - exception_when_existing=False, - session=None, user=None, machine_id=None, **kwargs -): - """Add clusterhost and add underlying host if it does not exist.""" - host = _add_host_if_not_exist( - machine_id, cluster, session=session, - user=user, **kwargs - ) - - return _add_clusterhost_only( - cluster, host, exception_when_existing=exception_when_existing, - session=session, user=user, **kwargs - ) - - -def _add_clusterhosts(cluster, machines, session=None, user=None): - """Add machines to cluster. - - Args: - machines: list of dict which contains clusterost attr to update. - - Examples: - [{'machine_id': 1, 'name': 'host1'}] - """ - check_cluster_editable( - cluster, user=user, - check_in_installing=True - ) - if cluster.state.state == 'SUCCESSFUL': - cluster.state.state == 'UPDATE_PREPARING' - for machine_dict in machines: - _add_clusterhost( - cluster, session=session, user=user, **machine_dict - ) - - -def _remove_clusterhosts(cluster, hosts, session=None, user=None): - """Remove hosts from cluster. - - Args: - hosts: list of host id. - """ - check_cluster_editable( - cluster, user=user, - check_in_installing=True - ) - utils.del_db_objects( - session, models.ClusterHost, - cluster_id=cluster.id, host_id=hosts - ) - - -def _set_clusterhosts(cluster, machines, session=None, user=None): - """set machines to cluster. - - Args: - machines: list of dict which contains clusterost attr to update. - - Examples: - [{'machine_id': 1, 'name': 'host1'}] - """ - check_cluster_editable( - cluster, user=user, - check_in_installing=True - ) - utils.del_db_objects( - session, models.ClusterHost, - cluster_id=cluster.id - ) - if cluster.state.state == 'SUCCESSFUL': - cluster.state.state = 'UPDATE_PREPARING' - for machine_dict in machines: - _add_clusterhost( - cluster, True, session=session, user=user, **machine_dict - ) - - -@utils.supported_filters(optional_support_keys=SUPPORTED_CLUSTERHOST_FIELDS) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_CLUSTERHOSTS -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS) -def list_cluster_hosts(cluster_id, user=None, session=None, **filters): - """List clusterhosts of a cluster.""" - cluster = _get_cluster(cluster_id, session=session) - return utils.list_db_objects( - session, models.ClusterHost, cluster_id=cluster.id, - **filters - ) - - -@utils.supported_filters(optional_support_keys=SUPPORTED_CLUSTERHOST_FIELDS) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_CLUSTERHOSTS -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS) -def list_clusterhosts(user=None, session=None, **filters): - """List all clusterhosts.""" - return utils.list_db_objects( - session, models.ClusterHost, **filters - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_CLUSTERHOSTS -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS) -def get_cluster_host( - cluster_id, host_id, exception_when_missing=True, - user=None, session=None, **kwargs -): - """Get clusterhost info by cluster id and host id.""" - return _get_cluster_host( - cluster_id, host_id, session=session, - exception_when_missing=exception_when_missing, - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_CLUSTERHOSTS -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS) -def get_clusterhost( - clusterhost_id, exception_when_missing=True, - user=None, session=None, **kwargs -): - """Get clusterhost info by clusterhost id.""" - return _get_clusterhost( - clusterhost_id, session=session, - exception_when_missing=exception_when_missing, - user=user - ) - - -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_CLUSTER_HOSTS -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS) -def add_cluster_host( - cluster_id, exception_when_existing=True, - user=None, session=None, **kwargs -): - """Add a host to a cluster.""" - cluster = _get_cluster(cluster_id, session=session) - check_cluster_editable( - cluster, user=user, - check_in_installing=True - ) - if cluster.state.state == 'SUCCESSFUL': - cluster.state.state = 'UPDATE_PREPARING' - return _add_clusterhost( - cluster, exception_when_existing, - session=session, user=user, **kwargs - ) - - -@utils.supported_filters( - optional_support_keys=UPDATED_HOST_FIELDS, - ignore_support_keys=( - UPDATED_CLUSTERHOST_FIELDS + - PATCHED_CLUSTERHOST_FIELDS - ) -) -def _update_host_if_necessary( - clusterhost, session=None, user=None, **kwargs -): - """Update underlying host if there is something to update.""" - host = clusterhost.host - if kwargs: - # ignore update underlying host if the host is not editable. - from compass.db.api import host as host_api - if host_api.is_host_editable( - host, user=clusterhost.cluster.creator, - check_in_installing=kwargs.get('reinstall_os', False), - ): - utils.update_db_object( - session, host, - **kwargs - ) - else: - logging.debug( - 'ignore update host %s since it is not editable' % host.name - ) - else: - logging.debug( - 'nothing to update for host %s', host.name - ) - return host - - -@utils.supported_filters( - optional_support_keys=( - UPDATED_CLUSTERHOST_FIELDS + - PATCHED_CLUSTERHOST_FIELDS - ), - ignore_support_keys=UPDATED_HOST_FIELDS -) -@utils.input_validates_with_args( - roles=_clusterhost_roles_validates, - patched_roles=_clusterhost_roles_validates -) -def _update_clusterhost_only( - clusterhost, session=None, user=None, **kwargs -): - """Update clusterhost only.""" - check_cluster_editable(clusterhost.cluster, user=user) - return utils.update_db_object( - session, clusterhost, **kwargs - ) - - -@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS) -def _update_clusterhost(clusterhost, session=None, user=None, **kwargs): - """Update clusterhost and underlying host if necessary.""" - _update_host_if_necessary( - clusterhost, session=session, user=user, **kwargs - ) - return _update_clusterhost_only( - clusterhost, session=session, user=user, **kwargs - ) - - -@utils.supported_filters( - optional_support_keys=(UPDATED_HOST_FIELDS + UPDATED_CLUSTERHOST_FIELDS), - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_CLUSTER_HOSTS -) -def update_cluster_host( - cluster_id, host_id, user=None, - session=None, **kwargs -): - """Update clusterhost by cluster id and host id.""" - logging.info('updating kwargs: %s', kwargs) - clusterhost = _get_cluster_host( - cluster_id, host_id, session=session - ) - return _update_clusterhost( - clusterhost, session=session, user=user, **kwargs - ) - - -@utils.supported_filters( - optional_support_keys=(UPDATED_HOST_FIELDS + UPDATED_CLUSTERHOST_FIELDS), - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_CLUSTER_HOSTS -) -def update_clusterhost( - clusterhost_id, user=None, - session=None, **kwargs -): - """Update clusterhost by clusterhost id.""" - clusterhost = _get_clusterhost( - clusterhost_id, session=session - ) - return _update_clusterhost( - clusterhost, session=session, user=user, **kwargs - ) - - -# replace roles to patched_roles in kwargs. -# It tells db roles field will be patched. -@utils.replace_filters( - roles='patched_roles' -) -@utils.supported_filters( - optional_support_keys=PATCHED_CLUSTERHOST_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_CLUSTER_HOSTS -) -def patch_cluster_host( - cluster_id, host_id, user=None, - session=None, **kwargs -): - """Patch clusterhost by cluster id and host id.""" - logging.info("kwargs are %s", kwargs) - clusterhost = _get_cluster_host( - cluster_id, host_id, session=session - ) - updated_clusterhost = _update_clusterhost( - clusterhost, session=session, user=user, **kwargs - ) - return updated_clusterhost - - -# replace roles to patched_roles in kwargs. -# It tells db roles field will be patched. -@utils.replace_filters( - roles='patched_roles' -) -@utils.supported_filters( - optional_support_keys=PATCHED_CLUSTERHOST_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_CLUSTER_HOSTS -) -def patch_clusterhost( - clusterhost_id, user=None, session=None, - **kwargs -): - """Patch clusterhost by clusterhost id.""" - clusterhost = _get_clusterhost( - clusterhost_id, session=session - ) - return _update_clusterhost( - clusterhost, session=session, user=user, **kwargs - ) - - -@user_api.check_user_permission( - permission.PERMISSION_DEL_CLUSTER_HOST -) -@utils.wrap_to_dict( - RESP_CLUSTERHOST_FIELDS + ['status', 'host'], - host=RESP_CLUSTERHOST_FIELDS -) -def _del_cluster_host( - clusterhost, - force=False, from_database_only=False, - delete_underlying_host=False, user=None, - session=None, **kwargs -): - """delete clusterhost. - - If force, the cluster host will be deleted anyway. - If from_database_only, the cluster host recored will only be - deleted from database. Otherwise a celery task sent to do - clean deletion. - If delete_underlying_host, the underlying host will also be deleted. - The backend will call _del_cluster_host again when the clusterhost is - deleted from os installer/package installer with from_database_only - set. - """ - # force set clusterhost state to ERROR when we want to delete the - # clusterhost anyway even the clusterhost is in installing or already - # installed. It let the api know the deleting is in doing when backend - # is doing the real deleting. In future we may import a new state like - # INDELETE to indicate the deleting is processing. - # We need discuss about if we can delete a clusterhost when it is already - # installed by api. - if clusterhost.state.state != 'UNINITIALIZED' and force: - clusterhost.state.state = 'ERROR' - if not force: - check_cluster_editable( - clusterhost.cluster, user=user, - check_in_installing=True - ) - # delete underlying host if delete_underlying_host is set. - if delete_underlying_host: - host = clusterhost.host - if host.state.state != 'UNINITIALIZED' and force: - host.state.state = 'ERROR' - # only delete the host when user have the permission to delete it. - import compass.db.api.host as host_api - if host_api.is_host_editable( - host, user=user, - check_in_installing=True - ): - # if there is no need to do the deletion by backend or - # from_database_only is set, we only delete the record - # in database. - if host.state.state == 'UNINITIALIZED' or from_database_only: - utils.del_db_object( - session, host - ) - - # if there is no need to do the deletion by backend or - # from_database_only is set, we only delete the record in database. - if clusterhost.state.state == 'UNINITIALIZED' or from_database_only: - return utils.del_db_object( - session, clusterhost - ) - else: - logging.info( - 'send del cluster %s host %s task to celery', - clusterhost.cluster_id, clusterhost.host_id - ) - from compass.tasks import client as celery_client - celery_client.celery.send_task( - 'compass.tasks.delete_cluster_host', - ( - user.email, clusterhost.cluster_id, clusterhost.host_id, - delete_underlying_host - ), - queue=user.email, - exchange=user.email, - routing_key=user.email - ) - return { - 'status': 'delete action sent', - 'host': clusterhost, - } - - -@utils.supported_filters([]) -@database.run_in_session() -def del_cluster_host( - cluster_id, host_id, - force=False, from_database_only=False, - delete_underlying_host=False, user=None, - session=None, **kwargs -): - """Delete clusterhost by cluster id and host id.""" - clusterhost = _get_cluster_host( - cluster_id, host_id, session=session - ) - return _del_cluster_host( - clusterhost, force=force, from_database_only=from_database_only, - delete_underlying_host=delete_underlying_host, user=user, - session=session, **kwargs - ) - - -@utils.supported_filters([]) -@database.run_in_session() -def del_clusterhost( - clusterhost_id, - force=False, from_database_only=False, - delete_underlying_host=False, user=None, - session=None, **kwargs -): - """Delete clusterhost by clusterhost id.""" - clusterhost = _get_clusterhost( - clusterhost_id, session=session - ) - return _del_cluster_host( - clusterhost, force=force, from_database_only=from_database_only, - delete_underlying_host=delete_underlying_host, user=user, - session=session, **kwargs - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_CLUSTERHOST_CONFIG -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS) -def get_cluster_host_config( - cluster_id, host_id, user=None, - session=None, **kwargs -): - """Get clusterhost config by cluster id and host id.""" - return _get_cluster_host( - cluster_id, host_id, session=session - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_CLUSTERHOST_CONFIG -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS) -def get_cluster_host_deployed_config( - cluster_id, host_id, user=None, session=None, **kwargs -): - """Get clusterhost deployed config by cluster id and host id.""" - return _get_cluster_host( - cluster_id, host_id, session=session - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_CLUSTERHOST_CONFIG -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS) -def get_clusterhost_config(clusterhost_id, user=None, session=None, **kwargs): - """Get clusterhost config by clusterhost id.""" - return _get_clusterhost( - clusterhost_id, session=session - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_CLUSTERHOST_CONFIG -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS) -def get_clusterhost_deployed_config( - clusterhost_id, user=None, - session=None, **kwargs -): - """Get clusterhost deployed config by clusterhost id.""" - return _get_clusterhost( - clusterhost_id, session=session - ) - - -def _clusterhost_os_config_validates( - config, clusterhost, session=None, user=None, **kwargs -): - """Validate clusterhost's underlying host os config.""" - from compass.db.api import host as host_api - host = clusterhost.host - host_api.check_host_editable(host, user=user) - metadata_api.validate_os_config( - config, host.os_id - ) - - -def _clusterhost_package_config_validates( - config, clusterhost, session=None, user=None, **kwargs -): - """Validate clusterhost's cluster package config.""" - cluster = clusterhost.cluster - check_cluster_editable(cluster, user=user) - metadata_api.validate_flavor_config( - config, cluster.flavor_id - ) - - -def _filter_clusterhost_host_editable( - config, clusterhost, session=None, user=None, **kwargs -): - """Filter fields if the underlying host is not editable.""" - from compass.db.api import host as host_api - host = clusterhost.host - return host_api.is_host_editable(host, user=user) - - -@utils.input_filters( - put_os_config=_filter_clusterhost_host_editable, - patched_os_config=_filter_clusterhost_host_editable -) -@utils.input_validates_with_args( - put_os_config=_clusterhost_os_config_validates, - put_package_config=_clusterhost_package_config_validates -) -@utils.output_validates_with_args( - os_config=_clusterhost_os_config_validates, - package_config=_clusterhost_package_config_validates -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS) -def _update_clusterhost_config(clusterhost, session=None, user=None, **kwargs): - """Update clusterhost config.""" - return utils.update_db_object( - session, clusterhost, **kwargs - ) - - -def _clusterhost_host_validated( - config, clusterhost, session=None, user=None, **kwargs -): - """Check clusterhost's underlying host is validated.""" - from compass.db.api import host as host_api - host = clusterhost.host - host_api.check_host_editable(host, user=user) - host_api.check_host_validated(host) - - -def _clusterhost_cluster_validated( - config, clusterhost, session=None, user=None, **kwargs -): - """Check clusterhost's cluster is validated.""" - cluster = clusterhost.cluster - check_cluster_editable(cluster, user=user) - check_clusterhost_validated(clusterhost) - - -@utils.input_filters( - deployed_os_config=_filter_clusterhost_host_editable, -) -@utils.input_validates_with_args( - deployed_os_config=_clusterhost_host_validated, - deployed_package_config=_clusterhost_cluster_validated -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS) -def _update_clusterhost_deployed_config( - clusterhost, session=None, user=None, **kwargs -): - """Update clusterhost deployed config.""" - return utils.update_db_object( - session, clusterhost, **kwargs - ) - - -# replace os_config to put_os_config and -# package_config to put_package_config in kwargs. -# It tells db these fields will be updated not patched. -@utils.replace_filters( - os_config='put_os_config', - package_config='put_package_config' -) -@utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_CONFIG_FIELDS, -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_CLUSTERHOST_CONFIG -) -def update_cluster_host_config( - cluster_id, host_id, user=None, session=None, **kwargs -): - """Update clusterhost config by cluster id and host id.""" - clusterhost = _get_cluster_host( - cluster_id, host_id, session=session - ) - return _update_clusterhost_config( - clusterhost, user=user, session=session, **kwargs - ) - - -# replace os_config to deployed_os_config and -# package_config to deployed_package_config in kwargs. -@utils.replace_filters( - os_config='deployed_os_config', - package_config='deployed_package_config' -) -@utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_CLUSTERHOST_CONFIG -) -def update_cluster_host_deployed_config( - cluster_id, host_id, user=None, session=None, **kwargs -): - """Update clusterhost deployed config by cluster id and host id.""" - clusterhost = _get_cluster_host( - cluster_id, host_id, session=session - ) - return _update_clusterhost_deployed_config( - clusterhost, session=session, user=user, **kwargs - ) - - -# replace os_config to put_os_config and -# package_config to put_package_config in kwargs. -# It tells db these fields will be updated not patched. -@utils.replace_filters( - os_config='put_os_config', - package_config='put_package_config' -) -@utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_CONFIG_FIELDS, -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_CLUSTERHOST_CONFIG -) -def update_clusterhost_config( - clusterhost_id, user=None, session=None, **kwargs -): - """Update clusterhost config by clusterhost id.""" - clusterhost = _get_clusterhost( - clusterhost_id, session=session - ) - return _update_clusterhost_config( - clusterhost, session=session, user=user, **kwargs - ) - - -# replace os_config to deployed_os_config and -# package_config to deployed_package_config in kwargs. -@utils.replace_filters( - os_config='deployed_os_config', - package_config='deployed_package_config' -) -@utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_CLUSTERHOST_CONFIG -) -def update_clusterhost_deployed_config( - clusterhost_id, user=None, session=None, **kwargs -): - """Update clusterhost deployed config by clusterhost id.""" - clusterhost = _get_clusterhost( - clusterhost_id, session=session - ) - return _update_clusterhost_deployed_config( - clusterhost, session=session, user=user, **kwargs - ) - - -# replace os_config to patched_os_config and -# package_config to patched_package_config in kwargs -# It tells db these fields will be patched not updated. -@utils.replace_filters( - os_config='patched_os_config', - package_config='patched_package_config' -) -@utils.supported_filters( - optional_support_keys=PATCHED_CLUSTERHOST_CONFIG_FIELDS, -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_CLUSTERHOST_CONFIG -) -def patch_cluster_host_config( - cluster_id, host_id, user=None, session=None, **kwargs -): - """patch clusterhost config by cluster id and host id.""" - clusterhost = _get_cluster_host( - cluster_id, host_id, session=session - ) - return _update_clusterhost_config( - clusterhost, session=session, user=user, **kwargs - ) - - -# replace os_config to patched_os_config and -# package_config to patched_package_config in kwargs -# It tells db these fields will be patched not updated. -@utils.replace_filters( - os_config='patched_os_config', - package_config='patched_package_config' -) -@utils.supported_filters( - optional_support_keys=PATCHED_CLUSTERHOST_CONFIG_FIELDS, -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_CLUSTERHOST_CONFIG -) -def patch_clusterhost_config( - clusterhost_id, user=None, session=None, **kwargs -): - """patch clusterhost config by clusterhost id.""" - clusterhost = _get_clusterhost( - clusterhost_id, session=session - ) - return _update_clusterhost_config( - clusterhost, session=session, user=user, **kwargs - ) - - -def _clusterhost_host_editable( - config, clusterhost, session=None, user=None, **kwargs -): - """Check clusterhost underlying host is editable.""" - from compass.db.api import host as host_api - host_api.check_host_editable(clusterhost.host, user=user) - - -def _clusterhost_cluster_editable( - config, clusterhost, session=None, user=None, **kwargs -): - """Check clusterhost's cluster is editable.""" - check_cluster_editable(clusterhost.cluster, user=user) - - -@utils.supported_filters( - optional_support_keys=['os_config', 'package_config'] -) -@utils.input_filters( - os_config=_filter_clusterhost_host_editable, -) -@utils.output_validates_with_args( - package_config=_clusterhost_cluster_editable -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS) -def _delete_clusterhost_config( - clusterhost, session=None, user=None, **kwargs -): - """delete clusterhost config.""" - return utils.update_db_object( - session, clusterhost, config_validated=False, - **kwargs - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEL_CLUSTERHOST_CONFIG -) -def delete_cluster_host_config( - cluster_id, host_id, user=None, session=None -): - """Delete a clusterhost config by cluster id and host id.""" - clusterhost = _get_cluster_host( - cluster_id, host_id, session=session - ) - return _delete_clusterhost_config( - clusterhost, session=session, user=user, - os_config={}, package_config={} - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEL_CLUSTERHOST_CONFIG -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS) -def delete_clusterhost_config(clusterhost_id, user=None, session=None): - """Delet a clusterhost config by clusterhost id.""" - clusterhost = _get_clusterhost( - clusterhost_id, session=session - ) - return _delete_clusterhost_config( - clusterhost, session=session, user=user, - os_config={}, package_config={} - ) - - -@utils.supported_filters( - optional_support_keys=['add_hosts', 'remove_hosts', 'set_hosts'] -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_CLUSTER_HOSTS -) -@utils.wrap_to_dict( - ['hosts'], - hosts=RESP_CLUSTERHOST_FIELDS -) -def update_cluster_hosts( - cluster_id, add_hosts={}, set_hosts=None, - remove_hosts={}, user=None, session=None -): - """Update cluster hosts.""" - cluster = _get_cluster(cluster_id, session=session) - if remove_hosts: - _remove_clusterhosts( - cluster, session=session, user=user, **remove_hosts - ) - if add_hosts: - _add_clusterhosts( - cluster, session=session, user=user, **add_hosts - ) - if set_hosts is not None: - _set_clusterhosts( - cluster, session=session, user=user, **set_hosts - ) - - return { - 'hosts': list_cluster_hosts(cluster_id, session=session) - } - - -def validate_clusterhost(clusterhost, session=None): - """validate clusterhost.""" - roles = clusterhost.roles - if not roles: - if clusterhost.cluster.flavor_name: - raise exception.InvalidParameter( - 'empty roles for clusterhost %s' % clusterhost.name - ) - - -def validate_cluster(cluster, session=None): - """Validate cluster.""" - if not cluster.clusterhosts: - raise exception.InvalidParameter( - 'cluster %s does not have any hosts' % cluster.name - ) - if cluster.flavor_name: - cluster_roles = cluster.flavor['roles'] - else: - cluster_roles = [] - necessary_roles = set([ - role['name'] for role in cluster_roles if not role.get('optional') - ]) - clusterhost_roles = set([]) - interface_subnets = {} - for clusterhost in cluster.clusterhosts: - roles = clusterhost.roles - for role in roles: - clusterhost_roles.add(role['name']) - host = clusterhost.host - for host_network in host.host_networks: - interface_subnets.setdefault( - host_network.interface, set([]) - ).add(host_network.subnet.subnet) - missing_roles = necessary_roles - clusterhost_roles - if missing_roles: - raise exception.InvalidParameter( - 'cluster %s have some roles %s not assigned to any host' % ( - cluster.name, list(missing_roles) - ) - ) - for interface, subnets in interface_subnets.items(): - if len(subnets) > 1: - raise exception.InvalidParameter( - 'cluster %s multi subnets %s in interface %s' % ( - cluster.name, list(subnets), interface - ) - ) - - -@utils.supported_filters(optional_support_keys=['review']) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_REVIEW_CLUSTER -) -@utils.wrap_to_dict( - RESP_REVIEW_FIELDS, - cluster=RESP_CONFIG_FIELDS, - hosts=RESP_CLUSTERHOST_CONFIG_FIELDS -) -def review_cluster(cluster_id, review={}, user=None, session=None, **kwargs): - """review cluster. - - Args: - cluster_id: the cluster id. - review: dict contains hosts to be reviewed. either contains key - hosts or clusterhosts. where hosts is a list of host id, - clusterhosts is a list of clusterhost id. - """ - from compass.db.api import host as host_api - cluster = _get_cluster(cluster_id, session=session) - check_cluster_editable(cluster, user=user) - host_ids = review.get('hosts', []) - clusterhost_ids = review.get('clusterhosts', []) - clusterhosts = [] - # Get clusterhosts need to be reviewed. - for clusterhost in cluster.clusterhosts: - if ( - clusterhost.clusterhost_id in clusterhost_ids or - clusterhost.host_id in host_ids - ): - clusterhosts.append(clusterhost) - - os_config = copy.deepcopy(cluster.os_config) - os_config = metadata_api.autofill_os_config( - os_config, cluster.os_id, cluster=cluster - ) - metadata_api.validate_os_config( - os_config, cluster.os_id, True - ) - for clusterhost in clusterhosts: - host = clusterhost.host - # ignore underlying host os config validation - # since the host is not editable - if not host_api.is_host_editable( - host, user=user, check_in_installing=False - ): - logging.info( - 'ignore update host %s config ' - 'since it is not editable' % host.name - ) - continue - host_os_config = copy.deepcopy(host.os_config) - host_os_config = metadata_api.autofill_os_config( - host_os_config, host.os_id, - host=host - ) - deployed_os_config = util.merge_dict( - os_config, host_os_config - ) - metadata_api.validate_os_config( - deployed_os_config, host.os_id, True - ) - host_api.validate_host(host) - utils.update_db_object( - session, host, os_config=host_os_config, config_validated=True - ) - - package_config = copy.deepcopy(cluster.package_config) - if cluster.flavor_name: - package_config = metadata_api.autofill_flavor_config( - package_config, cluster.flavor_id, - cluster=cluster - ) - metadata_api.validate_flavor_config( - package_config, cluster.flavor_id, True - ) - for clusterhost in clusterhosts: - clusterhost_package_config = copy.deepcopy( - clusterhost.package_config - ) - clusterhost_package_config = ( - metadata_api.autofill_flavor_config( - clusterhost_package_config, - cluster.flavor_id, - clusterhost=clusterhost - ) - ) - deployed_package_config = util.merge_dict( - package_config, clusterhost_package_config - ) - metadata_api.validate_flavor_config( - deployed_package_config, - cluster.flavor_id, True - ) - validate_clusterhost(clusterhost, session=session) - utils.update_db_object( - session, clusterhost, - package_config=clusterhost_package_config, - config_validated=True - ) - - validate_cluster(cluster, session=session) - utils.update_db_object( - session, cluster, os_config=os_config, package_config=package_config, - config_validated=True - ) - return { - 'cluster': cluster, - 'hosts': clusterhosts - } - - -@utils.supported_filters(optional_support_keys=['deploy']) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEPLOY_CLUSTER -) -@utils.wrap_to_dict( - RESP_DEPLOY_FIELDS, - cluster=RESP_CONFIG_FIELDS, - hosts=RESP_CLUSTERHOST_FIELDS -) -def deploy_cluster( - cluster_id, deploy={}, user=None, session=None, **kwargs -): - """deploy cluster. - - Args: - cluster_id: cluster id. - deploy: dict contains key either hosts or clusterhosts. - deploy['hosts'] is a list of host id, - deploy['clusterhosts'] is a list of clusterhost id. - """ - from compass.db.api import host as host_api - from compass.tasks import client as celery_client - cluster = _get_cluster(cluster_id, session=session) - host_ids = deploy.get('hosts', []) - clusterhost_ids = deploy.get('clusterhosts', []) - clusterhosts = [] - # get clusterhost to deploy. - for clusterhost in cluster.clusterhosts: - if ( - clusterhost.clusterhost_id in clusterhost_ids or - clusterhost.host_id in host_ids - ): - clusterhosts.append(clusterhost) - check_cluster_editable(cluster, user=user) - check_cluster_validated(cluster) - utils.update_db_object(session, cluster.state, state='INITIALIZED') - for clusterhost in clusterhosts: - host = clusterhost.host - # ignore checking if underlying host is validated if - # the host is not editable. - if host_api.is_host_editable(host, user=user): - host_api.check_host_validated(host) - utils.update_db_object(session, host.state, state='INITIALIZED') - if cluster.flavor_name: - check_clusterhost_validated(clusterhost) - utils.update_db_object( - session, clusterhost.state, state='INITIALIZED' - ) - - celery_client.celery.send_task( - 'compass.tasks.deploy_cluster', - ( - user.email, cluster_id, - [clusterhost.host_id for clusterhost in clusterhosts] - ), - queue=user.email, - exchange=user.email, - routing_key=user.email - ) - return { - 'status': 'deploy action sent', - 'cluster': cluster, - 'hosts': clusterhosts - } - - -@utils.supported_filters(optional_support_keys=['redeploy']) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEPLOY_CLUSTER -) -@utils.wrap_to_dict( - RESP_DEPLOY_FIELDS, - cluster=RESP_CONFIG_FIELDS, - hosts=RESP_CLUSTERHOST_FIELDS -) -def redeploy_cluster( - cluster_id, deploy={}, user=None, session=None, **kwargs -): - """redeploy cluster. - - Args: - cluster_id: cluster id. - """ - from compass.db.api import host as host_api - from compass.tasks import client as celery_client - cluster = _get_cluster(cluster_id, session=session) - - check_cluster_editable(cluster, user=user) - check_cluster_validated(cluster) - utils.update_db_object( - session, cluster.state, - state='INITIALIZED', - percentage=0, - ready=False - ) - for clusterhost in cluster.clusterhosts: - host = clusterhost.host - # ignore checking if underlying host is validated if - # the host is not editable. - host_api.check_host_validated(host) - utils.update_db_object( - session, host.state, - state='INITIALIZED', - percentage=0, - ready=False - ) - if cluster.flavor_name: - check_clusterhost_validated(clusterhost) - utils.update_db_object( - session, - clusterhost.state, - state='INITIALIZED', - percentage=0, - ready=False - ) - - celery_client.celery.send_task( - 'compass.tasks.redeploy_cluster', - ( - user.email, cluster_id - ), - queue=user.email, - exchange=user.email, - routing_key=user.email - ) - return { - 'status': 'redeploy action sent', - 'cluster': cluster - } - - -@utils.supported_filters(optional_support_keys=['apply_patch']) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEPLOY_CLUSTER -) -@utils.wrap_to_dict( - RESP_DEPLOY_FIELDS, - cluster=RESP_CONFIG_FIELDS, - hosts=RESP_CLUSTERHOST_FIELDS -) -def patch_cluster(cluster_id, user=None, session=None, **kwargs): - - from compass.tasks import client as celery_client - - cluster = _get_cluster(cluster_id, session=session) - celery_client.celery.send_task( - 'compass.tasks.patch_cluster', - ( - user.email, cluster_id, - ), - queue=user.email, - exchange=user.email, - routing_key=user.email - ) - return { - 'status': 'patch action sent', - 'cluster': cluster - } - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_GET_CLUSTER_STATE -) -@utils.wrap_to_dict(RESP_STATE_FIELDS) -def get_cluster_state(cluster_id, user=None, session=None, **kwargs): - """Get cluster state info.""" - return _get_cluster(cluster_id, session=session).state_dict() - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_GET_CLUSTERHOST_STATE -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS) -def get_cluster_host_state( - cluster_id, host_id, user=None, session=None, **kwargs -): - """Get clusterhost state merged with underlying host state.""" - return _get_cluster_host( - cluster_id, host_id, session=session - ).state_dict() - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_GET_CLUSTERHOST_STATE -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS) -def get_cluster_host_self_state( - cluster_id, host_id, user=None, session=None, **kwargs -): - """Get clusterhost itself state.""" - return _get_cluster_host( - cluster_id, host_id, session=session - ).state - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_GET_CLUSTERHOST_STATE -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS) -def get_clusterhost_state( - clusterhost_id, user=None, session=None, **kwargs -): - """Get clusterhost state merged with underlying host state.""" - return _get_clusterhost( - clusterhost_id, session=session - ).state_dict() - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_GET_CLUSTERHOST_STATE -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS) -def get_clusterhost_self_state( - clusterhost_id, user=None, session=None, **kwargs -): - """Get clusterhost itself state.""" - return _get_clusterhost( - clusterhost_id, session=session - ).state - - -@utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_STATE_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_CLUSTERHOST_STATE -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS) -def update_cluster_host_state( - cluster_id, host_id, user=None, session=None, **kwargs -): - """Update a clusterhost itself state.""" - clusterhost = _get_cluster_host( - cluster_id, host_id, session=session - ) - # Modify(harry): without progress_update.py to update cluster state - # update cluster state here - cluster = _get_cluster(clusterhost.cluster_id, session=session) - utils.update_db_object(session, clusterhost.state, **kwargs) - utils.update_db_object(session, cluster.state, **kwargs) - return clusterhost.state_dict() - - -def _update_clusterhost_state( - clusterhost, from_database_only=False, - session=None, user=None, **kwargs -): - """Update clusterhost state. - - If from_database_only, the state will only be updated in database. - Otherwise a task sent to celery and os installer/package installer - will also update its state if needed. - """ - if 'ready' in kwargs and kwargs['ready'] and not clusterhost.state.ready: - ready_triggered = True - else: - ready_triggered = False - cluster_ready = False - host = clusterhost.host - cluster = clusterhost.cluster - host_ready = not host.state.ready - if ready_triggered: - cluster_ready = True - for clusterhost_in_cluster in cluster.clusterhosts: - if ( - clusterhost_in_cluster.clusterhost_id - == clusterhost.clusterhost_id - ): - continue - if not clusterhost_in_cluster.state.ready: - cluster_ready = False - - logging.info( - 'clusterhost %s ready: %s', - clusterhost.name, ready_triggered - ) - logging.info('cluster ready: %s', cluster_ready) - logging.info('host ready: %s', host_ready) - if not ready_triggered or from_database_only: - logging.info('%s state is set to %s', clusterhost.name, kwargs) - utils.update_db_object(session, clusterhost.state, **kwargs) - if not clusterhost.state.ready: - logging.info('%s state ready is set to False', cluster.name) - utils.update_db_object(session, cluster.state, ready=False) - status = '%s state is updated' % clusterhost.name - else: - if not user: - user_id = cluster.creator_id - user_dict = user_api.get_user(user_id, session=session) - user_email = user_dict['email'] - else: - user_email = user.email - from compass.tasks import client as celery_client - celery_client.celery.send_task( - 'compass.tasks.package_installed', - ( - clusterhost.cluster_id, clusterhost.host_id, - cluster_ready, host_ready - ), - queue=user_email, - exchange=user_email, - routing_key=user_email - ) - status = '%s: cluster ready %s host ready %s' % ( - clusterhost.name, cluster_ready, host_ready - ) - logging.info('action status: %s', status) - return { - 'status': status, - 'clusterhost': clusterhost.state_dict() - } - - -@util.deprecated -@utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_STATE_INTERNAL_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_CLUSTERHOST_STATE -) -@utils.wrap_to_dict(['status', 'clusterhost']) -def update_cluster_host_state_internal( - cluster_id, host_id, from_database_only=False, - user=None, session=None, **kwargs -): - """Update a clusterhost state by installation process.""" - # TODO(xicheng): it should be merged into update_cluster_host_state - clusterhost = _get_cluster_host( - cluster_id, host_id, session=session - ) - return _update_clusterhost_state( - clusterhost, from_database_only=from_database_only, - session=session, users=user, **kwargs - ) - - -@utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_STATE_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_CLUSTERHOST_STATE -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_STATE_FIELDS) -def update_clusterhost_state( - clusterhost_id, user=None, session=None, **kwargs -): - """Update a clusterhost itself state.""" - clusterhost = _get_clusterhost( - clusterhost_id, session=session - ) - # Modify(harry): without progress_update.py to update cluster state - # update cluster state here - cluster = _get_cluster(clusterhost.cluster_id, session=session) - utils.update_db_object(session, clusterhost.state, **kwargs) - utils.update_db_object(session, cluster.state, **kwargs) - return clusterhost.state_dict() - - -@util.deprecated -@utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_STATE_INTERNAL_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_CLUSTERHOST_STATE -) -@utils.wrap_to_dict(['status', 'clusterhost']) -def update_clusterhost_state_internal( - clusterhost_id, from_database_only=False, - user=None, session=None, **kwargs -): - """Update a clusterhost state by installation process.""" - # TODO(xicheng): it should be merged into update_clusterhost_state - clusterhost = _get_clusterhost(clusterhost_id, session=session) - return _update_clusterhost_state( - clusterhost, from_database_only=from_database_only, - session=session, user=user, **kwargs - ) - - -@utils.supported_filters( - optional_support_keys=UPDATED_CLUSTER_STATE_FIELDS, - ignore_support_keys=(IGNORE_FIELDS + IGNORE_UPDATED_CLUSTER_STATE_FIELDS) -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_CLUSTER_STATE -) -@utils.wrap_to_dict(RESP_STATE_FIELDS) -def update_cluster_state( - cluster_id, user=None, session=None, **kwargs -): - """Update a cluster state.""" - cluster = _get_cluster( - cluster_id, session=session - ) - utils.update_db_object(session, cluster.state, **kwargs) - return cluster.state_dict() - - -@util.deprecated -@utils.supported_filters( - optional_support_keys=UPDATED_CLUSTER_STATE_INTERNAL_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_CLUSTER_STATE -) -@utils.wrap_to_dict(['status', 'cluster']) -def update_cluster_state_internal( - cluster_id, from_database_only=False, - user=None, session=None, **kwargs -): - """Update a cluster state by installation process. - - If from_database_only, the state will only be updated in database. - Otherwise a task sent to do state update in os installer and - package installer. - """ - # TODO(xicheng): it should be merged into update_cluster_state - cluster = _get_cluster(cluster_id, session=session) - if 'ready' in kwargs and kwargs['ready'] and not cluster.state.ready: - ready_triggered = True - else: - ready_triggered = False - clusterhost_ready = {} - if ready_triggered: - for clusterhost in cluster.clusterhosts: - clusterhost_ready[clusterhost.host_id] = ( - not clusterhost.state.ready - ) - - logging.info('cluster %s ready: %s', cluster_id, ready_triggered) - logging.info('clusterhost ready: %s', clusterhost_ready) - - if not ready_triggered or from_database_only: - logging.info('%s state is set to %s', cluster.name, kwargs) - utils.update_db_object(session, cluster.state, **kwargs) - if not cluster.state.ready: - for clusterhost in cluster.clusterhosts: - logging.info('%s state ready is to False', clusterhost.name) - utils.update_db_object( - session, clusterhost.state, ready=False - ) - status = '%s state is updated' % cluster.name - else: - if not user: - user_id = cluster.creator_id - user_dict = user_api.get_user(user_id, session=session) - user_email = user_dict['email'] - else: - user_email = user.email - from compass.tasks import client as celery_client - celery_client.celery.send_task( - 'compass.tasks.cluster_installed', - (clusterhost.cluster_id, clusterhost_ready), - queue=user_email, - exchange=user_email, - routing_key=user_email - ) - status = '%s installed action set clusterhost ready %s' % ( - cluster.name, clusterhost_ready - ) - logging.info('action status: %s', status) - return { - 'status': status, - 'cluster': cluster.state_dict() - } - - -@utils.supported_filters([]) -@database.run_in_session() -@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) -def get_cluster_host_log_histories( - cluster_id, host_id, user=None, session=None, **kwargs -): - """Get clusterhost log history by cluster id and host id.""" - return _get_cluster_host( - cluster_id, host_id, session=session - ).log_histories - - -@utils.supported_filters([]) -@database.run_in_session() -@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) -def get_clusterhost_log_histories( - clusterhost_id, user=None, - session=None, **kwargs -): - """Get clusterhost log history by clusterhost id.""" - return _get_clusterhost( - clusterhost_id, session=session - ).log_histories - - -def _get_cluster_host_log_history( - cluster_id, host_id, filename, session=None, **kwargs -): - """Get clusterhost log history by cluster id, host id and filename.""" - clusterhost = _get_cluster_host(cluster_id, host_id, session=session) - return utils.get_db_object( - session, models.ClusterHostLogHistory, - clusterhost_id=clusterhost.clusterhost_id, filename=filename, - **kwargs - ) - - -def _get_clusterhost_log_history( - clusterhost_id, filename, session=None, **kwargs -): - """Get clusterhost log history by clusterhost id and filename.""" - clusterhost = _get_clusterhost(clusterhost_id, session=session) - return utils.get_db_object( - session, models.ClusterHostLogHistory, - clusterhost_id=clusterhost.clusterhost_id, filename=filename, - **kwargs - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) -def get_cluster_host_log_history( - cluster_id, host_id, filename, user=None, session=None, **kwargs -): - """Get clusterhost log history by cluster id, host id and filename.""" - return _get_cluster_host_log_history( - cluster_id, host_id, filename, session=session - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) -def get_clusterhost_log_history( - clusterhost_id, filename, user=None, session=None, **kwargs -): - """Get host log history by clusterhost id and filename.""" - return _get_clusterhost_log_history( - clusterhost_id, filename, session=session - ) - - -@utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) -def update_cluster_host_log_history( - cluster_id, host_id, filename, user=None, session=None, **kwargs -): - """Update a host log history by cluster id, host id and filename.""" - cluster_host_log_history = _get_cluster_host_log_history( - cluster_id, host_id, filename, session=session - ) - return utils.update_db_object( - session, cluster_host_log_history, **kwargs - ) - - -@utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) -def update_clusterhost_log_history( - clusterhost_id, filename, user=None, session=None, **kwargs -): - """Update a host log history by clusterhost id and filename.""" - clusterhost_log_history = _get_clusterhost_log_history( - clusterhost_id, filename, session=session - ) - return utils.update_db_object(session, clusterhost_log_history, **kwargs) - - -@utils.supported_filters( - ADDED_CLUSTERHOST_LOG_FIELDS, - optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) -def add_clusterhost_log_history( - clusterhost_id, exception_when_existing=False, - filename=None, user=None, session=None, **kwargs -): - """add a host log history by clusterhost id and filename.""" - clusterhost = _get_clusterhost(clusterhost_id, session=session) - return utils.add_db_object( - session, models.ClusterHostLogHistory, - exception_when_existing, - clusterhost.clusterhost_id, filename, **kwargs - ) - - -@utils.supported_filters( - ADDED_CLUSTERHOST_LOG_FIELDS, - optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) -def add_cluster_host_log_history( - cluster_id, host_id, exception_when_existing=False, - filename=None, user=None, session=None, **kwargs -): - """add a host log history by cluster id, host id and filename.""" - clusterhost = _get_cluster_host( - cluster_id, host_id, session=session - ) - return utils.add_db_object( - session, models.ClusterHostLogHistory, exception_when_existing, - clusterhost.clusterhost_id, filename, **kwargs - ) diff --git a/compass-tasks/db/api/database.py b/compass-tasks/db/api/database.py deleted file mode 100644 index 49769d7..0000000 --- a/compass-tasks/db/api/database.py +++ /dev/null @@ -1,264 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Provider interface to manipulate database.""" -import functools -import logging -import netaddr - -from contextlib import contextmanager -from sqlalchemy import create_engine -from sqlalchemy.exc import IntegrityError -from sqlalchemy.exc import OperationalError -from sqlalchemy.orm import scoped_session -from sqlalchemy.orm import sessionmaker -from sqlalchemy.pool import NullPool -from sqlalchemy.pool import QueuePool -from sqlalchemy.pool import SingletonThreadPool -from sqlalchemy.pool import StaticPool -from threading import local - -from compass.db import exception -from compass.db import models -from compass.utils import logsetting -from compass.utils import setting_wrapper as setting - - -ENGINE = None -SESSION = sessionmaker(autocommit=False, autoflush=False) -SCOPED_SESSION = None -SESSION_HOLDER = local() - -POOL_MAPPING = { - 'instant': NullPool, - 'static': StaticPool, - 'queued': QueuePool, - 'thread_single': SingletonThreadPool -} - - -def init(database_url=None): - """Initialize database. - - Adjust sqlalchemy logging if necessary. - - :param database_url: string, database url. - """ - global ENGINE - global SCOPED_SESSION - if not database_url: - database_url = setting.SQLALCHEMY_DATABASE_URI - logging.info('init database %s', database_url) - root_logger = logging.getLogger() - fine_debug = root_logger.isEnabledFor(logsetting.LOGLEVEL_MAPPING['fine']) - if fine_debug: - logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) - finest_debug = root_logger.isEnabledFor( - logsetting.LOGLEVEL_MAPPING['finest'] - ) - if finest_debug: - logging.getLogger('sqlalchemy.dialects').setLevel(logging.INFO) - logging.getLogger('sqlalchemy.pool').setLevel(logging.INFO) - logging.getLogger('sqlalchemy.orm').setLevel(logging.INFO) - poolclass = POOL_MAPPING[setting.SQLALCHEMY_DATABASE_POOL_TYPE] - ENGINE = create_engine( - database_url, convert_unicode=True, - poolclass=poolclass - ) - SESSION.configure(bind=ENGINE) - SCOPED_SESSION = scoped_session(SESSION) - models.BASE.query = SCOPED_SESSION.query_property() - - -def in_session(): - """check if in database session scope.""" - bool(hasattr(SESSION_HOLDER, 'session')) - - -@contextmanager -def session(exception_when_in_session=True): - """database session scope. - - To operate database, it should be called in database session. - If not exception_when_in_session, the with session statement support - nested session and only the out most session commit/rollback the - transaction. - """ - if not ENGINE: - init() - - nested_session = False - if hasattr(SESSION_HOLDER, 'session'): - if exception_when_in_session: - logging.error('we are already in session') - raise exception.DatabaseException('session already exist') - else: - new_session = SESSION_HOLDER.session - nested_session = True - logging.log( - logsetting.getLevelByName('fine'), - 'reuse session %s', nested_session - ) - else: - new_session = SCOPED_SESSION() - setattr(SESSION_HOLDER, 'session', new_session) - logging.log( - logsetting.getLevelByName('fine'), - 'enter session %s', new_session - ) - try: - yield new_session - if not nested_session: - new_session.commit() - except Exception as error: - if not nested_session: - new_session.rollback() - logging.error('failed to commit session') - logging.exception(error) - if isinstance(error, IntegrityError): - for item in error.statement.split(): - if item.islower(): - object = item - break - raise exception.DuplicatedRecord( - '%s in %s' % (error.orig, object) - ) - elif isinstance(error, OperationalError): - raise exception.DatabaseException( - 'operation error in database' - ) - elif isinstance(error, exception.DatabaseException): - raise error - else: - raise exception.DatabaseException(str(error)) - finally: - if not nested_session: - new_session.close() - SCOPED_SESSION.remove() - delattr(SESSION_HOLDER, 'session') - logging.log( - logsetting.getLevelByName('fine'), - 'exit session %s', new_session - ) - - -def current_session(): - """Get the current session scope when it is called. - - :return: database session. - :raises: DatabaseException when it is not in session. - """ - try: - return SESSION_HOLDER.session - except Exception as error: - logging.error('It is not in the session scope') - logging.exception(error) - if isinstance(error, exception.DatabaseException): - raise error - else: - raise exception.DatabaseException(str(error)) - - -def run_in_session(exception_when_in_session=True): - """Decorator to make sure the decorated function run in session. - - When not exception_when_in_session, the run_in_session can be - decorated several times. - """ - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - try: - my_session = kwargs.get('session') - if my_session is not None: - return func(*args, **kwargs) - else: - with session( - exception_when_in_session=exception_when_in_session - ) as my_session: - kwargs['session'] = my_session - return func(*args, **kwargs) - except Exception as error: - logging.error( - 'got exception with func %s args %s kwargs %s', - func, args, kwargs - ) - logging.exception(error) - raise error - return wrapper - return decorator - - -def _setup_user_table(user_session): - """Initialize user table with default user.""" - logging.info('setup user table') - from compass.db.api import user - user.add_user( - session=user_session, - email=setting.COMPASS_ADMIN_EMAIL, - password=setting.COMPASS_ADMIN_PASSWORD, - is_admin=True - ) - - -def _setup_permission_table(permission_session): - """Initialize permission table.""" - logging.info('setup permission table.') - from compass.db.api import permission - permission.add_permissions_internal( - session=permission_session - ) - - -def _setup_switch_table(switch_session): - """Initialize switch table.""" - # TODO(xicheng): deprecate setup default switch. - logging.info('setup switch table') - from compass.db.api import switch - switch.add_switch( - True, setting.DEFAULT_SWITCH_IP, - session=switch_session, - machine_filters=['allow ports all'] - ) - - -def _update_others(other_session): - """Update other tables.""" - logging.info('update other tables') - from compass.db.api import utils - from compass.db import models - utils.update_db_objects( - other_session, models.Cluster - ) - utils.update_db_objects( - other_session, models.Host - ) - utils.update_db_objects( - other_session, models.ClusterHost - ) - - -@run_in_session() -def create_db(session=None): - """Create database.""" - models.BASE.metadata.create_all(bind=ENGINE) - _setup_permission_table(session) - _setup_user_table(session) - _setup_switch_table(session) - _update_others(session) - - -def drop_db(): - """Drop database.""" - models.BASE.metadata.drop_all(bind=ENGINE) diff --git a/compass-tasks/db/api/health_check_report.py b/compass-tasks/db/api/health_check_report.py deleted file mode 100644 index aaea7a7..0000000 --- a/compass-tasks/db/api/health_check_report.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, - -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Cluster health check report.""" -import logging - -from compass.db.api import cluster as cluster_api -from compass.db.api import database -from compass.db.api import host as host_api -from compass.db.api import permission -from compass.db.api import user as user_api -from compass.db.api import utils -from compass.db import exception -from compass.db import models - - -REQUIRED_INSERT_FIELDS = ['name'] -OPTIONAL_INSERT_FIELDS = [ - 'display_name', 'report', 'category', 'state', 'error_message' -] -UPDATE_FIELDS = ['report', 'state', 'error_message'] -RESP_FIELDS = [ - 'cluster_id', 'name', 'display_name', 'report', - 'category', 'state', 'error_message' -] -RESP_ACTION_FIELDS = ['cluster_id', 'status'] - - -@utils.supported_filters(REQUIRED_INSERT_FIELDS, OPTIONAL_INSERT_FIELDS) -@database.run_in_session() -@utils.wrap_to_dict(RESP_FIELDS) -def add_report_record(cluster_id, name=None, report={}, - state='verifying', session=None, **kwargs): - """Create a health check report record.""" - # Replace any white space into '-' - words = name.split() - name = '-'.join(words) - cluster = cluster_api.get_cluster_internal(cluster_id, session=session) - return utils.add_db_object( - session, models.HealthCheckReport, True, cluster.id, name, - report=report, state=state, **kwargs - ) - - -def _get_report(cluster_id, name, session=None): - cluster = cluster_api.get_cluster_internal(cluster_id, session=session) - return utils.get_db_object( - session, models.HealthCheckReport, cluster_id=cluster.id, name=name - ) - - -@utils.supported_filters(UPDATE_FIELDS) -@database.run_in_session() -@utils.wrap_to_dict(RESP_FIELDS) -def update_report(cluster_id, name, session=None, **kwargs): - """Update health check report.""" - report = _get_report(cluster_id, name, session=session) - if report.state == 'finished': - err_msg = 'Report cannot be updated if state is in "finished"' - raise exception.Forbidden(err_msg) - - return utils.update_db_object(session, report, **kwargs) - - -@utils.supported_filters(UPDATE_FIELDS) -@database.run_in_session() -@utils.wrap_to_dict(RESP_FIELDS) -def update_multi_reports(cluster_id, session=None, **kwargs): - """Bulk update reports.""" - # TODO(grace): rename the fuction if needed to reflect the fact. - return set_error(cluster_id, session=session, **kwargs) - - -def set_error(cluster_id, report={}, session=None, - state='error', error_message=None): - cluster = cluster_api.get_cluster_internal(cluster_id, session=session) - logging.debug( - "updates all reports as %s in cluster %s", - state, cluster_id - ) - return utils.update_db_objects( - session, models.HealthCheckReport, - updates={ - 'report': {}, - 'state': 'error', - 'error_message': error_message - }, cluster_id=cluster.id - ) - - -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_HEALTH_REPORT -) -@utils.wrap_to_dict(RESP_FIELDS) -def list_health_reports(cluster_id, user=None, session=None): - """List all reports in the specified cluster.""" - cluster = cluster_api.get_cluster_internal(cluster_id, session=session) - return utils.list_db_objects( - session, models.HealthCheckReport, cluster_id=cluster.id - ) - - -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_GET_HEALTH_REPORT -) -@utils.wrap_to_dict(RESP_FIELDS) -def get_health_report(cluster_id, name, user=None, session=None): - return _get_report( - cluster_id, name, session=session - ) - - -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DELETE_REPORT -) -@utils.wrap_to_dict(RESP_FIELDS) -def delete_reports(cluster_id, name=None, user=None, session=None): - # TODO(grace): better to separate this function into two. - # One is to delete a report of a cluster, the other to delete all - # reports under a cluster. - if name: - report = _get_report(cluster_id, name, session=session) - return utils.del_db_object(session, report) - else: - cluster = cluster_api.get_cluster_internal( - cluster_id, session=session - ) - return utils.del_db_objects( - session, models.HealthCheckReport, cluster_id=cluster.id - ) - - -@utils.supported_filters(optional_support_keys=['check_health']) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_CHECK_CLUSTER_HEALTH -) -@utils.wrap_to_dict(RESP_ACTION_FIELDS) -def start_check_cluster_health(cluster_id, send_report_url, - user=None, session=None, check_health={}): - """Start to check cluster health.""" - cluster = cluster_api.get_cluster_internal(cluster_id, session=session) - - if cluster.state.state != 'SUCCESSFUL': - logging.debug("state is %s" % cluster.state.state) - err_msg = "Healthcheck starts only after cluster finished deployment!" - raise exception.Forbidden(err_msg) - - reports = utils.list_db_objects( - session, models.HealthCheckReport, - cluster_id=cluster.id, state='verifying' - ) - if reports: - err_msg = 'Healthcheck in progress, please wait for it to complete!' - raise exception.Forbidden(err_msg) - - # Clear all preivous report - # TODO(grace): the delete should be moved into celery task. - # We should consider the case that celery task is down. - utils.del_db_objects( - session, models.HealthCheckReport, cluster_id=cluster.id - ) - - from compass.tasks import client as celery_client - celery_client.celery.send_task( - 'compass.tasks.cluster_health', - (cluster.id, send_report_url, user.email), - queue=user.email, - exchange=user.email, - routing_key=user.email - ) - return { - "cluster_id": cluster.id, - "status": "start to check cluster health." - } diff --git a/compass-tasks/db/api/host.py b/compass-tasks/db/api/host.py deleted file mode 100644 index 15e0bb6..0000000 --- a/compass-tasks/db/api/host.py +++ /dev/null @@ -1,1120 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Host database operations.""" -import functools -import logging -import netaddr -import re - -from compass.db.api import database -from compass.db.api import metadata_holder as metadata_api -from compass.db.api import permission -from compass.db.api import user as user_api -from compass.db.api import utils -from compass.db import exception -from compass.db import models -from compass.utils import util - - -SUPPORTED_FIELDS = ['name', 'os_name', 'owner', 'mac', 'id'] -SUPPORTED_MACHINE_HOST_FIELDS = [ - 'mac', 'tag', 'location', 'os_name', 'os_id' -] -SUPPORTED_NETOWORK_FIELDS = [ - 'interface', 'ip', 'is_mgmt', 'is_promiscuous' -] -RESP_FIELDS = [ - 'id', 'name', 'hostname', 'os_name', 'owner', 'mac', - 'switch_ip', 'port', 'switches', 'os_installer', 'os_id', 'ip', - 'reinstall_os', 'os_installed', 'tag', 'location', 'networks', - 'created_at', 'updated_at' -] -RESP_CLUSTER_FIELDS = [ - 'id', 'name', 'os_name', 'reinstall_distributed_system', - 'owner', 'adapter_name', 'flavor_name', - 'distributed_system_installed', 'created_at', 'updated_at' -] -RESP_NETWORK_FIELDS = [ - 'id', 'ip', 'interface', 'netmask', 'is_mgmt', 'is_promiscuous', - 'created_at', 'updated_at' -] -RESP_CONFIG_FIELDS = [ - 'os_config', - 'config_setp', - 'config_validated', - 'networks', - 'created_at', - 'updated_at' -] -RESP_DEPLOYED_CONFIG_FIELDS = [ - 'deployed_os_config' -] -RESP_DEPLOY_FIELDS = [ - 'status', 'host' -] -UPDATED_FIELDS = ['name', 'reinstall_os'] -UPDATED_CONFIG_FIELDS = [ - 'put_os_config' -] -PATCHED_CONFIG_FIELDS = [ - 'patched_os_config' -] -UPDATED_DEPLOYED_CONFIG_FIELDS = [ - 'deployed_os_config' -] -ADDED_NETWORK_FIELDS = [ - 'interface', 'ip', 'subnet_id' -] -OPTIONAL_ADDED_NETWORK_FIELDS = ['is_mgmt', 'is_promiscuous'] -UPDATED_NETWORK_FIELDS = [ - 'interface', 'ip', 'subnet_id', 'subnet', 'is_mgmt', - 'is_promiscuous' -] -IGNORE_FIELDS = [ - 'id', 'created_at', 'updated_at' -] -RESP_STATE_FIELDS = [ - 'id', 'state', 'percentage', 'message', 'severity', 'ready' -] -UPDATED_STATE_FIELDS = [ - 'state', 'percentage', 'message', 'severity' -] -UPDATED_STATE_INTERNAL_FIELDS = [ - 'ready' -] -RESP_LOG_FIELDS = [ - 'id', 'filename', 'position', 'partial_line', 'percentage', - 'message', 'severity', 'line_matcher_name' -] -ADDED_LOG_FIELDS = [ - 'filename' -] -UPDATED_LOG_FIELDS = [ - 'position', 'partial_line', 'percentage', - 'message', 'severity', 'line_matcher_name' -] - - -@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_HOSTS -) -@utils.wrap_to_dict(RESP_FIELDS) -def list_hosts(user=None, session=None, **filters): - """List hosts.""" - return utils.list_db_objects( - session, models.Host, **filters - ) - - -@utils.supported_filters( - optional_support_keys=SUPPORTED_MACHINE_HOST_FIELDS) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_HOSTS -) -@utils.output_filters( - missing_ok=True, - tag=utils.general_filter_callback, - location=utils.general_filter_callback, - os_name=utils.general_filter_callback, - os_id=utils.general_filter_callback -) -@utils.wrap_to_dict(RESP_FIELDS) -def list_machines_or_hosts(user=None, session=None, **filters): - """List machines or hosts if possible.""" - machines = utils.list_db_objects( - session, models.Machine, **filters - ) - machines_or_hosts = [] - for machine in machines: - host = machine.host - if host: - machines_or_hosts.append(host) - else: - machines_or_hosts.append(machine) - return machines_or_hosts - - -def _get_host(host_id, session=None, **kwargs): - """Get host by id.""" - if isinstance(host_id, (int, long)): - return utils.get_db_object( - session, models.Host, - id=host_id, **kwargs - ) - else: - raise exception.InvalidParameter( - 'host id %s type is not int compatible' % host_id - ) - - -def get_host_internal(host_id, session=None, **kwargs): - """Helper function to get host. - - Used by other files under db/api. - """ - return _get_host(host_id, session=session, **kwargs) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_HOSTS -) -@utils.wrap_to_dict(RESP_FIELDS) -def get_host( - host_id, exception_when_missing=True, - user=None, session=None, **kwargs -): - """get host info.""" - return _get_host( - host_id, - exception_when_missing=exception_when_missing, - session=session - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_HOSTS -) -@utils.wrap_to_dict(RESP_FIELDS) -def get_machine_or_host( - host_id, exception_when_missing=True, - user=None, session=None, **kwargs -): - """get machine or host if possible.""" - from compass.db.api import machine as machine_api - machine = machine_api.get_machine_internal( - host_id, - exception_when_missing=exception_when_missing, - session=session - ) - if machine.host: - return machine.host - else: - return machine - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_HOST_CLUSTERS -) -@utils.wrap_to_dict(RESP_CLUSTER_FIELDS) -def get_host_clusters(host_id, user=None, session=None, **kwargs): - """get host clusters.""" - host = _get_host(host_id, session=session) - return [clusterhost.cluster for clusterhost in host.clusterhosts] - - -def check_host_validated(host): - """Check host is validated.""" - if not host.config_validated: - raise exception.Forbidden( - 'host %s is not validated' % host.name - ) - - -def check_host_editable( - host, user=None, - check_in_installing=False -): - """Check host is editable. - - If we try to set reinstall_os or check the host is not in installing - state, we should set check_in_installing to True. - Otherwise we will check the host is not in installing or installed. - We also make sure the user is admin or the owner of the host to avoid - unauthorized user to update host attributes. - """ - if check_in_installing: - if host.state.state == 'INSTALLING': - raise exception.Forbidden( - 'host %s is not editable ' - 'when state is in installing' % host.name - ) - elif not host.reinstall_os: - raise exception.Forbidden( - 'host %s is not editable ' - 'when not to be reinstalled' % host.name - ) - if user and not user.is_admin and host.creator_id != user.id: - raise exception.Forbidden( - 'host %s is not editable ' - 'when user is not admin or the owner of the host' % host.name - ) - - -def is_host_editable( - host, user=None, - check_in_installing=False -): - """Get if host is editable.""" - try: - check_host_editable( - host, user=user, - check_in_installing=check_in_installing - ) - return True - except exception.Forbidden: - return False - - -def validate_host(host): - """Validate host. - - Makesure hostname is not empty, there is only one mgmt network, - The mgmt network is not in promiscuous mode. - """ - if not host.hostname: - raise exception.Invalidparameter( - 'host %s does not set hostname' % host.name - ) - if not host.host_networks: - raise exception.InvalidParameter( - 'host %s does not have any network' % host.name - ) - mgmt_interface_set = False - for host_network in host.host_networks: - if host_network.is_mgmt: - if mgmt_interface_set: - raise exception.InvalidParameter( - 'host %s multi interfaces set mgmt ' % host.name - ) - if host_network.is_promiscuous: - raise exception.InvalidParameter( - 'host %s interface %s is mgmt but promiscuous' % ( - host.name, host_network.interface - ) - ) - mgmt_interface_set = True - if not mgmt_interface_set: - raise exception.InvalidParameter( - 'host %s has no mgmt interface' % host.name - ) - - -@utils.supported_filters( - optional_support_keys=UPDATED_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates(name=utils.check_name) -@utils.wrap_to_dict(RESP_FIELDS) -def _update_host(host_id, session=None, user=None, **kwargs): - """Update a host internal.""" - host = _get_host(host_id, session=session) - if host.state.state == "SUCCESSFUL" and not host.reinstall_os: - logging.info("ignoring successful host: %s", host_id) - return {} - check_host_editable( - host, user=user, - check_in_installing=kwargs.get('reinstall_os', False) - ) - return utils.update_db_object(session, host, **kwargs) - - -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_HOST -) -def update_host(host_id, user=None, session=None, **kwargs): - """Update a host.""" - return _update_host(host_id, session=session, user=user, **kwargs) - - -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_HOST -) -def update_hosts(data=[], user=None, session=None): - """Update hosts.""" - # TODO(xicheng): this batch function is not similar as others. - # try to make it similar output as others and batch update should - # tolerate partial failure. - hosts = [] - for host_data in data: - hosts.append(_update_host(session=session, user=user, **host_data)) - return hosts - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEL_HOST -) -@utils.wrap_to_dict( - RESP_FIELDS + ['status', 'host'], - host=RESP_FIELDS -) -def del_host( - host_id, force=False, from_database_only=False, - user=None, session=None, **kwargs -): - """Delete a host. - - If force, we delete the host anyway. - If from_database_only, we only delete the host record in databaes. - Otherwise we send to del host task to celery to delete the host - record in os installer and package installer, clean installation logs - and at last clean database record. - The backend will call this function again after it deletes the record - in os installer and package installer with from_database_only set. - """ - from compass.db.api import cluster as cluster_api - host = _get_host(host_id, session=session) - # force set host state to ERROR when we want to delete the - # host anyway even the host is in installing or already - # installed. It let the api know the deleting is in doing when backend - # is doing the real deleting. In future we may import a new state like - # INDELETE to indicate the deleting is processing. - # We need discuss about if we can delete a host when it is already - # installed by api. - if host.state.state != 'UNINITIALIZED' and force: - host.state.state = 'ERROR' - check_host_editable( - host, user=user, - check_in_installing=True - ) - cluster_ids = [] - for clusterhost in host.clusterhosts: - if clusterhost.state.state != 'UNINITIALIZED' and force: - clusterhost.state.state = 'ERROR' - # TODO(grace): here we check all clusters which use this host editable. - # Because in backend we do not have functions to delete host without - # reference its cluster. After deleting pure host supported in backend, - # we should change code here to is_cluster_editable. - # Here delete a host may fail even we set force flag. - cluster_api.check_cluster_editable( - clusterhost.cluster, user=user, - check_in_installing=True - ) - cluster_ids.append(clusterhost.cluster_id) - - # Delete host record directly if there is no need to delete it - # in backend or from_database_only is set. - if host.state.state == 'UNINITIALIZED' or from_database_only: - return utils.del_db_object(session, host) - else: - logging.info( - 'send del host %s task to celery', host_id - ) - if not user: - user_id = host.creator_id - user_dict = user_api.get_user(user_id, session=session) - user_email = user_dict['email'] - else: - user_email = user.email - from compass.tasks import client as celery_client - celery_client.celery.send_task( - 'compass.tasks.delete_host', - ( - user.email, host.id, cluster_ids - ), - queue=user_email, - exchange=user_email, - routing_key=user_email - ) - return { - 'status': 'delete action sent', - 'host': host, - } - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_HOST_CONFIG -) -@utils.wrap_to_dict(RESP_CONFIG_FIELDS) -def get_host_config(host_id, user=None, session=None, **kwargs): - """Get host config.""" - return _get_host(host_id, session=session) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_HOST_CONFIG -) -@utils.wrap_to_dict(RESP_DEPLOYED_CONFIG_FIELDS) -def get_host_deployed_config(host_id, user=None, session=None, **kwargs): - """Get host deployed config.""" - return _get_host(host_id, session=session) - - -# replace os_config to deployed_os_config in kwargs. -@utils.replace_filters( - os_config='deployed_os_config' -) -@utils.supported_filters( - UPDATED_DEPLOYED_CONFIG_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_HOST_CONFIG -) -@utils.wrap_to_dict(RESP_CONFIG_FIELDS) -def update_host_deployed_config(host_id, user=None, session=None, **kwargs): - """Update host deployed config.""" - host = _get_host(host_id, session=session) - check_host_editable(host, user=user) - check_host_validated(host) - return utils.update_db_object(session, host, **kwargs) - - -def _host_os_config_validates( - config, host, session=None, user=None, **kwargs -): - """Check host os config's validation.""" - metadata_api.validate_os_config( - config, host.os_id - ) - - -@utils.input_validates_with_args( - put_os_config=_host_os_config_validates -) -@utils.output_validates_with_args( - os_config=_host_os_config_validates -) -@utils.wrap_to_dict(RESP_CONFIG_FIELDS) -def _update_host_config(host, session=None, user=None, **kwargs): - """Update host config.""" - check_host_editable(host, user=user) - return utils.update_db_object(session, host, **kwargs) - - -# replace os_config to put_os_config in kwargs. -# It tells db the os_config will be updated not patched. -@utils.replace_filters( - os_config='put_os_config' -) -@utils.supported_filters( - UPDATED_CONFIG_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_HOST_CONFIG -) -def update_host_config(host_id, user=None, session=None, **kwargs): - """Update host config.""" - host = _get_host(host_id, session=session) - return _update_host_config( - host, session=session, user=user, **kwargs - ) - - -# replace os_config to patched_os_config in kwargs. -# It tells db os_config will be patched not be updated. -@utils.replace_filters( - os_config='patched_os_config' -) -@utils.supported_filters( - PATCHED_CONFIG_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_HOST_CONFIG -) -def patch_host_config(host_id, user=None, session=None, **kwargs): - """Patch host config.""" - host = _get_host(host_id, session=session) - return _update_host_config( - host, session=session, user=user, **kwargs - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEL_HOST_CONFIG -) -@utils.wrap_to_dict(RESP_CONFIG_FIELDS) -def del_host_config(host_id, user=None, session=None): - """delete a host config.""" - host = _get_host(host_id, session=session) - check_host_editable(host, user=user) - return utils.update_db_object( - session, host, os_config={}, config_validated=False - ) - - -@utils.supported_filters( - optional_support_keys=SUPPORTED_NETOWORK_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_HOST_NETWORKS -) -@utils.wrap_to_dict(RESP_NETWORK_FIELDS) -def list_host_networks(host_id, user=None, session=None, **filters): - """Get host networks for a host.""" - host = _get_host(host_id, session=session) - return utils.list_db_objects( - session, models.HostNetwork, - host_id=host.id, **filters - ) - - -@utils.supported_filters( - optional_support_keys=SUPPORTED_NETOWORK_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_HOST_NETWORKS -) -@utils.wrap_to_dict(RESP_NETWORK_FIELDS) -def list_hostnetworks(user=None, session=None, **filters): - """Get host networks.""" - return utils.list_db_objects( - session, models.HostNetwork, **filters - ) - - -def _get_hostnetwork(host_network_id, session=None, **kwargs): - """Get hostnetwork by hostnetwork id.""" - if isinstance(host_network_id, (int, long)): - return utils.get_db_object( - session, models.HostNetwork, - id=host_network_id, **kwargs - ) - raise exception.InvalidParameter( - 'host network id %s type is not int compatible' % host_network_id - ) - - -def _get_host_network(host_id, host_network_id, session=None, **kwargs): - """Get hostnetwork by host id and hostnetwork id.""" - host = _get_host(host_id, session=session) - host_network = _get_hostnetwork(host_network_id, session=session, **kwargs) - if host_network.host_id != host.id: - raise exception.RecordNotExists( - 'host %s does not own host network %s' % ( - host.id, host_network.id - ) - ) - return host_network - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_HOST_NETWORKS -) -@utils.wrap_to_dict(RESP_NETWORK_FIELDS) -def get_host_network( - host_id, host_network_id, - user=None, session=None, **kwargs -): - """Get host network.""" - return _get_host_network( - host_id, host_network_id, session=session - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_HOST_NETWORKS -) -@utils.wrap_to_dict(RESP_NETWORK_FIELDS) -def get_hostnetwork(host_network_id, user=None, session=None, **kwargs): - """Get host network.""" - return _get_hostnetwork(host_network_id, session=session) - - -@utils.supported_filters( - ADDED_NETWORK_FIELDS, - optional_support_keys=OPTIONAL_ADDED_NETWORK_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates( - ip=utils.check_ip -) -@utils.wrap_to_dict(RESP_NETWORK_FIELDS) -def _add_host_network( - host_id, exception_when_existing=True, - session=None, user=None, interface=None, ip=None, **kwargs -): - """Add hostnetwork to a host.""" - host = _get_host(host_id, session=session) - check_host_editable(host, user=user) - user_id = user.id - return utils.add_db_object( - session, models.HostNetwork, - exception_when_existing, - host.id, interface, user_id, ip=ip, **kwargs - ) - - -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_HOST_NETWORK -) -def add_host_network( - host_id, exception_when_existing=True, - interface=None, user=None, session=None, **kwargs -): - """Create a hostnetwork to a host.""" - return _add_host_network( - host_id, - exception_when_existing, - interface=interface, session=session, user=user, **kwargs - ) - - -def _get_hostnetwork_by_ip( - ip, session=None, **kwargs -): - ip_int = long(netaddr.IPAddress(ip)) - return utils.get_db_object( - session, models.HostNetwork, - ip_int=ip_int, **kwargs - ) - - -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_HOST_NETWORK -) -def add_host_networks( - exception_when_existing=False, - data=[], user=None, session=None -): - """Create host networks.""" - hosts = [] - failed_hosts = [] - for host_data in data: - host_id = host_data['host_id'] - host = _get_host(host_id, session=session) - networks = host_data['networks'] - host_networks = [] - failed_host_networks = [] - for network in networks: - host_network = _get_hostnetwork_by_ip( - network['ip'], session=session, - exception_when_missing=False - ) - if ( - host_network and not ( - host_network.host_id == host.id and - host_network.interface == network['interface'] - ) - ): - logging.error('ip %s exists in host network %s' % ( - network['ip'], host_network.id - )) - failed_host_networks.append(network) - else: - host_networks.append(_add_host_network( - host.id, exception_when_existing, - session=session, user=user, **network - )) - if host_networks: - hosts.append({'host_id': host.id, 'networks': host_networks}) - if failed_host_networks: - failed_hosts.append({ - 'host_id': host.id, 'networks': failed_host_networks - }) - return { - 'hosts': hosts, - 'failed_hosts': failed_hosts - } - - -@utils.wrap_to_dict(RESP_NETWORK_FIELDS) -def _update_host_network( - host_network, session=None, user=None, **kwargs -): - """Update host network.""" - check_host_editable(host_network.host, user=user) - return utils.update_db_object(session, host_network, **kwargs) - - -@utils.supported_filters( - optional_support_keys=UPDATED_NETWORK_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates( - ip=utils.check_ip -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_HOST_NETWORK -) -def update_host_network( - host_id, host_network_id, user=None, session=None, **kwargs -): - """Update a host network by host id and host network id.""" - host = _get_host( - host_id, session=session - ) - if host.state.state == "SUCCESSFUL" and not host.reinstall_os: - logging.info("ignoring updating request for successful hosts") - return {} - - host_network = _get_host_network( - host_id, host_network_id, session=session - ) - return _update_host_network( - host_network, session=session, user=user, **kwargs - ) - - -@utils.supported_filters( - optional_support_keys=UPDATED_NETWORK_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates( - ip=utils.check_ip -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_HOST_NETWORK -) -def update_hostnetwork(host_network_id, user=None, session=None, **kwargs): - """Update a host network by host network id.""" - host_network = _get_hostnetwork( - host_network_id, session=session - ) - return _update_host_network( - host_network, session=session, user=user, **kwargs - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEL_HOST_NETWORK -) -@utils.wrap_to_dict(RESP_NETWORK_FIELDS) -def del_host_network( - host_id, host_network_id, user=None, - session=None, **kwargs -): - """Delete a host network by host id and host network id.""" - host_network = _get_host_network( - host_id, host_network_id, session=session - ) - check_host_editable(host_network.host, user=user) - return utils.del_db_object(session, host_network) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEL_HOST_NETWORK -) -@utils.wrap_to_dict(RESP_NETWORK_FIELDS) -def del_hostnetwork(host_network_id, user=None, session=None, **kwargs): - """Delete a host network by host network id.""" - host_network = _get_hostnetwork( - host_network_id, session=session - ) - check_host_editable(host_network.host, user=user) - return utils.del_db_object(session, host_network) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_GET_HOST_STATE -) -@utils.wrap_to_dict(RESP_STATE_FIELDS) -def get_host_state(host_id, user=None, session=None, **kwargs): - """Get host state info.""" - return _get_host(host_id, session=session).state - - -@utils.supported_filters( - optional_support_keys=UPDATED_STATE_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_HOST_STATE -) -@utils.wrap_to_dict(RESP_STATE_FIELDS) -def update_host_state(host_id, user=None, session=None, **kwargs): - """Update a host state.""" - host = _get_host(host_id, session=session) - utils.update_db_object(session, host.state, **kwargs) - return host.state - - -@util.deprecated -@utils.supported_filters( - optional_support_keys=UPDATED_STATE_INTERNAL_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_HOST_STATE -) -@utils.wrap_to_dict(['status', 'host']) -def update_host_state_internal( - host_id, from_database_only=False, - user=None, session=None, **kwargs -): - """Update a host state. - - This function is called when host os is installed. - If from_database_only, the state is updated in database. - Otherwise a celery task sent to os installer and package installer - to do some future actions. - """ - # TODO(xicheng): should be merged into update_host_state - host = _get_host(host_id, session=session) - logging.info("======host state: %s", host.state) - if 'ready' in kwargs and kwargs['ready'] and not host.state.ready: - ready_triggered = True - else: - ready_triggered = False - clusterhosts_ready = {} - clusters_os_ready = {} - if ready_triggered: - for clusterhost in host.clusterhosts: - cluster = clusterhost.cluster - if cluster.flavor_name: - clusterhosts_ready[cluster.id] = False - else: - clusterhosts_ready[cluster.id] = True - all_os_ready = True - for clusterhost_in_cluster in cluster.clusterhosts: - host_in_cluster = clusterhost_in_cluster.host - if host_in_cluster.id == host.id: - continue - if not host_in_cluster.state.ready: - all_os_ready = False - clusters_os_ready[cluster.id] = all_os_ready - logging.debug('host %s ready: %s', host_id, ready_triggered) - logging.debug("clusterhosts_ready is: %s", clusterhosts_ready) - logging.debug("clusters_os_ready is %s", clusters_os_ready) - - if not ready_triggered or from_database_only: - logging.debug('%s state is set to %s', host.name, kwargs) - utils.update_db_object(session, host.state, **kwargs) - if not host.state.ready: - for clusterhost in host.clusterhosts: - utils.update_db_object( - session, clusterhost.state, ready=False - ) - utils.update_db_object( - session, clusterhost.cluster.state, ready=False - ) - status = '%s state is updated' % host.name - else: - if not user: - user_id = host.creator_id - user_dict = user_api.get_user(user_id, session=session) - user_email = user_dict['email'] - else: - user_email = user.email - from compass.tasks import client as celery_client - celery_client.celery.send_task( - 'compass.tasks.os_installed', - ( - host.id, clusterhosts_ready, - clusters_os_ready - ), - queue=user_email, - exchange=user_email, - routing_key=user_email - ) - status = '%s: clusterhosts ready %s clusters os ready %s' % ( - host.name, clusterhosts_ready, clusters_os_ready - ) - logging.info('action status: %s', status) - return { - 'status': status, - 'host': host.state - } - - -@utils.supported_filters([]) -@database.run_in_session() -@utils.wrap_to_dict(RESP_LOG_FIELDS) -def get_host_log_histories(host_id, user=None, session=None, **kwargs): - """Get host log history.""" - host = _get_host(host_id, session=session) - return utils.list_db_objects( - session, models.HostLogHistory, id=host.id, **kwargs - ) - - -def _get_host_log_history(host_id, filename, session=None, **kwargs): - host = _get_host(host_id, session=session) - return utils.get_db_object( - session, models.HostLogHistory, id=host.id, - filename=filename, **kwargs - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@utils.wrap_to_dict(RESP_LOG_FIELDS) -def get_host_log_history(host_id, filename, user=None, session=None, **kwargs): - """Get host log history.""" - return _get_host_log_history( - host_id, filename, session=session - ) - - -@utils.supported_filters( - optional_support_keys=UPDATED_LOG_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@utils.wrap_to_dict(RESP_LOG_FIELDS) -def update_host_log_history( - host_id, filename, user=None, - session=None, **kwargs -): - """Update a host log history.""" - host_log_history = _get_host_log_history( - host_id, filename, session=session - ) - return utils.update_db_object(session, host_log_history, **kwargs) - - -@utils.supported_filters( - ADDED_LOG_FIELDS, - optional_support_keys=UPDATED_LOG_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@utils.wrap_to_dict(RESP_LOG_FIELDS) -def add_host_log_history( - host_id, exception_when_existing=False, - filename=None, user=None, session=None, **kwargs -): - """add a host log history.""" - host = _get_host(host_id, session=session) - return utils.add_db_object( - session, models.HostLogHistory, exception_when_existing, - host.id, filename, **kwargs - ) - - -@utils.supported_filters(optional_support_keys=['poweron']) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEPLOY_HOST -) -@utils.wrap_to_dict( - RESP_DEPLOY_FIELDS, - host=RESP_CONFIG_FIELDS -) -def poweron_host( - host_id, poweron={}, user=None, session=None, **kwargs -): - """power on host.""" - from compass.tasks import client as celery_client - host = _get_host(host_id, session=session) - check_host_validated(host) - if not user: - user_id = host.creator_id - user_dict = user_api.get_user(user_id, session=session) - user_email = user_dict['email'] - else: - user_email = user.email - celery_client.celery.send_task( - 'compass.tasks.poweron_host', - (host.id,), - queue=user_email, - exchange=user_email, - routing_key=user_email - ) - return { - 'status': 'poweron %s action sent' % host.name, - 'host': host - } - - -@utils.supported_filters(optional_support_keys=['poweroff']) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEPLOY_HOST -) -@utils.wrap_to_dict( - RESP_DEPLOY_FIELDS, - host=RESP_CONFIG_FIELDS -) -def poweroff_host( - host_id, poweroff={}, user=None, session=None, **kwargs -): - """power off host.""" - from compass.tasks import client as celery_client - host = _get_host(host_id, session=session) - check_host_validated(host) - if not user: - user_id = host.creator_id - user_dict = user_api.get_user(user_id, session=session) - user_email = user_dict['email'] - else: - user_email = user.email - celery_client.celery.send_task( - 'compass.tasks.poweroff_host', - (host.id,), - queue=user_email, - exchange=user_email, - routing_key=user_email - ) - return { - 'status': 'poweroff %s action sent' % host.name, - 'host': host - } - - -@utils.supported_filters(optional_support_keys=['reset']) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEPLOY_HOST -) -@utils.wrap_to_dict( - RESP_DEPLOY_FIELDS, - host=RESP_CONFIG_FIELDS -) -def reset_host( - host_id, reset={}, user=None, session=None, **kwargs -): - """reset host.""" - from compass.tasks import client as celery_client - host = _get_host(host_id, session=session) - check_host_validated(host) - if not user: - user_id = host.creator_id - user_dict = user_api.get_user(user_id, session=session) - user_email = user_dict['email'] - else: - user_email = user.email - celery_client.celery.send_task( - 'compass.tasks.reset_host', - (host.id,), - queue=user_email, - exchange=user_email, - routing_key=user_email - ) - return { - 'status': 'reset %s action sent' % host.name, - 'host': host - } diff --git a/compass-tasks/db/api/machine.py b/compass-tasks/db/api/machine.py deleted file mode 100644 index b7b16b2..0000000 --- a/compass-tasks/db/api/machine.py +++ /dev/null @@ -1,317 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Switch database operations.""" -import logging -import re - -from compass.db.api import database -from compass.db.api import permission -from compass.db.api import user as user_api -from compass.db.api import utils -from compass.db import exception -from compass.db import models - -from compass.utils import setting_wrapper as setting -from compass.utils import util - - -MACHINE_PRIMARY_FILEDS = ['mac', 'owner_id'] -SUPPORTED_FIELDS = [ - 'mac', 'tag', 'location', - 'machine_attributes', 'owner_id'] -IGNORE_FIELDS = ['id', 'created_at', 'updated_at'] -UPDATED_FIELDS = [ - 'ipmi_credentials', 'machine_attributes', - 'tag', 'location'] -PATCHED_FIELDS = [ - 'patched_ipmi_credentials', 'patched_tag', - 'patched_location' -] -RESP_FIELDS = [ - 'id', 'mac', 'ipmi_credentials', 'switches', 'switch_ip', - 'port', 'vlans', 'machine_attributes', 'owner_id', - 'tag', 'location', 'created_at', 'updated_at' -] -RESP_DEPLOY_FIELDS = [ - 'status', 'machine' -] - - -def _get_machine(machine_id, session=None, **kwargs): - """Get machine by id.""" - if isinstance(machine_id, (int, long)): - return utils.get_db_object( - session, models.Machine, - id=machine_id, **kwargs - ) - raise exception.InvalidParameter( - 'machine id %s type is not int compatible' % machine_id - ) - - -@utils.supported_filters( - MACHINE_PRIMARY_FILEDS, - optional_support_keys=SUPPORTED_FIELDS -) -@utils.input_validates(mac=utils.check_mac) -def _add_machine(mac, owner_id=None, session=None, **kwargs): - """Add a machine.""" - if isinstance(owner_id, (int, long)): - return utils.add_db_object( - session, models.Machine, - True, - mac, - owner_id=owner_id, - **kwargs - ) - raise exception.InvalidParameter( - 'owner id %s type is not int compatible' % owner_id - ) - - -@database.run_in_session() -@utils.wrap_to_dict(RESP_FIELDS) -def add_machine( - mac, owner_id=None, session=None, user=None, **kwargs -): - """Add a machine.""" - return _add_machine( - mac, - owner_id=owner_id, - session=session, **kwargs - ) - - -def get_machine_internal(machine_id, session=None, **kwargs): - """Helper function to other files under db/api.""" - return _get_machine(machine_id, session=session, **kwargs) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_MACHINES -) -@utils.wrap_to_dict(RESP_FIELDS) -def get_machine( - machine_id, exception_when_missing=True, - user=None, session=None, **kwargs -): - """get a machine.""" - return _get_machine( - machine_id, session=session, - exception_when_missing=exception_when_missing - ) - - -@utils.supported_filters( - optional_support_keys=SUPPORTED_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_MACHINES -) -@utils.output_filters( - tag=utils.general_filter_callback, - location=utils.general_filter_callback -) -@utils.wrap_to_dict(RESP_FIELDS) -def list_machines(user=None, session=None, **filters): - """List machines.""" - machines = utils.list_db_objects( - session, models.Machine, **filters - ) - if not user.is_admin and len(machines): - machines = [m for m in machines if m.owner_id == user.id] - return machines - - -@utils.wrap_to_dict(RESP_FIELDS) -def _update_machine(machine_id, session=None, **kwargs): - """Update a machine.""" - machine = _get_machine(machine_id, session=session) - return utils.update_db_object(session, machine, **kwargs) - - -@utils.supported_filters( - optional_support_keys=UPDATED_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates(ipmi_credentials=utils.check_ipmi_credentials) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_MACHINE -) -def update_machine(machine_id, user=None, session=None, **kwargs): - """Update a machine.""" - return _update_machine( - machine_id, session=session, **kwargs - ) - - -# replace [ipmi_credentials, tag, location] to -# [patched_ipmi_credentials, patched_tag, patched_location] -# in kwargs. It tells db these fields will be patched. -@utils.replace_filters( - ipmi_credentials='patched_ipmi_credentials', - tag='patched_tag', - location='patched_location' -) -@utils.supported_filters( - optional_support_keys=PATCHED_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@utils.output_validates(ipmi_credentials=utils.check_ipmi_credentials) -@user_api.check_user_permission( - permission.PERMISSION_ADD_MACHINE -) -def patch_machine(machine_id, user=None, session=None, **kwargs): - """Patch a machine.""" - return _update_machine( - machine_id, session=session, **kwargs - ) - - -def _check_machine_deletable(machine): - """Check a machine deletable.""" - if machine.host: - host = machine.host - raise exception.NotAcceptable( - 'machine %s has host %s on it' % ( - machine.mac, host.name - ) - ) - - -@utils.supported_filters() -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEL_MACHINE -) -@utils.wrap_to_dict(RESP_FIELDS) -def del_machine(machine_id, user=None, session=None, **kwargs): - """Delete a machine.""" - machine = _get_machine(machine_id, session=session) - _check_machine_deletable(machine) - return utils.del_db_object(session, machine) - - -@utils.supported_filters(optional_support_keys=['poweron']) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEPLOY_HOST -) -@utils.wrap_to_dict( - RESP_DEPLOY_FIELDS, - machine=RESP_FIELDS -) -def poweron_machine( - machine_id, poweron={}, user=None, session=None, **kwargs -): - """power on machine.""" - from compass.tasks import client as celery_client - machine = _get_machine( - machine_id, session=session - ) - if not user: - user_id = machine.owner_id - user_dict = user_api.get_user(user_id, session=session) - user_email = user_dict['email'] - else: - user_email = user.email - celery_client.celery.send_task( - 'compass.tasks.poweron_machine', - (machine_id,), - queue=user_email, - exchange=user_email, - routing_key=user_email - ) - return { - 'status': 'poweron %s action sent' % machine.mac, - 'machine': machine - } - - -@utils.supported_filters(optional_support_keys=['poweroff']) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEPLOY_HOST -) -@utils.wrap_to_dict( - RESP_DEPLOY_FIELDS, - machine=RESP_FIELDS -) -def poweroff_machine( - machine_id, poweroff={}, user=None, session=None, **kwargs -): - """power off machine.""" - from compass.tasks import client as celery_client - machine = _get_machine( - machine_id, session=session - ) - if not user: - user_id = machine.owner_id - user_dict = user_api.get_user(user_id, session=session) - user_email = user_dict['email'] - else: - user_email = user.email - celery_client.celery.send_task( - 'compass.tasks.poweroff_machine', - (machine_id,), - queue=user_email, - exchange=user_email, - routing_key=user_email - ) - return { - 'status': 'poweroff %s action sent' % machine.mac, - 'machine': machine - } - - -@utils.supported_filters(optional_support_keys=['reset']) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEPLOY_HOST -) -@utils.wrap_to_dict( - RESP_DEPLOY_FIELDS, - machine=RESP_FIELDS -) -def reset_machine( - machine_id, reset={}, user=None, session=None, **kwargs -): - """reset machine.""" - from compass.tasks import client as celery_client - machine = _get_machine( - machine_id, session=session - ) - if not user: - user_id = machine.owner_id - user_dict = user_api.get_user(user_id, session=session) - user_email = user_dict['email'] - else: - user_email = user.email - celery_client.celery.send_task( - 'compass.tasks.reset_machine', - (machine_id,), - queue=user_email, - exchange=user_email, - routing_key=user_email - ) - return { - 'status': 'reset %s action sent' % machine.mac, - 'machine': machine - } diff --git a/compass-tasks/db/api/metadata.py b/compass-tasks/db/api/metadata.py deleted file mode 100644 index 16310c8..0000000 --- a/compass-tasks/db/api/metadata.py +++ /dev/null @@ -1,517 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Metadata related database operations.""" -import copy -import logging -import string - -from compass.db.api import adapter as adapter_api -from compass.db.api import database -from compass.db.api import utils -from compass.db import callback as metadata_callback -from compass.db import exception -from compass.db import models -from compass.db import validator as metadata_validator - - -from compass.utils import setting_wrapper as setting -from compass.utils import util - - -OS_FIELDS = None -PACKAGE_FIELDS = None -FLAVOR_FIELDS = None -OSES_METADATA = None -PACKAGES_METADATA = None -FLAVORS_METADATA = None -OSES_METADATA_UI_CONVERTERS = None -FLAVORS_METADATA_UI_CONVERTERS = None - - -def _get_field_from_configuration(configs): - """Get fields from configurations.""" - fields = {} - for config in configs: - if not isinstance(config, dict): - raise exception.InvalidParameter( - 'config %s is not dict' % config - ) - field_name = config['NAME'] - fields[field_name] = { - 'name': field_name, - 'id': field_name, - 'field_type': config.get('FIELD_TYPE', basestring), - 'display_type': config.get('DISPLAY_TYPE', 'text'), - 'validator': config.get('VALIDATOR', None), - 'js_validator': config.get('JS_VALIDATOR', None), - 'description': config.get('DESCRIPTION', field_name) - } - return fields - - -def _get_os_fields_from_configuration(): - """Get os fields from os field config dir.""" - env_locals = {} - env_locals.update(metadata_validator.VALIDATOR_LOCALS) - env_locals.update(metadata_callback.CALLBACK_LOCALS) - configs = util.load_configs( - setting.OS_FIELD_DIR, - env_locals=env_locals - ) - return _get_field_from_configuration( - configs - ) - - -def _get_package_fields_from_configuration(): - """Get package fields from package field config dir.""" - env_locals = {} - env_locals.update(metadata_validator.VALIDATOR_LOCALS) - env_locals.update(metadata_callback.CALLBACK_LOCALS) - configs = util.load_configs( - setting.PACKAGE_FIELD_DIR, - env_locals=env_locals - ) - return _get_field_from_configuration( - configs - ) - - -def _get_flavor_fields_from_configuration(): - """Get flavor fields from flavor field config dir.""" - env_locals = {} - env_locals.update(metadata_validator.VALIDATOR_LOCALS) - env_locals.update(metadata_callback.CALLBACK_LOCALS) - configs = util.load_configs( - setting.FLAVOR_FIELD_DIR, - env_locals=env_locals - ) - return _get_field_from_configuration( - configs - ) - - -def _get_metadata_from_configuration( - path, name, config, - fields, **kwargs -): - """Recursively get metadata from configuration. - - Args: - path: used to indicate the path to the root element. - mainly for trouble shooting. - name: the key of the metadata section. - config: the value of the metadata section. - fields: all fields defined in os fields or package fields dir. - """ - if not isinstance(config, dict): - raise exception.InvalidParameter( - '%s config %s is not dict' % (path, config) - ) - metadata_self = config.get('_self', {}) - if 'field' in metadata_self: - field_name = metadata_self['field'] - field = fields[field_name] - else: - field = {} - # mapping to may contain $ like $partition. Here we replace the - # $partition to the key of the correspendent config. The backend then - # can use this kind of feature to support multi partitions when we - # only declare the partition metadata in one place. - mapping_to_template = metadata_self.get('mapping_to', None) - if mapping_to_template: - mapping_to = string.Template( - mapping_to_template - ).safe_substitute( - **kwargs - ) - else: - mapping_to = None - self_metadata = { - 'name': name, - 'display_name': metadata_self.get('display_name', name), - 'field_type': field.get('field_type', dict), - 'display_type': field.get('display_type', None), - 'description': metadata_self.get( - 'description', field.get('description', None) - ), - 'is_required': metadata_self.get('is_required', False), - 'required_in_whole_config': metadata_self.get( - 'required_in_whole_config', False), - 'mapping_to': mapping_to, - 'validator': metadata_self.get( - 'validator', field.get('validator', None) - ), - 'js_validator': metadata_self.get( - 'js_validator', field.get('js_validator', None) - ), - 'default_value': metadata_self.get('default_value', None), - 'default_callback': metadata_self.get('default_callback', None), - 'default_callback_params': metadata_self.get( - 'default_callback_params', {}), - 'options': metadata_self.get('options', None), - 'options_callback': metadata_self.get('options_callback', None), - 'options_callback_params': metadata_self.get( - 'options_callback_params', {}), - 'autofill_callback': metadata_self.get( - 'autofill_callback', None), - 'autofill_callback_params': metadata_self.get( - 'autofill_callback_params', {}), - 'required_in_options': metadata_self.get( - 'required_in_options', False) - } - self_metadata.update(kwargs) - metadata = {'_self': self_metadata} - # Key extension used to do two things: - # one is to return the extended metadata that $ - # will be replace to possible extensions. - # The other is to record the $ to extended value - # and used in future mapping_to subsititution. - # TODO(grace): select proper name instead of key_extensions if - # you think it is better. - # Suppose key_extension is {'$partition': ['/var', '/']} for $partition - # the metadata for $partition will be mapped to { - # '/var': ..., '/': ...} and kwargs={'partition': '/var'} and - # kwargs={'partition': '/'} will be parsed to recursive metadata parsing - # for sub metadata under '/var' and '/'. Then in the metadata parsing - # for the sub metadata, this kwargs will be used to substitute mapping_to. - key_extensions = metadata_self.get('key_extensions', {}) - general_keys = [] - for key, value in config.items(): - if key.startswith('_'): - continue - if key in key_extensions: - if not key.startswith('$'): - raise exception.InvalidParameter( - '%s subkey %s should start with $' % ( - path, key - ) - ) - extended_keys = key_extensions[key] - for extended_key in extended_keys: - if extended_key.startswith('$'): - raise exception.InvalidParameter( - '%s extended key %s should not start with $' % ( - path, extended_key - ) - ) - sub_kwargs = dict(kwargs) - sub_kwargs[key[1:]] = extended_key - metadata[extended_key] = _get_metadata_from_configuration( - '%s/%s' % (path, extended_key), extended_key, value, - fields, **sub_kwargs - ) - else: - if key.startswith('$'): - general_keys.append(key) - metadata[key] = _get_metadata_from_configuration( - '%s/%s' % (path, key), key, value, - fields, **kwargs - ) - if len(general_keys) > 1: - raise exception.InvalidParameter( - 'foud multi general keys in %s: %s' % ( - path, general_keys - ) - ) - return metadata - - -def _get_oses_metadata_from_configuration(): - """Get os metadata from os metadata config dir.""" - oses_metadata = {} - env_locals = {} - env_locals.update(metadata_validator.VALIDATOR_LOCALS) - env_locals.update(metadata_callback.CALLBACK_LOCALS) - configs = util.load_configs( - setting.OS_METADATA_DIR, - env_locals=env_locals - ) - for config in configs: - os_name = config['OS'] - os_metadata = oses_metadata.setdefault(os_name, {}) - for key, value in config['METADATA'].items(): - os_metadata[key] = _get_metadata_from_configuration( - key, key, value, OS_FIELDS - ) - - oses = adapter_api.OSES - parents = {} - for os_name, os in oses.items(): - parent = os.get('parent', None) - parents[os_name] = parent - for os_name, os in oses.items(): - oses_metadata[os_name] = util.recursive_merge_dict( - os_name, oses_metadata, parents - ) - return oses_metadata - - -def _get_packages_metadata_from_configuration(): - """Get package metadata from package metadata config dir.""" - packages_metadata = {} - env_locals = {} - env_locals.update(metadata_validator.VALIDATOR_LOCALS) - env_locals.update(metadata_callback.CALLBACK_LOCALS) - configs = util.load_configs( - setting.PACKAGE_METADATA_DIR, - env_locals=env_locals - ) - for config in configs: - adapter_name = config['ADAPTER'] - package_metadata = packages_metadata.setdefault(adapter_name, {}) - for key, value in config['METADATA'].items(): - package_metadata[key] = _get_metadata_from_configuration( - key, key, value, PACKAGE_FIELDS - ) - adapters = adapter_api.ADAPTERS - parents = {} - for adapter_name, adapter in adapters.items(): - parent = adapter.get('parent', None) - parents[adapter_name] = parent - for adapter_name, adapter in adapters.items(): - packages_metadata[adapter_name] = util.recursive_merge_dict( - adapter_name, packages_metadata, parents - ) - return packages_metadata - - -def _get_flavors_metadata_from_configuration(): - """Get flavor metadata from flavor metadata config dir.""" - flavors_metadata = {} - env_locals = {} - env_locals.update(metadata_validator.VALIDATOR_LOCALS) - env_locals.update(metadata_callback.CALLBACK_LOCALS) - configs = util.load_configs( - setting.FLAVOR_METADATA_DIR, - env_locals=env_locals - ) - for config in configs: - adapter_name = config['ADAPTER'] - flavor_name = config['FLAVOR'] - flavor_metadata = flavors_metadata.setdefault( - adapter_name, {} - ).setdefault(flavor_name, {}) - for key, value in config['METADATA'].items(): - flavor_metadata[key] = _get_metadata_from_configuration( - key, key, value, FLAVOR_FIELDS - ) - - packages_metadata = PACKAGES_METADATA - adapters_flavors = adapter_api.ADAPTERS_FLAVORS - for adapter_name, adapter_flavors in adapters_flavors.items(): - package_metadata = packages_metadata.get(adapter_name, {}) - for flavor_name, flavor in adapter_flavors.items(): - flavor_metadata = flavors_metadata.setdefault( - adapter_name, {} - ).setdefault(flavor_name, {}) - util.merge_dict(flavor_metadata, package_metadata, override=False) - return flavors_metadata - - -def _filter_metadata(metadata, **kwargs): - if not isinstance(metadata, dict): - return metadata - filtered_metadata = {} - for key, value in metadata.items(): - if key == '_self': - default_value = value.get('default_value', None) - if default_value is None: - default_callback_params = value.get( - 'default_callback_params', {} - ) - callback_params = dict(kwargs) - if default_callback_params: - callback_params.update(default_callback_params) - default_callback = value.get('default_callback', None) - if default_callback: - default_value = default_callback(key, **callback_params) - options = value.get('options', None) - if options is None: - options_callback_params = value.get( - 'options_callback_params', {} - ) - callback_params = dict(kwargs) - if options_callback_params: - callback_params.update(options_callback_params) - - options_callback = value.get('options_callback', None) - if options_callback: - options = options_callback(key, **callback_params) - filtered_metadata[key] = value - if default_value is not None: - filtered_metadata[key]['default_value'] = default_value - if options is not None: - filtered_metadata[key]['options'] = options - else: - filtered_metadata[key] = _filter_metadata(value, **kwargs) - return filtered_metadata - - -def _load_metadata(force_reload=False): - """Load metadata information into memory. - - If force_reload, the metadata information will be reloaded - even if the metadata is already loaded. - """ - adapter_api.load_adapters_internal(force_reload=force_reload) - global OS_FIELDS - if force_reload or OS_FIELDS is None: - OS_FIELDS = _get_os_fields_from_configuration() - global PACKAGE_FIELDS - if force_reload or PACKAGE_FIELDS is None: - PACKAGE_FIELDS = _get_package_fields_from_configuration() - global FLAVOR_FIELDS - if force_reload or FLAVOR_FIELDS is None: - FLAVOR_FIELDS = _get_flavor_fields_from_configuration() - global OSES_METADATA - if force_reload or OSES_METADATA is None: - OSES_METADATA = _get_oses_metadata_from_configuration() - global PACKAGES_METADATA - if force_reload or PACKAGES_METADATA is None: - PACKAGES_METADATA = _get_packages_metadata_from_configuration() - global FLAVORS_METADATA - if force_reload or FLAVORS_METADATA is None: - FLAVORS_METADATA = _get_flavors_metadata_from_configuration() - global OSES_METADATA_UI_CONVERTERS - if force_reload or OSES_METADATA_UI_CONVERTERS is None: - OSES_METADATA_UI_CONVERTERS = ( - _get_oses_metadata_ui_converters_from_configuration() - ) - global FLAVORS_METADATA_UI_CONVERTERS - if force_reload or FLAVORS_METADATA_UI_CONVERTERS is None: - FLAVORS_METADATA_UI_CONVERTERS = ( - _get_flavors_metadata_ui_converters_from_configuration() - ) - - -def _get_oses_metadata_ui_converters_from_configuration(): - """Get os metadata ui converters from os metadata mapping config dir. - - os metadata ui converter is used to convert os metadata to - the format UI can understand and show. - """ - oses_metadata_ui_converters = {} - configs = util.load_configs(setting.OS_MAPPING_DIR) - for config in configs: - os_name = config['OS'] - oses_metadata_ui_converters[os_name] = config.get('CONFIG_MAPPING', {}) - - oses = adapter_api.OSES - parents = {} - for os_name, os in oses.items(): - parent = os.get('parent', None) - parents[os_name] = parent - for os_name, os in oses.items(): - oses_metadata_ui_converters[os_name] = util.recursive_merge_dict( - os_name, oses_metadata_ui_converters, parents - ) - return oses_metadata_ui_converters - - -def _get_flavors_metadata_ui_converters_from_configuration(): - """Get flavor metadata ui converters from flavor mapping config dir.""" - flavors_metadata_ui_converters = {} - configs = util.load_configs(setting.FLAVOR_MAPPING_DIR) - for config in configs: - adapter_name = config['ADAPTER'] - flavor_name = config['FLAVOR'] - flavors_metadata_ui_converters.setdefault( - adapter_name, {} - )[flavor_name] = config.get('CONFIG_MAPPING', {}) - adapters = adapter_api.ADAPTERS - parents = {} - for adapter_name, adapter in adapters.items(): - parent = adapter.get('parent', None) - parents[adapter_name] = parent - for adapter_name, adapter in adapters.items(): - flavors_metadata_ui_converters[adapter_name] = ( - util.recursive_merge_dict( - adapter_name, flavors_metadata_ui_converters, parents - ) - ) - return flavors_metadata_ui_converters - - -def get_packages_metadata_internal(force_reload=False): - """Get deployable package metadata.""" - _load_metadata(force_reload=force_reload) - metadata_mapping = {} - adapters = adapter_api.ADAPTERS - for adapter_name, adapter in adapters.items(): - if adapter.get('deployable'): - metadata_mapping[adapter_name] = _filter_metadata( - PACKAGES_METADATA.get(adapter_name, {}) - ) - else: - logging.info( - 'ignore metadata since its adapter %s is not deployable', - adapter_name - ) - return metadata_mapping - - -def get_flavors_metadata_internal(force_reload=False): - """Get deployable flavor metadata.""" - _load_metadata(force_reload=force_reload) - metadata_mapping = {} - adapters_flavors = adapter_api.ADAPTERS_FLAVORS - for adapter_name, adapter_flavors in adapters_flavors.items(): - adapter = adapter_api.ADAPTERS[adapter_name] - if not adapter.get('deployable'): - logging.info( - 'ignore metadata since its adapter %s is not deployable', - adapter_name - ) - continue - for flavor_name, flavor in adapter_flavors.items(): - flavor_metadata = FLAVORS_METADATA.get( - adapter_name, {} - ).get(flavor_name, {}) - metadata = _filter_metadata(flavor_metadata) - metadata_mapping.setdefault( - adapter_name, {} - )[flavor_name] = metadata - return metadata_mapping - - -def get_flavors_metadata_ui_converters_internal(force_reload=False): - """Get usable flavor metadata ui converters.""" - _load_metadata(force_reload=force_reload) - return FLAVORS_METADATA_UI_CONVERTERS - - -def get_oses_metadata_internal(force_reload=False): - """Get deployable os metadata.""" - _load_metadata(force_reload=force_reload) - metadata_mapping = {} - oses = adapter_api.OSES - for os_name, os in oses.items(): - if os.get('deployable'): - metadata_mapping[os_name] = _filter_metadata( - OSES_METADATA.get(os_name, {}) - ) - else: - logging.info( - 'ignore metadata since its os %s is not deployable', - os_name - ) - return metadata_mapping - - -def get_oses_metadata_ui_converters_internal(force_reload=False): - """Get usable os metadata ui converters.""" - _load_metadata(force_reload=force_reload) - return OSES_METADATA_UI_CONVERTERS diff --git a/compass-tasks/db/api/metadata_holder.py b/compass-tasks/db/api/metadata_holder.py deleted file mode 100644 index 24afc67..0000000 --- a/compass-tasks/db/api/metadata_holder.py +++ /dev/null @@ -1,731 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Metadata related object holder.""" -import logging - -from compass.db.api import adapter as adapter_api -from compass.db.api import adapter_holder as adapter_holder_api -from compass.db.api import database -from compass.db.api import metadata as metadata_api -from compass.db.api import permission -from compass.db.api import user as user_api -from compass.db.api import utils -from compass.db import exception -from compass.db import models -from compass.utils import setting_wrapper as setting -from compass.utils import util - - -RESP_METADATA_FIELDS = [ - 'os_config', 'package_config' -] -RESP_UI_METADATA_FIELDS = [ - 'os_global_config', 'flavor_config' -] - - -def load_metadatas(force_reload=False): - """Load metadatas.""" - # TODO(xicheng): today we load metadata in memory as it original - # format in files in metadata.py. We get these inmemory metadata - # and do some translation, store the translated metadata into memory - # too in metadata_holder.py. api can only access the global inmemory - # data in metadata_holder.py. - _load_os_metadatas(force_reload=force_reload) - _load_package_metadatas(force_reload=force_reload) - _load_flavor_metadatas(force_reload=force_reload) - _load_os_metadata_ui_converters(force_reload=force_reload) - _load_flavor_metadata_ui_converters(force_reload=force_reload) - - -def _load_os_metadata_ui_converters(force_reload=False): - global OS_METADATA_UI_CONVERTERS - if force_reload or OS_METADATA_UI_CONVERTERS is None: - logging.info('load os metadatas ui converters into memory') - OS_METADATA_UI_CONVERTERS = ( - metadata_api.get_oses_metadata_ui_converters_internal( - force_reload=force_reload - ) - ) - - -def _load_os_metadatas(force_reload=False): - """Load os metadata from inmemory db and map it by os_id.""" - global OS_METADATA_MAPPING - if force_reload or OS_METADATA_MAPPING is None: - logging.info('load os metadatas into memory') - OS_METADATA_MAPPING = metadata_api.get_oses_metadata_internal( - force_reload=force_reload - ) - - -def _load_flavor_metadata_ui_converters(force_reload=False): - """Load flavor metadata ui converters from inmemory db. - - The loaded metadata is mapped by flavor id. - """ - global FLAVOR_METADATA_UI_CONVERTERS - if force_reload or FLAVOR_METADATA_UI_CONVERTERS is None: - logging.info('load flavor metadata ui converters into memory') - FLAVOR_METADATA_UI_CONVERTERS = {} - adapters_flavors_metadata_ui_converters = ( - metadata_api.get_flavors_metadata_ui_converters_internal( - force_reload=force_reload - ) - ) - for adapter_name, adapter_flavors_metadata_ui_converters in ( - adapters_flavors_metadata_ui_converters.items() - ): - for flavor_name, flavor_metadata_ui_converter in ( - adapter_flavors_metadata_ui_converters.items() - ): - FLAVOR_METADATA_UI_CONVERTERS[ - '%s:%s' % (adapter_name, flavor_name) - ] = flavor_metadata_ui_converter - - -@util.deprecated -def _load_package_metadatas(force_reload=False): - """Load deployable package metadata from inmemory db.""" - global PACKAGE_METADATA_MAPPING - if force_reload or PACKAGE_METADATA_MAPPING is None: - logging.info('load package metadatas into memory') - PACKAGE_METADATA_MAPPING = ( - metadata_api.get_packages_metadata_internal( - force_reload=force_reload - ) - ) - - -def _load_flavor_metadatas(force_reload=False): - """Load flavor metadata from inmemory db. - - The loaded metadata are mapped by flavor id. - """ - global FLAVOR_METADATA_MAPPING - if force_reload or FLAVOR_METADATA_MAPPING is None: - logging.info('load flavor metadatas into memory') - FLAVOR_METADATA_MAPPING = {} - adapters_flavors_metadata = ( - metadata_api.get_flavors_metadata_internal( - force_reload=force_reload - ) - ) - for adapter_name, adapter_flavors_metadata in ( - adapters_flavors_metadata.items() - ): - for flavor_name, flavor_metadata in ( - adapter_flavors_metadata.items() - ): - FLAVOR_METADATA_MAPPING[ - '%s:%s' % (adapter_name, flavor_name) - ] = flavor_metadata - - -OS_METADATA_MAPPING = None -PACKAGE_METADATA_MAPPING = None -FLAVOR_METADATA_MAPPING = None -OS_METADATA_UI_CONVERTERS = None -FLAVOR_METADATA_UI_CONVERTERS = None - - -def validate_os_config( - config, os_id, whole_check=False, **kwargs -): - """Validate os config.""" - load_metadatas() - if os_id not in OS_METADATA_MAPPING: - raise exception.InvalidParameter( - 'os %s is not found in os metadata mapping' % os_id - ) - _validate_config( - '', config, OS_METADATA_MAPPING[os_id], - whole_check, **kwargs - ) - - -@util.deprecated -def validate_package_config( - config, adapter_id, whole_check=False, **kwargs -): - """Validate package config.""" - load_metadatas() - if adapter_id not in PACKAGE_METADATA_MAPPING: - raise exception.InvalidParameter( - 'adapter %s is not found in package metedata mapping' % adapter_id - ) - _validate_config( - '', config, PACKAGE_METADATA_MAPPING[adapter_id], - whole_check, **kwargs - ) - - -def validate_flavor_config( - config, flavor_id, whole_check=False, **kwargs -): - """Validate flavor config.""" - load_metadatas() - if not flavor_id: - logging.info('There is no flavor, skipping flavor validation...') - elif flavor_id not in FLAVOR_METADATA_MAPPING: - raise exception.InvalidParameter( - 'flavor %s is not found in flavor metedata mapping' % flavor_id - ) - else: - _validate_config( - '', config, FLAVOR_METADATA_MAPPING[flavor_id], - whole_check, **kwargs - ) - - -def _filter_metadata(metadata, **kwargs): - """Filter metadata before return it to api. - - Some metadata fields are not json compatible or - only used in db/api internally. - We should strip these fields out before return to api. - """ - if not isinstance(metadata, dict): - return metadata - filtered_metadata = {} - for key, value in metadata.items(): - if key == '_self': - filtered_metadata[key] = { - 'name': value['name'], - 'description': value.get('description', None), - 'default_value': value.get('default_value', None), - 'is_required': value.get('is_required', False), - 'required_in_whole_config': value.get( - 'required_in_whole_config', False), - 'js_validator': value.get('js_validator', None), - 'options': value.get('options', None), - 'required_in_options': value.get( - 'required_in_options', False), - 'field_type': value.get( - 'field_type_data', 'str'), - 'display_type': value.get('display_type', None), - 'mapping_to': value.get('mapping_to', None) - } - else: - filtered_metadata[key] = _filter_metadata(value, **kwargs) - return filtered_metadata - - -@util.deprecated -def _get_package_metadata(adapter_id): - """get package metadata.""" - load_metadatas() - if adapter_id not in PACKAGE_METADATA_MAPPING: - raise exception.RecordNotExists( - 'adpater %s does not exist' % adapter_id - ) - return _filter_metadata( - PACKAGE_METADATA_MAPPING[adapter_id] - ) - - -@util.deprecated -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_METADATA_FIELDS) -def get_package_metadata(adapter_id, user=None, session=None, **kwargs): - """Get package metadata from adapter.""" - return { - 'package_config': _get_package_metadata(adapter_id) - } - - -def _get_flavor_metadata(flavor_id): - """get flavor metadata.""" - load_metadatas() - if not flavor_id: - logging.info('There is no flavor id, skipping...') - elif flavor_id not in FLAVOR_METADATA_MAPPING: - raise exception.RecordNotExists( - 'flavor %s does not exist' % flavor_id - ) - else: - return _filter_metadata(FLAVOR_METADATA_MAPPING[flavor_id]) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_METADATA_FIELDS) -def get_flavor_metadata(flavor_id, user=None, session=None, **kwargs): - """Get flavor metadata by flavor.""" - return { - 'package_config': _get_flavor_metadata(flavor_id) - } - - -def _get_os_metadata(os_id): - """get os metadata.""" - load_metadatas() - if os_id not in OS_METADATA_MAPPING: - raise exception.RecordNotExists( - 'os %s does not exist' % os_id - ) - return _filter_metadata(OS_METADATA_MAPPING[os_id]) - - -def _get_os_metadata_ui_converter(os_id): - """get os metadata ui converter.""" - load_metadatas() - if os_id not in OS_METADATA_UI_CONVERTERS: - raise exception.RecordNotExists( - 'os %s does not exist' % os_id - ) - return OS_METADATA_UI_CONVERTERS[os_id] - - -def _get_flavor_metadata_ui_converter(flavor_id): - """get flavor metadata ui converter.""" - load_metadatas() - if flavor_id not in FLAVOR_METADATA_UI_CONVERTERS: - raise exception.RecordNotExists( - 'flavor %s does not exist' % flavor_id - ) - return FLAVOR_METADATA_UI_CONVERTERS[flavor_id] - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_METADATA_FIELDS) -def get_os_metadata(os_id, user=None, session=None, **kwargs): - """get os metadatas.""" - return {'os_config': _get_os_metadata(os_id)} - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_UI_METADATA_FIELDS) -def get_os_ui_metadata(os_id, user=None, session=None, **kwargs): - """Get os metadata ui converter by os.""" - metadata = _get_os_metadata(os_id) - metadata_ui_converter = _get_os_metadata_ui_converter(os_id) - return _get_ui_metadata(metadata, metadata_ui_converter) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_UI_METADATA_FIELDS) -def get_flavor_ui_metadata(flavor_id, user=None, session=None, **kwargs): - """Get flavor ui metadata by flavor.""" - metadata = _get_flavor_metadata(flavor_id) - metadata_ui_converter = _get_flavor_metadata_ui_converter(flavor_id) - return _get_ui_metadata(metadata, metadata_ui_converter) - - -def _get_ui_metadata(metadata, metadata_ui_converter): - """convert metadata to ui metadata. - - Args: - metadata: metadata we defined in metadata files. - metadata_ui_converter: metadata ui converter defined in metadata - mapping files. Used to convert orignal - metadata to ui understandable metadata. - - Returns: - ui understandable metadata. - """ - ui_metadata = {} - ui_metadata[metadata_ui_converter['mapped_name']] = [] - for mapped_child in metadata_ui_converter['mapped_children']: - data_dict = {} - for ui_key, ui_value in mapped_child.items(): - for key, value in ui_value.items(): - if 'data' == key: - result_data = [] - _get_ui_metadata_data( - metadata[ui_key], value, result_data - ) - data_dict['data'] = result_data - else: - data_dict[key] = value - ui_metadata[metadata_ui_converter['mapped_name']].append(data_dict) - return ui_metadata - - -def _get_ui_metadata_data(metadata, config, result_data): - """Get ui metadata data and fill to result.""" - data_dict = {} - for key, config_value in config.items(): - if isinstance(config_value, dict) and key != 'content_data': - if key in metadata.keys(): - _get_ui_metadata_data(metadata[key], config_value, result_data) - else: - _get_ui_metadata_data(metadata, config_value, result_data) - elif isinstance(config_value, list): - option_list = [] - for item in config_value: - if isinstance(item, dict): - option_list.append(item) - data_dict[key] = option_list - else: - if isinstance(metadata['_self'][item], bool): - data_dict[item] = str(metadata['_self'][item]).lower() - else: - data_dict[item] = metadata['_self'][item] - else: - data_dict[key] = config_value - if data_dict: - result_data.append(data_dict) - return result_data - - -@util.deprecated -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_METADATA_FIELDS) -def get_package_os_metadata( - adapter_id, os_id, - user=None, session=None, **kwargs -): - """Get metadata by adapter and os.""" - adapter = adapter_holder_api.get_adapter( - adapter_id, user=user, session=session - ) - os_ids = [os['id'] for os in adapter['supported_oses']] - if os_id not in os_ids: - raise exception.InvalidParameter( - 'os %s is not in the supported os list of adapter %s' % ( - os_id, adapter_id - ) - ) - metadatas = {} - metadatas['os_config'] = _get_os_metadata( - os_id - ) - metadatas['package_config'] = _get_package_metadata( - adapter_id - ) - return metadatas - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_METADATA_FIELDS) -def get_flavor_os_metadata( - flavor_id, os_id, - user=None, session=None, **kwargs -): - """Get metadata by flavor and os.""" - flavor = adapter_holder_api.get_flavor( - flavor_id, user=user, session=session - ) - adapter_id = flavor['adapter_id'] - adapter = adapter_holder_api.get_adapter( - adapter_id, user=user, session=session - ) - os_ids = [os['id'] for os in adapter['supported_oses']] - if os_id not in os_ids: - raise exception.InvalidParameter( - 'os %s is not in the supported os list of adapter %s' % ( - os_id, adapter_id - ) - ) - metadatas = {} - metadatas['os_config'] = _get_os_metadata( - session, os_id - ) - metadatas['package_config'] = _get_flavor_metadata( - session, flavor_id - ) - return metadatas - - -def _validate_self( - config_path, config_key, config, - metadata, whole_check, - **kwargs -): - """validate config by metadata self section.""" - logging.debug('validate config self %s', config_path) - if '_self' not in metadata: - if isinstance(config, dict): - _validate_config( - config_path, config, metadata, whole_check, **kwargs - ) - return - field_type = metadata['_self'].get('field_type', basestring) - if not isinstance(config, field_type): - raise exception.InvalidParameter( - '%s config type is not %s: %s' % (config_path, field_type, config) - ) - is_required = metadata['_self'].get( - 'is_required', False - ) - required_in_whole_config = metadata['_self'].get( - 'required_in_whole_config', False - ) - if isinstance(config, basestring): - if config == '' and not is_required and not required_in_whole_config: - # ignore empty config when it is optional - return - required_in_options = metadata['_self'].get( - 'required_in_options', False - ) - options = metadata['_self'].get('options', None) - if required_in_options: - if field_type in [int, basestring, float, bool]: - if options and config not in options: - raise exception.InvalidParameter( - '%s config is not in %s: %s' % ( - config_path, options, config - ) - ) - elif field_type in [list, tuple]: - if options and not set(config).issubset(set(options)): - raise exception.InvalidParameter( - '%s config is not in %s: %s' % ( - config_path, options, config - ) - ) - elif field_type == dict: - if options and not set(config.keys()).issubset(set(options)): - raise exception.InvalidParameter( - '%s config is not in %s: %s' % ( - config_path, options, config - ) - ) - validator = metadata['_self'].get('validator', None) - logging.debug('validate by validator %s', validator) - if validator: - if not validator(config_key, config, **kwargs): - raise exception.InvalidParameter( - '%s config is invalid' % config_path - ) - if isinstance(config, dict): - _validate_config( - config_path, config, metadata, whole_check, **kwargs - ) - - -def _validate_config( - config_path, config, metadata, whole_check, - **kwargs -): - """validate config by metadata.""" - logging.debug('validate config %s', config_path) - generals = {} - specified = {} - for key, value in metadata.items(): - if key.startswith('$'): - generals[key] = value - elif key.startswith('_'): - pass - else: - specified[key] = value - config_keys = set(config.keys()) - specified_keys = set(specified.keys()) - intersect_keys = config_keys & specified_keys - not_found_keys = config_keys - specified_keys - redundant_keys = specified_keys - config_keys - for key in redundant_keys: - if '_self' not in specified[key]: - continue - if specified[key]['_self'].get('is_required', False): - raise exception.InvalidParameter( - '%s/%s does not find but it is required' % ( - config_path, key - ) - ) - if ( - whole_check and - specified[key]['_self'].get( - 'required_in_whole_config', False - ) - ): - raise exception.InvalidParameter( - '%s/%s does not find but it is required in whole config' % ( - config_path, key - ) - ) - for key in intersect_keys: - _validate_self( - '%s/%s' % (config_path, key), - key, config[key], specified[key], whole_check, - **kwargs - ) - for key in not_found_keys: - if not generals: - raise exception.InvalidParameter( - 'key %s missing in metadata %s' % ( - key, config_path - ) - ) - for general_key, general_value in generals.items(): - _validate_self( - '%s/%s' % (config_path, key), - key, config[key], general_value, whole_check, - **kwargs - ) - - -def _autofill_self_config( - config_path, config_key, config, - metadata, - **kwargs -): - """Autofill config by metadata self section.""" - if '_self' not in metadata: - if isinstance(config, dict): - _autofill_config( - config_path, config, metadata, **kwargs - ) - return config - logging.debug( - 'autofill %s by metadata %s', config_path, metadata['_self'] - ) - autofill_callback = metadata['_self'].get( - 'autofill_callback', None - ) - autofill_callback_params = metadata['_self'].get( - 'autofill_callback_params', {} - ) - callback_params = dict(kwargs) - if autofill_callback_params: - callback_params.update(autofill_callback_params) - default_value = metadata['_self'].get( - 'default_value', None - ) - if default_value is not None: - callback_params['default_value'] = default_value - options = metadata['_self'].get( - 'options', None - ) - if options is not None: - callback_params['options'] = options - if autofill_callback: - config = autofill_callback( - config_key, config, **callback_params - ) - if config is None: - new_config = {} - else: - new_config = config - if isinstance(new_config, dict): - _autofill_config( - config_path, new_config, metadata, **kwargs - ) - if new_config: - config = new_config - return config - - -def _autofill_config( - config_path, config, metadata, **kwargs -): - """autofill config by metadata.""" - generals = {} - specified = {} - for key, value in metadata.items(): - if key.startswith('$'): - generals[key] = value - elif key.startswith('_'): - pass - else: - specified[key] = value - config_keys = set(config.keys()) - specified_keys = set(specified.keys()) - intersect_keys = config_keys & specified_keys - not_found_keys = config_keys - specified_keys - redundant_keys = specified_keys - config_keys - for key in redundant_keys: - self_config = _autofill_self_config( - '%s/%s' % (config_path, key), - key, None, specified[key], **kwargs - ) - if self_config is not None: - config[key] = self_config - for key in intersect_keys: - config[key] = _autofill_self_config( - '%s/%s' % (config_path, key), - key, config[key], specified[key], - **kwargs - ) - for key in not_found_keys: - for general_key, general_value in generals.items(): - config[key] = _autofill_self_config( - '%s/%s' % (config_path, key), - key, config[key], general_value, - **kwargs - ) - return config - - -def autofill_os_config( - config, os_id, **kwargs -): - load_metadatas() - if os_id not in OS_METADATA_MAPPING: - raise exception.InvalidParameter( - 'os %s is not found in os metadata mapping' % os_id - ) - - return _autofill_config( - '', config, OS_METADATA_MAPPING[os_id], **kwargs - ) - - -def autofill_package_config( - config, adapter_id, **kwargs -): - load_metadatas() - if adapter_id not in PACKAGE_METADATA_MAPPING: - raise exception.InvalidParameter( - 'adapter %s is not found in package metadata mapping' % adapter_id - ) - - return _autofill_config( - '', config, PACKAGE_METADATA_MAPPING[adapter_id], **kwargs - ) - - -def autofill_flavor_config( - config, flavor_id, **kwargs -): - load_metadatas() - if not flavor_id: - logging.info('There is no flavor, skipping...') - elif flavor_id not in FLAVOR_METADATA_MAPPING: - raise exception.InvalidParameter( - 'flavor %s is not found in flavor metadata mapping' % flavor_id - ) - else: - return _autofill_config( - '', config, FLAVOR_METADATA_MAPPING[flavor_id], **kwargs - ) diff --git a/compass-tasks/db/api/network.py b/compass-tasks/db/api/network.py deleted file mode 100644 index e2bf7d3..0000000 --- a/compass-tasks/db/api/network.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Network related database operations.""" -import logging -import netaddr -import re - -from compass.db.api import database -from compass.db.api import permission -from compass.db.api import user as user_api -from compass.db.api import utils -from compass.db import exception -from compass.db import models - - -SUPPORTED_FIELDS = ['subnet', 'name'] -RESP_FIELDS = [ - 'id', 'name', 'subnet', 'created_at', 'updated_at' -] -ADDED_FIELDS = ['subnet'] -OPTIONAL_ADDED_FIELDS = ['name'] -IGNORE_FIELDS = [ - 'id', 'created_at', 'updated_at' -] -UPDATED_FIELDS = ['subnet', 'name'] - - -def _check_subnet(subnet): - """Check subnet format is correct.""" - try: - netaddr.IPNetwork(subnet) - except Exception as error: - logging.exception(error) - raise exception.InvalidParameter( - 'subnet %s format unrecognized' % subnet) - - -@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_SUBNETS -) -@utils.wrap_to_dict(RESP_FIELDS) -def list_subnets(user=None, session=None, **filters): - """List subnets.""" - return utils.list_db_objects( - session, models.Subnet, **filters - ) - - -def _get_subnet(subnet_id, session=None, **kwargs): - """Get subnet by subnet id.""" - if isinstance(subnet_id, (int, long)): - return utils.get_db_object( - session, models.Subnet, - id=subnet_id, **kwargs - ) - raise exception.InvalidParameter( - 'subnet id %s type is not int compatible' % subnet_id - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_SUBNETS -) -@utils.wrap_to_dict(RESP_FIELDS) -def get_subnet( - subnet_id, exception_when_missing=True, - user=None, session=None, **kwargs -): - """Get subnet info.""" - return _get_subnet( - subnet_id, session=session, - exception_when_missing=exception_when_missing - ) - - -@utils.supported_filters( - ADDED_FIELDS, optional_support_keys=OPTIONAL_ADDED_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates(subnet=_check_subnet) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_SUBNET -) -@utils.wrap_to_dict(RESP_FIELDS) -def add_subnet( - exception_when_existing=True, subnet=None, - user=None, session=None, **kwargs -): - """Create a subnet.""" - return utils.add_db_object( - session, models.Subnet, - exception_when_existing, subnet, **kwargs - ) - - -@utils.supported_filters( - optional_support_keys=UPDATED_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates(subnet=_check_subnet) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_SUBNET -) -@utils.wrap_to_dict(RESP_FIELDS) -def update_subnet(subnet_id, user=None, session=None, **kwargs): - """Update a subnet.""" - subnet = _get_subnet( - subnet_id, session=session - ) - return utils.update_db_object(session, subnet, **kwargs) - - -def _check_subnet_deletable(subnet): - """Check a subnet deletable.""" - if subnet.host_networks: - host_networks = [ - '%s:%s=%s' % ( - host_network.host.name, host_network.interface, - host_network.ip - ) - for host_network in subnet.host_networks - ] - raise exception.NotAcceptable( - 'subnet %s contains host networks %s' % ( - subnet.subnet, host_networks - ) - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEL_SUBNET -) -@utils.wrap_to_dict(RESP_FIELDS) -def del_subnet(subnet_id, user=None, session=None, **kwargs): - """Delete a subnet.""" - subnet = _get_subnet( - subnet_id, session=session - ) - _check_subnet_deletable(subnet) - return utils.del_db_object(session, subnet) diff --git a/compass-tasks/db/api/permission.py b/compass-tasks/db/api/permission.py deleted file mode 100644 index f4d777a..0000000 --- a/compass-tasks/db/api/permission.py +++ /dev/null @@ -1,357 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Permission database operations.""" -import re - -from compass.db.api import database -from compass.db.api import user as user_api -from compass.db.api import utils -from compass.db import exception -from compass.db import models -from compass.utils import util - - -SUPPORTED_FIELDS = ['id', 'name', 'alias', 'description'] -RESP_FIELDS = ['id', 'name', 'alias', 'description'] - - -class PermissionWrapper(object): - def __init__(self, name, alias, description): - self.name = name - self.alias = alias - self.description = description - - def to_dict(self): - return { - 'name': self.name, - 'alias': self.alias, - 'description': self.description - } - - -PERMISSION_LIST_PERMISSIONS = PermissionWrapper( - 'list_permissions', 'list permissions', 'list all permissions' -) -PERMISSION_LIST_SWITCHES = PermissionWrapper( - 'list_switches', 'list switches', 'list all switches' -) -PERMISSION_LIST_SWITCH_FILTERS = PermissionWrapper( - 'list_switch_filters', - 'list switch filters', - 'list switch filters' -) -PERMISSION_ADD_SWITCH = PermissionWrapper( - 'add_switch', 'add switch', 'add switch' -) -PERMISSION_UPDATE_SWITCH_FILTERS = PermissionWrapper( - 'update_switch_filters', - 'update switch filters', - 'update switch filters' -) -PERMISSION_DEL_SWITCH = PermissionWrapper( - 'delete_switch', 'delete switch', 'delete switch' -) -PERMISSION_LIST_SWITCH_MACHINES = PermissionWrapper( - 'list_switch_machines', 'list switch machines', 'list switch machines' -) -PERMISSION_ADD_SWITCH_MACHINE = PermissionWrapper( - 'add_switch_machine', 'add switch machine', 'add switch machine' -) -PERMISSION_DEL_SWITCH_MACHINE = PermissionWrapper( - 'del_switch_machine', 'delete switch machine', 'del switch machine' -) -PERMISSION_UPDATE_SWITCH_MACHINES = PermissionWrapper( - 'update_switch_machines', - 'update switch machines', - 'update switch machines' -) -PERMISSION_LIST_MACHINES = PermissionWrapper( - 'list_machines', 'list machines', 'list machines' -) -PERMISSION_ADD_MACHINE = PermissionWrapper( - 'add_machine', 'add machine', 'add machine' -) -PERMISSION_DEL_MACHINE = PermissionWrapper( - 'delete_machine', 'delete machine', 'delete machine' -) -PERMISSION_LIST_ADAPTERS = PermissionWrapper( - 'list_adapters', 'list adapters', 'list adapters' -) -PERMISSION_LIST_METADATAS = PermissionWrapper( - 'list_metadatas', 'list metadatas', 'list metadatas' -) -PERMISSION_LIST_SUBNETS = PermissionWrapper( - 'list_subnets', 'list subnets', 'list subnets' -) -PERMISSION_ADD_SUBNET = PermissionWrapper( - 'add_subnet', 'add subnet', 'add subnet' -) -PERMISSION_DEL_SUBNET = PermissionWrapper( - 'del_subnet', 'del subnet', 'del subnet' -) -PERMISSION_LIST_CLUSTERS = PermissionWrapper( - 'list_clusters', 'list clusters', 'list clusters' -) -PERMISSION_ADD_CLUSTER = PermissionWrapper( - 'add_cluster', 'add cluster', 'add cluster' -) -PERMISSION_DEL_CLUSTER = PermissionWrapper( - 'del_cluster', 'del cluster', 'del cluster' -) -PERMISSION_LIST_CLUSTER_CONFIG = PermissionWrapper( - 'list_cluster_config', 'list cluster config', 'list cluster config' -) -PERMISSION_ADD_CLUSTER_CONFIG = PermissionWrapper( - 'add_cluster_config', 'add cluster config', 'add cluster config' -) -PERMISSION_DEL_CLUSTER_CONFIG = PermissionWrapper( - 'del_cluster_config', 'del cluster config', 'del cluster config' -) -PERMISSION_UPDATE_CLUSTER_HOSTS = PermissionWrapper( - 'update_cluster_hosts', - 'update cluster hosts', - 'update cluster hosts' -) -PERMISSION_DEL_CLUSTER_HOST = PermissionWrapper( - 'del_clusterhost', 'delete clusterhost', 'delete clusterhost' -) -PERMISSION_REVIEW_CLUSTER = PermissionWrapper( - 'review_cluster', 'review cluster', 'review cluster' -) -PERMISSION_DEPLOY_CLUSTER = PermissionWrapper( - 'deploy_cluster', 'deploy cluster', 'deploy cluster' -) -PERMISSION_DEPLOY_HOST = PermissionWrapper( - 'deploy_host', 'deploy host', 'deploy host' -) -PERMISSION_GET_CLUSTER_STATE = PermissionWrapper( - 'get_cluster_state', 'get cluster state', 'get cluster state' -) -PERMISSION_UPDATE_CLUSTER_STATE = PermissionWrapper( - 'update_cluster_state', 'update cluster state', - 'update cluster state' -) -PERMISSION_LIST_HOSTS = PermissionWrapper( - 'list_hosts', 'list hosts', 'list hosts' -) -PERMISSION_LIST_HOST_CLUSTERS = PermissionWrapper( - 'list_host_clusters', - 'list host clusters', - 'list host clusters' -) -PERMISSION_UPDATE_HOST = PermissionWrapper( - 'update_host', 'update host', 'update host' -) -PERMISSION_DEL_HOST = PermissionWrapper( - 'del_host', 'del host', 'del host' -) -PERMISSION_LIST_HOST_CONFIG = PermissionWrapper( - 'list_host_config', 'list host config', 'list host config' -) -PERMISSION_ADD_HOST_CONFIG = PermissionWrapper( - 'add_host_config', 'add host config', 'add host config' -) -PERMISSION_DEL_HOST_CONFIG = PermissionWrapper( - 'del_host_config', 'del host config', 'del host config' -) -PERMISSION_LIST_HOST_NETWORKS = PermissionWrapper( - 'list_host_networks', - 'list host networks', - 'list host networks' -) -PERMISSION_ADD_HOST_NETWORK = PermissionWrapper( - 'add_host_network', 'add host network', 'add host network' -) -PERMISSION_DEL_HOST_NETWORK = PermissionWrapper( - 'del_host_network', 'del host network', 'del host network' -) -PERMISSION_GET_HOST_STATE = PermissionWrapper( - 'get_host_state', 'get host state', 'get host state' -) -PERMISSION_UPDATE_HOST_STATE = PermissionWrapper( - 'update_host_state', 'update host sate', 'update host state' -) -PERMISSION_LIST_CLUSTERHOSTS = PermissionWrapper( - 'list_clusterhosts', 'list cluster hosts', 'list cluster hosts' -) -PERMISSION_LIST_CLUSTERHOST_CONFIG = PermissionWrapper( - 'list_clusterhost_config', - 'list clusterhost config', - 'list clusterhost config' -) -PERMISSION_ADD_CLUSTERHOST_CONFIG = PermissionWrapper( - 'add_clusterhost_config', - 'add clusterhost config', - 'add clusterhost config' -) -PERMISSION_DEL_CLUSTERHOST_CONFIG = PermissionWrapper( - 'del_clusterhost_config', - 'del clusterhost config', - 'del clusterhost config' -) -PERMISSION_GET_CLUSTERHOST_STATE = PermissionWrapper( - 'get_clusterhost_state', - 'get clusterhost state', - 'get clusterhost state' -) -PERMISSION_UPDATE_CLUSTERHOST_STATE = PermissionWrapper( - 'update_clusterhost_state', - 'update clusterhost state', - 'update clusterhost state' -) -PERMISSION_LIST_HEALTH_REPORT = PermissionWrapper( - 'list_health_reports', - 'list health check report', - 'list health check report' -) -PERMISSION_GET_HEALTH_REPORT = PermissionWrapper( - 'get_health_report', - 'get health report', - 'get health report' -) -PERMISSION_CHECK_CLUSTER_HEALTH = PermissionWrapper( - 'start_check_cluster_health', - 'start check cluster health', - 'start check cluster health' -) -PERMISSION_SET_HEALTH_CHECK_ERROR = PermissionWrapper( - 'set_error_state', - 'set health check into error state', - 'set health check into error state' -) -PERMISSION_DELETE_REPORT = PermissionWrapper( - 'delete_reports', - 'delete health reports', - 'delete health reports' -) -PERMISSIONS = [ - PERMISSION_LIST_PERMISSIONS, - PERMISSION_LIST_SWITCHES, - PERMISSION_ADD_SWITCH, - PERMISSION_DEL_SWITCH, - PERMISSION_LIST_SWITCH_FILTERS, - PERMISSION_UPDATE_SWITCH_FILTERS, - PERMISSION_LIST_SWITCH_MACHINES, - PERMISSION_ADD_SWITCH_MACHINE, - PERMISSION_DEL_SWITCH_MACHINE, - PERMISSION_UPDATE_SWITCH_MACHINES, - PERMISSION_LIST_MACHINES, - PERMISSION_ADD_MACHINE, - PERMISSION_DEL_MACHINE, - PERMISSION_LIST_ADAPTERS, - PERMISSION_LIST_METADATAS, - PERMISSION_LIST_SUBNETS, - PERMISSION_ADD_SUBNET, - PERMISSION_DEL_SUBNET, - PERMISSION_LIST_CLUSTERS, - PERMISSION_ADD_CLUSTER, - PERMISSION_DEL_CLUSTER, - PERMISSION_LIST_CLUSTER_CONFIG, - PERMISSION_ADD_CLUSTER_CONFIG, - PERMISSION_DEL_CLUSTER_CONFIG, - PERMISSION_UPDATE_CLUSTER_HOSTS, - PERMISSION_DEL_CLUSTER_HOST, - PERMISSION_REVIEW_CLUSTER, - PERMISSION_DEPLOY_CLUSTER, - PERMISSION_GET_CLUSTER_STATE, - PERMISSION_UPDATE_CLUSTER_STATE, - PERMISSION_LIST_HOSTS, - PERMISSION_LIST_HOST_CLUSTERS, - PERMISSION_UPDATE_HOST, - PERMISSION_DEL_HOST, - PERMISSION_LIST_HOST_CONFIG, - PERMISSION_ADD_HOST_CONFIG, - PERMISSION_DEL_HOST_CONFIG, - PERMISSION_LIST_HOST_NETWORKS, - PERMISSION_ADD_HOST_NETWORK, - PERMISSION_DEL_HOST_NETWORK, - PERMISSION_GET_HOST_STATE, - PERMISSION_UPDATE_HOST_STATE, - PERMISSION_DEPLOY_HOST, - PERMISSION_LIST_CLUSTERHOSTS, - PERMISSION_LIST_CLUSTERHOST_CONFIG, - PERMISSION_ADD_CLUSTERHOST_CONFIG, - PERMISSION_DEL_CLUSTERHOST_CONFIG, - PERMISSION_GET_CLUSTERHOST_STATE, - PERMISSION_UPDATE_CLUSTERHOST_STATE, - PERMISSION_LIST_HEALTH_REPORT, - PERMISSION_GET_HEALTH_REPORT, - PERMISSION_CHECK_CLUSTER_HEALTH, - PERMISSION_SET_HEALTH_CHECK_ERROR, - PERMISSION_DELETE_REPORT -] - - -@util.deprecated -def list_permissions_internal(session, **filters): - """internal functions used only by other db.api modules.""" - return utils.list_db_objects(session, models.Permission, **filters) - - -@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) -@database.run_in_session() -@user_api.check_user_permission(PERMISSION_LIST_PERMISSIONS) -@utils.wrap_to_dict(RESP_FIELDS) -def list_permissions(user=None, session=None, **filters): - """list permissions.""" - return utils.list_db_objects( - session, models.Permission, **filters - ) - - -def _get_permission(permission_id, session=None, **kwargs): - """Get permission object by the unique key of Permission table.""" - if isinstance(permission_id, (int, long)): - return utils.get_db_object( - session, models.Permission, id=permission_id, **kwargs) - raise exception.InvalidParameter( - 'permission id %s type is not int compatible' % permission_id - ) - - -def get_permission_internal(permission_id, session=None, **kwargs): - return _get_permission(permission_id, session=session, **kwargs) - - -@utils.supported_filters() -@database.run_in_session() -@user_api.check_user_permission(PERMISSION_LIST_PERMISSIONS) -@utils.wrap_to_dict(RESP_FIELDS) -def get_permission( - permission_id, exception_when_missing=True, - user=None, session=None, **kwargs -): - """get permissions.""" - return _get_permission( - permission_id, session=session, - exception_when_missing=exception_when_missing - ) - - -def add_permissions_internal(session=None): - """internal functions used by other db.api modules only.""" - permissions = [] - for permission in PERMISSIONS: - permissions.append( - utils.add_db_object( - session, models.Permission, - True, - permission.name, - alias=permission.alias, - description=permission.description - ) - ) - - return permissions diff --git a/compass-tasks/db/api/switch.py b/compass-tasks/db/api/switch.py deleted file mode 100644 index 647eec0..0000000 --- a/compass-tasks/db/api/switch.py +++ /dev/null @@ -1,1213 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Switch database operations.""" -import logging -import netaddr -import re - -from compass.db.api import database -from compass.db.api import permission -from compass.db.api import user as user_api -from compass.db.api import utils -from compass.db import exception -from compass.db import models -from compass.utils import setting_wrapper as setting -from compass.utils import util - - -SUPPORTED_FIELDS = ['ip_int', 'vendor', 'state'] -SUPPORTED_FILTER_FIELDS = ['ip_int', 'vendor', 'state'] -SUPPORTED_SWITCH_MACHINES_FIELDS = [ - 'switch_ip_int', 'port', 'vlans', 'mac', 'tag', 'location', - 'owner_id' -] -SUPPORTED_MACHINES_FIELDS = [ - 'port', 'vlans', 'mac', 'tag', 'location', 'owner_id' -] -SUPPORTED_SWITCH_MACHINES_HOSTS_FIELDS = [ - 'switch_ip_int', 'port', 'vlans', 'mac', - 'tag', 'location', 'os_name' -] -SUPPORTED_MACHINES_HOSTS_FIELDS = [ - 'port', 'vlans', 'mac', 'tag', 'location', - 'os_name' -] -IGNORE_FIELDS = ['id', 'created_at', 'updated_at'] -ADDED_FIELDS = ['ip'] -OPTIONAL_ADDED_FIELDS = [ - 'credentials', 'vendor', 'state', 'err_msg', 'machine_filters' -] -UPDATED_FIELDS = [ - 'ip', 'credentials', 'vendor', 'state', - 'err_msg', 'put_machine_filters' -] -PATCHED_FIELDS = ['patched_credentials', 'patched_machine_filters'] -UPDATED_FILTERS_FIELDS = ['put_machine_filters'] -PATCHED_FILTERS_FIELDS = ['patched_machine_filters'] -ADDED_MACHINES_FIELDS = ['mac'] -OPTIONAL_ADDED_MACHINES_FIELDS = [ - 'ipmi_credentials', 'tag', 'location', 'owner_id' -] -ADDED_SWITCH_MACHINES_FIELDS = ['port'] -OPTIONAL_ADDED_SWITCH_MACHINES_FIELDS = ['vlans'] -UPDATED_MACHINES_FIELDS = [ - 'ipmi_credentials', - 'tag', 'location' -] -UPDATED_SWITCH_MACHINES_FIELDS = ['port', 'vlans', 'owner_id'] -PATCHED_MACHINES_FIELDS = [ - 'patched_ipmi_credentials', - 'patched_tag', 'patched_location' -] -PATCHED_SWITCH_MACHINES_FIELDS = ['patched_vlans'] -RESP_FIELDS = [ - 'id', 'ip', 'credentials', 'vendor', 'state', 'err_msg', - 'filters', 'created_at', 'updated_at' -] -RESP_FILTERS_FIELDS = [ - 'id', 'ip', 'filters', 'created_at', 'updated_at' -] -RESP_ACTION_FIELDS = [ - 'status', 'details' -] -RESP_MACHINES_FIELDS = [ - 'id', 'switch_id', 'switch_ip', 'machine_id', 'switch_machine_id', - 'port', 'vlans', 'mac', 'owner_id', - 'ipmi_credentials', 'tag', 'location', - 'created_at', 'updated_at' -] -RESP_MACHINES_HOSTS_FIELDS = [ - 'id', 'switch_id', 'switch_ip', 'machine_id', 'switch_machine_id', - 'port', 'vlans', 'mac', - 'ipmi_credentials', 'tag', 'location', 'ip', - 'name', 'hostname', 'os_name', 'owner', - 'os_installer', 'reinstall_os', 'os_installed', - 'clusters', 'created_at', 'updated_at' -] -RESP_CLUSTER_FIELDS = [ - 'name', 'id' -] - - -def _check_machine_filters(machine_filters): - """Check if machine filters format is acceptable.""" - logging.debug('check machine filters: %s', machine_filters) - models.Switch.parse_filters(machine_filters) - - -def _check_vlans(vlans): - """Check vlans format is acceptable.""" - for vlan in vlans: - if not isinstance(vlan, int): - raise exception.InvalidParameter( - 'vlan %s is not int' % vlan - ) - - -@utils.supported_filters( - ADDED_FIELDS, - optional_support_keys=OPTIONAL_ADDED_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates( - ip=utils.check_ip, - credentials=utils.check_switch_credentials, - machine_filters=_check_machine_filters -) -@utils.wrap_to_dict(RESP_FIELDS) -def _add_switch( - ip, exception_when_existing=True, - machine_filters=setting.SWITCHES_DEFAULT_FILTERS, - session=None, **kwargs -): - """Add switch by switch ip.""" - ip_int = long(netaddr.IPAddress(ip)) - return utils.add_db_object( - session, models.Switch, exception_when_existing, ip_int, - machine_filters=machine_filters, **kwargs - ) - - -def get_switch_internal( - switch_id, session=None, **kwargs -): - """Get switch by switch id. - - Should only be used by other files under db/api - """ - return _get_switch(switch_id, session=session, **kwargs) - - -def _get_switch(switch_id, session=None, **kwargs): - """Get Switch object switch id.""" - if isinstance(switch_id, (int, long)): - return utils.get_db_object( - session, models.Switch, - id=switch_id, **kwargs - ) - raise exception.InvalidParameter( - 'switch id %s type is not int compatible' % switch_id) - - -def _get_switch_by_ip(switch_ip, session=None, **kwargs): - """Get switch by switch ip.""" - switch_ip_int = long(netaddr.IPAddress(switch_ip)) - return utils.get_db_object( - session, models.Switch, - ip_int=switch_ip_int, **kwargs - ) - - -def _get_switch_machine(switch_id, machine_id, session=None, **kwargs): - """Get switch machine by switch id and machine id.""" - switch = _get_switch(switch_id, session=session) - from compass.db.api import machine as machine_api - machine = machine_api.get_machine_internal(machine_id, session=session) - return utils.get_db_object( - session, models.SwitchMachine, - switch_id=switch.id, machine_id=machine.id, **kwargs - ) - - -def _get_switchmachine(switch_machine_id, session=None, **kwargs): - """Get switch machine by switch_machine_id.""" - if not isinstance(switch_machine_id, (int, long)): - raise exception.InvalidParameter( - 'switch machine id %s type is not int compatible' % ( - switch_machine_id - ) - ) - return utils.get_db_object( - session, models.SwitchMachine, - switch_machine_id=switch_machine_id, **kwargs - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_SWITCHES -) -@utils.wrap_to_dict(RESP_FIELDS) -def get_switch( - switch_id, exception_when_missing=True, - user=None, session=None, **kwargs -): - """get a switch by switch id.""" - return _get_switch( - switch_id, session=session, - exception_when_missing=exception_when_missing - ) - - -@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_SWITCHES -) -@utils.wrap_to_dict(RESP_FIELDS) -def list_switches(user=None, session=None, **filters): - """List switches.""" - # TODO(xicheng): should discuss with weidong. - # If we can deprecate the use of DEFAULT_SWITCH_IP, - # The code will be simpler. - # The UI should use /machines-hosts instead of - # /switches-machines-hosts and can show multi switch ip/port - # under one row of machine info. - switches = utils.list_db_objects( - session, models.Switch, **filters - ) - if 'ip_int' in filters: - return switches - else: - return [ - switch for switch in switches - if switch.ip != setting.DEFAULT_SWITCH_IP - ] - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEL_SWITCH -) -@utils.wrap_to_dict(RESP_FIELDS) -def del_switch(switch_id, user=None, session=None, **kwargs): - """Delete a switch. - - If switch is not the default switch, and the machine under this switch - is only connected to this switch, the machine will be moved to connect - to default switch. Otherwise we can only simply delete the switch - machine. The purpose here to make sure there is no machine not - connecting to any switch. - """ - # TODO(xicheng): Simplify the logic if the default switch feature - # can be deprecated. - switch = _get_switch(switch_id, session=session) - default_switch = _get_switch_by_ip( - setting.DEFAULT_SWITCH_IP, session=session - ) - if switch.id != default_switch.id: - for switch_machine in switch.switch_machines: - machine = switch_machine.machine - if len(machine.switch_machines) <= 1: - utils.add_db_object( - session, models.SwitchMachine, - False, - default_switch.id, machine.id, - port=switch_machine.port - ) - return utils.del_db_object(session, switch) - - -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_SWITCH -) -def add_switch( - exception_when_existing=True, ip=None, - user=None, session=None, **kwargs -): - """Create a switch.""" - return _add_switch( - ip, - exception_when_existing=exception_when_existing, - session=session, **kwargs - ) - - -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_SWITCH -) -def add_switches( - exception_when_existing=False, - data=[], user=None, session=None -): - """Create switches.""" - # TODO(xicheng): simplify the batch api. - switches = [] - fail_switches = [] - for switch_data in data: - switch_object = _get_switch_by_ip( - switch_data['ip'], session=session, - exception_when_missing=False - ) - if switch_object: - logging.error('ip %s exists in switch %s' % ( - switch_data['ip'], switch_object.id - )) - fail_switches.append(switch_data) - else: - switches.append( - _add_switch( - exception_when_existing=exception_when_existing, - session=session, - **switch_data - ) - ) - return { - 'switches': switches, - 'fail_switches': fail_switches - } - - -@utils.wrap_to_dict(RESP_FIELDS) -def _update_switch(switch_id, session=None, **kwargs): - """Update a switch.""" - switch = _get_switch(switch_id, session=session) - return utils.update_db_object(session, switch, **kwargs) - - -# replace machine_filters in kwargs to put_machine_filters, -# which is used to tell db this is a put action for the field. -@utils.replace_filters( - machine_filters='put_machine_filters' -) -@utils.supported_filters( - optional_support_keys=UPDATED_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates( - credentials=utils.check_switch_credentials, - put_machine_filters=_check_machine_filters -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_SWITCH -) -def update_switch(switch_id, user=None, session=None, **kwargs): - """Update fields of a switch.""" - return _update_switch(switch_id, session=session, **kwargs) - - -# replace credentials to patched_credentials, -# machine_filters to patched_machine_filters in kwargs. -# This is to tell db they are patch action to the above fields. -@utils.replace_filters( - credentials='patched_credentials', - machine_filters='patched_machine_filters' -) -@utils.supported_filters( - optional_support_keys=PATCHED_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates( - patched_machine_filters=_check_machine_filters -) -@database.run_in_session() -@utils.output_validates( - credentials=utils.check_switch_credentials -) -@user_api.check_user_permission( - permission.PERMISSION_ADD_SWITCH -) -def patch_switch(switch_id, user=None, session=None, **kwargs): - """Patch fields of a switch.""" - return _update_switch(switch_id, session=session, **kwargs) - - -@util.deprecated -@utils.supported_filters(optional_support_keys=SUPPORTED_FILTER_FIELDS) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_SWITCH_FILTERS -) -@utils.wrap_to_dict(RESP_FILTERS_FIELDS) -def list_switch_filters(user=None, session=None, **filters): - """List all switches' filters.""" - return utils.list_db_objects( - session, models.Switch, **filters - ) - - -@util.deprecated -@utils.supported_filters() -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_SWITCH_FILTERS -) -@utils.wrap_to_dict(RESP_FILTERS_FIELDS) -def get_switch_filters( - switch_id, exception_when_missing=True, - user=None, session=None, **kwargs -): - """get filters of a switch.""" - return _get_switch( - switch_id, session=session, - exception_when_missing=exception_when_missing - ) - - -@util.deprecated -@utils.replace_filters( - machine_filters='put_machine_filters' -) -@utils.supported_filters( - optional_support_keys=UPDATED_FILTERS_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates(put_machine_filters=_check_machine_filters) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_SWITCH_FILTERS -) -@utils.wrap_to_dict(RESP_FILTERS_FIELDS) -def update_switch_filters(switch_id, user=None, session=None, **kwargs): - """Update filters of a switch.""" - switch = _get_switch(switch_id, session=session) - return utils.update_db_object(session, switch, **kwargs) - - -@util.deprecated -@utils.replace_filters( - machine_filters='patched_machine_filters' -) -@utils.supported_filters( - optional_support_keys=PATCHED_FILTERS_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates(patched_machine_filters=_check_machine_filters) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_SWITCH_FILTERS -) -@utils.wrap_to_dict(RESP_FILTERS_FIELDS) -def patch_switch_filter(switch_id, user=None, session=None, **kwargs): - """Patch filters to a switch.""" - switch = _get_switch(switch_id, session=session) - return utils.update_db_object(session, switch, **kwargs) - - -@util.deprecated -def get_switch_machines_internal(session, **filters): - return utils.list_db_objects( - session, models.SwitchMachine, **filters - ) - - -def _filter_port(port_filter, obj): - """filter switch machines by port. - - supported port_filter keys: [ - 'startswith', 'endswith', 'resp_lt', - 'resp_le', 'resp_gt', 'resp_ge', 'resp_range' - ] - - port_filter examples: - { - 'startswitch': 'ae', 'endswith': '', - 'resp_ge': 20, 'resp_le': 30, - } - """ - port_prefix = port_filter.get('startswith', '') - port_suffix = port_filter.get('endswith', '') - pattern = re.compile(r'%s(\d+)%s' % (port_prefix, port_suffix)) - match = pattern.match(obj) - if not match: - return False - port_number = int(match.group(1)) - if ( - 'resp_lt' in port_filter and - port_number >= port_filter['resp_lt'] - ): - return False - if ( - 'resp_le' in port_filter and - port_number > port_filter['resp_le'] - ): - return False - if ( - 'resp_gt' in port_filter and - port_number <= port_filter['resp_gt'] - ): - return False - if ( - 'resp_ge' in port_filter and - port_number < port_filter['resp_ge'] - ): - return False - if 'resp_range' in port_filter: - resp_range = port_filter['resp_range'] - if not isinstance(resp_range, list): - resp_range = [resp_range] - in_range = False - for port_start, port_end in resp_range: - if port_start <= port_number <= port_end: - in_range = True - break - if not in_range: - return False - return True - - -def _filter_vlans(vlan_filter, obj): - """Filter switch machines by vlan. - - supported keys in vlan_filter: - ['resp_in'] - """ - vlans = set(obj) - if 'resp_in' in vlan_filter: - resp_vlans = set(vlan_filter['resp_in']) - if not (vlans & resp_vlans): - return False - return True - - -@utils.output_filters( - port=_filter_port, vlans=_filter_vlans, - tag=utils.general_filter_callback, - location=utils.general_filter_callback -) -@utils.wrap_to_dict(RESP_MACHINES_FIELDS) -def _filter_switch_machines(switch_machines): - """Get filtered switch machines. - - The filters are defined in each switch. - """ - return [ - switch_machine for switch_machine in switch_machines - if not switch_machine.filtered - ] - - -@utils.output_filters( - missing_ok=True, - port=_filter_port, vlans=_filter_vlans, - tag=utils.general_filter_callback, - location=utils.general_filter_callback, - os_name=utils.general_filter_callback, -) -@utils.wrap_to_dict( - RESP_MACHINES_HOSTS_FIELDS, - clusters=RESP_CLUSTER_FIELDS -) -def _filter_switch_machines_hosts(switch_machines): - """Similar as _filter_switch_machines, but also return host info.""" - filtered_switch_machines = [ - switch_machine for switch_machine in switch_machines - if not switch_machine.filtered - ] - switch_machines_hosts = [] - for switch_machine in filtered_switch_machines: - machine = switch_machine.machine - host = machine.host - if host: - switch_machine_host_dict = host.to_dict() - else: - switch_machine_host_dict = machine.to_dict() - switch_machine_host_dict.update( - switch_machine.to_dict() - ) - switch_machines_hosts.append(switch_machine_host_dict) - return switch_machines_hosts - - -@utils.supported_filters( - optional_support_keys=SUPPORTED_MACHINES_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_SWITCH_MACHINES -) -def list_switch_machines( - switch_id, user=None, session=None, **filters -): - """Get switch machines of a switch.""" - switch = _get_switch(switch_id, session=session) - switch_machines = utils.list_db_objects( - session, models.SwitchMachine, switch_id=switch.id, **filters - ) - if not user.is_admin and len(switch_machines): - switch_machines = [m for m in switch_machines if m.machine.owner_id == user.id] - return _filter_switch_machines(switch_machines) - - -# replace ip_int to switch_ip_int in kwargs -@utils.replace_filters( - ip_int='switch_ip_int' -) -@utils.supported_filters( - optional_support_keys=SUPPORTED_SWITCH_MACHINES_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_SWITCH_MACHINES -) -def list_switchmachines(user=None, session=None, **filters): - """List switch machines.""" - switch_machines = utils.list_db_objects( - session, models.SwitchMachine, **filters - ) - return _filter_switch_machines( - switch_machines - ) - - -@utils.supported_filters( - optional_support_keys=SUPPORTED_MACHINES_HOSTS_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_SWITCH_MACHINES -) -def list_switch_machines_hosts( - switch_id, user=None, session=None, **filters -): - """Get switch machines and possible hosts of a switch.""" - switch = _get_switch(switch_id, session=session) - switch_machines = utils.list_db_objects( - session, models.SwitchMachine, switch_id=switch.id, **filters - ) - return _filter_switch_machines_hosts( - switch_machines - ) - - -# replace ip_int to switch_ip_int in kwargs -@utils.replace_filters( - ip_int='switch_ip_int' -) -@utils.supported_filters( - optional_support_keys=SUPPORTED_SWITCH_MACHINES_HOSTS_FIELDS -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_SWITCH_MACHINES -) -def list_switchmachines_hosts(user=None, session=None, **filters): - """List switch machines hnd possible hosts.""" - switch_machines = utils.list_db_objects( - session, models.SwitchMachine, **filters - ) - if not user.is_admin and len(switch_machines): - switch_machines = [m for m in switch_machines if m.machine.owner_id == user.id] - return _filter_switch_machines_hosts( - switch_machines - ) - - -@utils.supported_filters( - ADDED_MACHINES_FIELDS, - optional_support_keys=OPTIONAL_ADDED_MACHINES_FIELDS, - ignore_support_keys=OPTIONAL_ADDED_SWITCH_MACHINES_FIELDS -) -@utils.input_validates(mac=utils.check_mac) -def _add_machine_if_not_exist(mac=None, session=None, **kwargs): - """Add machine if the mac does not exist in any machine.""" - return utils.add_db_object( - session, models.Machine, False, - mac, **kwargs) - - -@utils.supported_filters( - ADDED_SWITCH_MACHINES_FIELDS, - optional_support_keys=OPTIONAL_ADDED_SWITCH_MACHINES_FIELDS, - ignore_support_keys=OPTIONAL_ADDED_MACHINES_FIELDS -) -@utils.input_validates(vlans=_check_vlans) -def _add_switch_machine_only( - switch, machine, exception_when_existing=True, - session=None, owner_id=None, port=None, **kwargs -): - """add a switch machine.""" - return utils.add_db_object( - session, models.SwitchMachine, - exception_when_existing, - switch.id, machine.id, port=port, - owner_id=owner_id, - **kwargs - ) - - -@utils.supported_filters( - ADDED_MACHINES_FIELDS + ADDED_SWITCH_MACHINES_FIELDS, - optional_support_keys=( - OPTIONAL_ADDED_MACHINES_FIELDS + - OPTIONAL_ADDED_SWITCH_MACHINES_FIELDS - ), - ignore_support_keys=IGNORE_FIELDS -) -@utils.wrap_to_dict(RESP_MACHINES_FIELDS) -def _add_switch_machine( - switch_id, exception_when_existing=True, - mac=None, port=None, session=None, owner_id=None, **kwargs -): - """Add switch machine. - - If underlying machine does not exist, also create the underlying - machine. - """ - switch = _get_switch(switch_id, session=session) - machine = _add_machine_if_not_exist( - mac=mac, session=session, owner_id=owner_id, **kwargs - ) - return _add_switch_machine_only( - switch, machine, - exception_when_existing, - port=port, session=session, **kwargs - ) - - -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_SWITCH_MACHINE -) -def add_switch_machine( - switch_id, exception_when_existing=True, - mac=None, user=None, session=None, - owner_id=None, **kwargs -): - """Add switch machine to a switch.""" - return _add_switch_machine( - switch_id, - exception_when_existing=exception_when_existing, - mac=mac, session=session, owner_id=owner_id, **kwargs - ) - - -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_SWITCH_MACHINE -) -@utils.wrap_to_dict( - [ - 'switches_machines', - 'duplicate_switches_machines', - 'fail_switches_machines' - ], - switches_machines=RESP_MACHINES_FIELDS, - duplicate_switches_machines=RESP_MACHINES_FIELDS -) -def add_switch_machines( - exception_when_existing=False, - data=[], user=None, session=None, owner_id=None -): - """Add switch machines.""" - switch_machines = [] - duplicate_switch_machines = [] - failed_switch_machines = [] - switches_mapping = {} - switch_machines_mapping = {} - switch_ips = [] - for item_data in data: - switch_ip = item_data['switch_ip'] - if switch_ip not in switches_mapping: - switch_object = _get_switch_by_ip( - switch_ip, session=session, - exception_when_missing=False - ) - if switch_object: - switch_ips.append(switch_ip) - switches_mapping[switch_ip] = switch_object - else: - logging.error( - 'switch %s does not exist' % switch_ip - ) - item_data.pop('switch_ip') - failed_switch_machines.append(item_data) - else: - switch_object = switches_mapping[switch_ip] - if switch_object: - item_data.pop('switch_ip') - switch_machines_mapping.setdefault( - switch_object.id, [] - ).append(item_data) - - for switch_ip in switch_ips: - switch_object = switches_mapping[switch_ip] - switch_id = switch_object.id - machines = switch_machines_mapping[switch_id] - for machine in machines: - mac = machine['mac'] - machine_object = _add_machine_if_not_exist( - mac=mac, session=session - ) - switch_machine_object = _get_switch_machine( - switch_id, machine_object.id, session=session, - exception_when_missing=False - ) - if switch_machine_object: - port = machine['port'] - switch_machine_id = switch_machine_object.switch_machine_id - exist_port = switch_machine_object.port - if exist_port != port: - logging.error( - 'switch machine %s exist port %s is ' - 'different from added port %s' % ( - switch_machine_id, - exist_port, port - ) - ) - failed_switch_machines.append(machine) - else: - logging.error( - 'iswitch machine %s is dulicate, ' - 'will not be override' % switch_machine_id - ) - duplicate_switch_machines.append(machine) - else: - del machine['mac'] - switch_machines.append(_add_switch_machine_only( - switch_object, machine_object, - exception_when_existing, - session=session, owner_id=owner_id, **machine - )) - return { - 'switches_machines': switch_machines, - 'duplicate_switches_machines': duplicate_switch_machines, - 'fail_switches_machines': failed_switch_machines - } - - -@utils.supported_filters(optional_support_keys=['find_machines']) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_SWITCH_MACHINES -) -@utils.wrap_to_dict(RESP_ACTION_FIELDS) -def poll_switch(switch_id, user=None, session=None, **kwargs): - """poll switch to get machines.""" - from compass.tasks import client as celery_client - switch = _get_switch(switch_id, session=session) - celery_client.celery.send_task( - 'compass.tasks.pollswitch', - (user.email, switch.ip, switch.credentials), - queue=user.email, - exchange=user.email, - routing_key=user.email - ) - return { - 'status': 'action %s sent' % kwargs, - 'details': { - } - } - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_SWITCH_MACHINES -) -@utils.wrap_to_dict(RESP_MACHINES_FIELDS) -def get_switch_machine( - switch_id, machine_id, exception_when_missing=True, - user=None, session=None, **kwargs -): - """get a switch machine by switch id and machine id.""" - return _get_switch_machine( - switch_id, machine_id, session=session, - exception_when_missing=exception_when_missing - ) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_SWITCH_MACHINES -) -@utils.wrap_to_dict(RESP_MACHINES_FIELDS) -def get_switchmachine( - switch_machine_id, exception_when_missing=True, - user=None, session=None, **kwargs -): - """get a switch machine by switch_machine_id.""" - return _get_switchmachine( - switch_machine_id, session=session, - exception_when_missing=exception_when_missing - ) - - -@utils.supported_filters( - optional_support_keys=( - UPDATED_MACHINES_FIELDS + PATCHED_MACHINES_FIELDS - ), - ignore_support_keys=( - UPDATED_SWITCH_MACHINES_FIELDS + PATCHED_SWITCH_MACHINES_FIELDS - ) -) -def _update_machine_if_necessary( - machine, session=None, **kwargs -): - """Update machine is there is something to update.""" - utils.update_db_object( - session, machine, **kwargs - ) - - -@utils.supported_filters( - optional_support_keys=( - UPDATED_SWITCH_MACHINES_FIELDS + PATCHED_SWITCH_MACHINES_FIELDS - ), - ignore_support_keys=( - UPDATED_MACHINES_FIELDS + PATCHED_MACHINES_FIELDS - ) -) -def _update_switch_machine_only(switch_machine, session=None, **kwargs): - """Update switch machine.""" - return utils.update_db_object( - session, switch_machine, **kwargs - ) - - -def _update_switch_machine( - switch_machine, session=None, **kwargs -): - """Update switch machine. - - If there are some attributes of underlying machine need to update, - also update them in underlying machine. - """ - _update_machine_if_necessary( - switch_machine.machine, session=session, **kwargs - ) - return _update_switch_machine_only( - switch_machine, session=session, **kwargs - ) - - -@utils.supported_filters( - optional_support_keys=( - UPDATED_MACHINES_FIELDS + UPDATED_SWITCH_MACHINES_FIELDS - ), - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates(vlans=_check_vlans) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_SWITCH_MACHINE -) -@utils.wrap_to_dict(RESP_MACHINES_FIELDS) -def update_switch_machine( - switch_id, machine_id, user=None, - session=None, **kwargs -): - """Update switch machine by switch id and machine id.""" - switch_machine = _get_switch_machine( - switch_id, machine_id, session=session - ) - return _update_switch_machine( - switch_machine, - session=session, **kwargs - ) - - -@utils.supported_filters( - optional_support_keys=( - UPDATED_MACHINES_FIELDS + UPDATED_SWITCH_MACHINES_FIELDS - ), - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates(vlans=_check_vlans) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_SWITCH_MACHINE -) -@utils.wrap_to_dict(RESP_MACHINES_FIELDS) -def update_switchmachine(switch_machine_id, user=None, session=None, **kwargs): - """Update switch machine by switch_machine_id.""" - switch_machine = _get_switchmachine( - switch_machine_id, session=session - ) - return _update_switch_machine( - switch_machine, - session=session, **kwargs - ) - - -# replace [vlans, ipmi_credentials, tag, location] to -# [patched_vlans, patched_ipmi_credentials, patched_tag, -# patched_location] in kwargs. It tells db these fields will -# be patched. -@utils.replace_filters( - vlans='patched_vlans', - ipmi_credentials='patched_ipmi_credentials', - tag='patched_tag', - location='patched_location' -) -@utils.supported_filters( - optional_support_keys=( - PATCHED_MACHINES_FIELDS + PATCHED_SWITCH_MACHINES_FIELDS - ), - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates(patched_vlans=_check_vlans) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_SWITCH_MACHINE -) -@utils.wrap_to_dict(RESP_MACHINES_FIELDS) -def patch_switch_machine( - switch_id, machine_id, user=None, - session=None, **kwargs -): - """Patch switch machine by switch_id and machine_id.""" - switch_machine = _get_switch_machine( - switch_id, machine_id, session=session - ) - return _update_switch_machine( - switch_machine, - session=session, **kwargs - ) - - -# replace [vlans, ipmi_credentials, tag, location] to -# [patched_vlans, patched_ipmi_credentials, patched_tag, -# patched_location] in kwargs. It tells db these fields will -# be patched. -@utils.replace_filters( - vlans='patched_vlans', - ipmi_credentials='patched_ipmi_credentials', - tag='patched_tag', - location='patched_location' -) -@utils.supported_filters( - optional_support_keys=( - PATCHED_MACHINES_FIELDS + PATCHED_SWITCH_MACHINES_FIELDS - ), - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates(patched_vlans=_check_vlans) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_ADD_SWITCH_MACHINE -) -@utils.wrap_to_dict(RESP_MACHINES_FIELDS) -def patch_switchmachine(switch_machine_id, user=None, session=None, **kwargs): - """Patch switch machine by switch_machine_id.""" - switch_machine = _get_switchmachine( - switch_machine_id, session=session - ) - return _update_switch_machine( - switch_machine, - session=session, **kwargs - ) - - -def _del_switch_machine( - switch_machine, session=None -): - """Delete switch machine. - - If this is the last switch machine associated to underlying machine, - add a switch machine record to default switch to make the machine - searchable. - """ - default_switch = _get_switch_by_ip( - setting.DEFAULT_SWITCH_IP, session=session - ) - machine = switch_machine.machine - if len(machine.switch_machines) <= 1: - utils.add_db_object( - session, models.SwitchMachine, - False, - default_switch.id, machine.id, - port=switch_machine.port - ) - return utils.del_db_object(session, switch_machine) - - -@utils.supported_filters() -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEL_SWITCH_MACHINE -) -@utils.wrap_to_dict(RESP_MACHINES_FIELDS) -def del_switch_machine( - switch_id, machine_id, user=None, - session=None, **kwargs -): - """Delete switch machine by switch id and machine id.""" - switch_machine = _get_switch_machine( - switch_id, machine_id, session=session - ) - return _del_switch_machine(switch_machine, session=session) - - -@utils.supported_filters() -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_DEL_SWITCH_MACHINE -) -@utils.wrap_to_dict(RESP_MACHINES_FIELDS) -def del_switchmachine(switch_machine_id, user=None, session=None, **kwargs): - """Delete switch machine by switch_machine_id.""" - switch_machine = _get_switchmachine( - switch_machine_id, session=session - ) - return _del_switch_machine(switch_machine, session=session) - - -@utils.supported_filters( - ['machine_id'], - optional_support_keys=UPDATED_SWITCH_MACHINES_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -def _add_machine_to_switch( - switch_id, machine_id, session=None, **kwargs -): - """Add machine to switch.""" - switch = _get_switch(switch_id, session=session) - from compass.db.api import machine as machine_api - machine = machine_api.get_machine_internal( - machine_id, session=session - ) - _add_switch_machine_only( - switch, machine, False, - owner_id=machine.owner_id, **kwargs - ) - - -def _add_machines(switch, machines, session=None): - """Add machines to switch. - - Args: - machines: list of dict which contains attributes to - add machine to switch. - - machines example: - {{'machine_id': 1, 'port': 'ae20'}] - """ - for machine in machines: - _add_machine_to_switch( - switch.id, session=session, **machine - ) - - -def _remove_machines(switch, machines, session=None): - """Remove machines from switch. - - Args: - machines: list of machine id. - - machines example: - [1,2] - """ - utils.del_db_objects( - session, models.SwitchMachine, - switch_id=switch.id, machine_id=machines - ) - - -def _set_machines(switch, machines, session=None): - """Reset machines to a switch. - - Args: - machines: list of dict which contains attributes to - add machine to switch. - - machines example: - {{'machine_id': 1, 'port': 'ae20'}] - """ - utils.del_db_objects( - session, models.SwitchMachine, - switch_id=switch.id - ) - for switch_machine in machines: - _add_machine_to_switch( - switch.id, session=session, **switch_machine - ) - - -@utils.supported_filters( - optional_support_keys=[ - 'add_machines', 'remove_machines', 'set_machines' - ] -) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_UPDATE_SWITCH_MACHINES -) -@utils.wrap_to_dict(RESP_MACHINES_FIELDS) -def update_switch_machines( - switch_id, add_machines=[], remove_machines=[], - set_machines=None, user=None, session=None, **kwargs -): - """update switch's machines""" - switch = _get_switch(switch_id, session=session) - if remove_machines: - _remove_machines( - switch, remove_machines, session=session - ) - if add_machines: - _add_machines( - switch, add_machines, session=session - ) - if set_machines is not None: - _set_machines( - switch, set_machines, session=session - ) - return switch.switch_machines diff --git a/compass-tasks/db/api/user.py b/compass-tasks/db/api/user.py deleted file mode 100644 index db039eb..0000000 --- a/compass-tasks/db/api/user.py +++ /dev/null @@ -1,553 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""User database operations.""" -import datetime -import functools -import logging -import re - -from flask.ext.login import UserMixin - -from compass.db.api import database -from compass.db.api import utils -from compass.db import exception -from compass.db import models - -from compass.utils import setting_wrapper as setting -from compass.utils import util - - -SUPPORTED_FIELDS = ['email', 'is_admin', 'active'] -PERMISSION_SUPPORTED_FIELDS = ['name'] -SELF_UPDATED_FIELDS = ['email', 'firstname', 'lastname', 'password'] -ADMIN_UPDATED_FIELDS = ['is_admin', 'active'] -IGNORE_FIELDS = ['id', 'created_at', 'updated_at'] -UPDATED_FIELDS = [ - 'email', 'firstname', 'lastname', 'password', 'is_admin', 'active' -] -ADDED_FIELDS = ['email', 'password'] -OPTIONAL_ADDED_FIELDS = ['is_admin', 'active'] -PERMISSION_ADDED_FIELDS = ['permission_id'] -RESP_FIELDS = [ - 'id', 'email', 'is_admin', 'active', 'firstname', - 'lastname', 'created_at', 'updated_at' -] -RESP_TOKEN_FIELDS = [ - 'id', 'user_id', 'token', 'expire_timestamp' -] -PERMISSION_RESP_FIELDS = [ - 'id', 'user_id', 'permission_id', 'name', 'alias', 'description', - 'created_at', 'updated_at' -] - - -def _check_email(email): - """Check email is email format.""" - if '@' not in email: - raise exception.InvalidParameter( - 'there is no @ in email address %s.' % email - ) - - -def _check_user_permission(user, permission, session=None): - """Check user has permission.""" - if not user: - logging.info('empty user means the call is from internal') - return - if user.is_admin: - return - - user_permission = utils.get_db_object( - session, models.UserPermission, - False, user_id=user.id, name=permission.name - ) - if not user_permission: - raise exception.Forbidden( - 'user %s does not have permission %s' % ( - user.email, permission.name - ) - ) - - -def check_user_permission(permission): - """Decorator to check user having permission.""" - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - user = kwargs.get('user') - if user is not None: - session = kwargs.get('session') - if session is None: - raise exception.DatabaseException( - 'wrapper check_user_permission does not run in session' - ) - _check_user_permission(user, permission, session=session) - return func(*args, **kwargs) - else: - return func(*args, **kwargs) - return wrapper - return decorator - - -def check_user_admin(): - """Decorator to check user is admin.""" - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - user = kwargs.get('user') - if user is not None: - if not user.is_admin: - raise exception.Forbidden( - 'User %s is not admin.' % ( - user.email - ) - ) - return func(*args, **kwargs) - else: - return func(*args, **kwargs) - return wrapper - return decorator - - -def check_user_admin_or_owner(): - """Decorator to check user is admin or the owner of the resource.""" - def decorator(func): - @functools.wraps(func) - def wrapper(user_id, *args, **kwargs): - user = kwargs.get('user') - if user is not None: - session = kwargs.get('session') - if session is None: - raise exception.DatabaseException( - 'wrapper check_user_admin_or_owner is ' - 'not called in session' - ) - check_user = _get_user(user_id, session=session) - if not user.is_admin and user.id != check_user.id: - raise exception.Forbidden( - 'User %s is not admin or the owner of user %s.' % ( - user.email, check_user.email - ) - ) - - return func( - user_id, *args, **kwargs - ) - else: - return func( - user_id, *args, **kwargs - ) - return wrapper - return decorator - - -def _add_user_permissions(user, session=None, **permission_filters): - """add permissions to a user.""" - from compass.db.api import permission as permission_api - for api_permission in permission_api.list_permissions( - session=session, **permission_filters - ): - utils.add_db_object( - session, models.UserPermission, False, - user.id, api_permission['id'] - ) - - -def _remove_user_permissions(user, session=None, **permission_filters): - """remove permissions from a user.""" - from compass.db.api import permission as permission_api - permission_ids = [ - api_permission['id'] - for api_permission in permission_api.list_permissions( - session=session, **permission_filters - ) - ] - utils.del_db_objects( - session, models.UserPermission, - user_id=user.id, permission_id=permission_ids - ) - - -def _set_user_permissions(user, session=None, **permission_filters): - """set permissions to a user.""" - utils.del_db_objects( - session, models.UserPermission, - user_id=user.id - ) - _add_user_permissions(session, user, **permission_filters) - - -class UserWrapper(UserMixin): - """Wrapper class provided to flask.""" - - def __init__( - self, id, email, crypted_password, - active=True, is_admin=False, - expire_timestamp=None, token='', **kwargs - ): - self.id = id - self.email = email - self.password = crypted_password - self.active = active - self.is_admin = is_admin - self.expire_timestamp = expire_timestamp - if not token: - self.token = self.get_auth_token() - else: - self.token = token - super(UserWrapper, self).__init__() - - def authenticate(self, password): - if not util.encrypt(password, self.password) == self.password: - raise exception.Unauthorized('%s password mismatch' % self.email) - - def get_auth_token(self): - return util.encrypt(self.email) - - def is_active(self): - return self.active - - def get_id(self): - return self.token - - def is_authenticated(self): - current_time = datetime.datetime.now() - return ( - not self.expire_timestamp or - current_time < self.expire_timestamp - ) - - def __str__(self): - return '%s[email:%s,password:%s]' % ( - self.__class__.__name__, self.email, self.password) - - -@database.run_in_session() -def get_user_object(email, session=None, **kwargs): - """get user and convert to UserWrapper object.""" - user = utils.get_db_object( - session, models.User, False, email=email - ) - if not user: - raise exception.Unauthorized( - '%s unauthorized' % email - ) - user_dict = user.to_dict() - user_dict.update(kwargs) - return UserWrapper(**user_dict) - - -@database.run_in_session(exception_when_in_session=False) -def get_user_object_from_token(token, session=None): - """Get user from token and convert to UserWrapper object. - - ::note: - get_user_object_from_token may be called in session. - """ - expire_timestamp = { - 'ge': datetime.datetime.now() - } - user_token = utils.get_db_object( - session, models.UserToken, False, - token=token, expire_timestamp=expire_timestamp - ) - if not user_token: - raise exception.Unauthorized( - 'invalid user token: %s' % token - ) - user_dict = _get_user( - user_token.user_id, session=session - ).to_dict() - user_dict['token'] = token - expire_timestamp = user_token.expire_timestamp - user_dict['expire_timestamp'] = expire_timestamp - return UserWrapper(**user_dict) - - -@utils.supported_filters() -@database.run_in_session() -@utils.wrap_to_dict(RESP_TOKEN_FIELDS) -def record_user_token( - token, expire_timestamp, user=None, session=None -): - """record user token in database.""" - user_token = utils.get_db_object( - session, models.UserToken, False, - user_id=user.id, token=token - ) - if not user_token: - return utils.add_db_object( - session, models.UserToken, True, - token, user_id=user.id, - expire_timestamp=expire_timestamp - ) - elif expire_timestamp > user_token.expire_timestamp: - return utils.update_db_object( - session, user_token, expire_timestamp=expire_timestamp - ) - return user_token - - -@utils.supported_filters() -@database.run_in_session() -@utils.wrap_to_dict(RESP_TOKEN_FIELDS) -def clean_user_token(token, user=None, session=None): - """clean user token in database.""" - return utils.del_db_objects( - session, models.UserToken, - token=token, user_id=user.id - ) - - -def _get_user(user_id, session=None, **kwargs): - """Get user object by user id.""" - if isinstance(user_id, (int, long)): - return utils.get_db_object( - session, models.User, id=user_id, **kwargs - ) - raise exception.InvalidParameter( - 'user id %s type is not int compatible' % user_id - ) - - -@utils.supported_filters() -@database.run_in_session() -@check_user_admin_or_owner() -@utils.wrap_to_dict(RESP_FIELDS) -def get_user( - user_id, exception_when_missing=True, - user=None, session=None, **kwargs -): - """get a user.""" - return _get_user( - user_id, session=session, - exception_when_missing=exception_when_missing - ) - - -@utils.supported_filters() -@database.run_in_session() -@utils.wrap_to_dict(RESP_FIELDS) -def get_current_user( - exception_when_missing=True, user=None, - session=None, **kwargs -): - """get current user.""" - return _get_user( - user.id, session=session, - exception_when_missing=exception_when_missing - ) - - -@utils.supported_filters( - optional_support_keys=SUPPORTED_FIELDS -) -@database.run_in_session() -@check_user_admin() -@utils.wrap_to_dict(RESP_FIELDS) -def list_users(user=None, session=None, **filters): - """List all users.""" - return utils.list_db_objects( - session, models.User, **filters - ) - - -@utils.input_validates(email=_check_email) -@utils.supported_filters( - ADDED_FIELDS, - optional_support_keys=OPTIONAL_ADDED_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@check_user_admin() -@utils.wrap_to_dict(RESP_FIELDS) -def add_user( - exception_when_existing=True, user=None, - session=None, email=None, **kwargs -): - """Create a user and return created user object.""" - add_user = utils.add_db_object( - session, models.User, - exception_when_existing, email, - **kwargs) - _add_user_permissions( - add_user, - session=session, - name=setting.COMPASS_DEFAULT_PERMISSIONS - ) - return add_user - - -@utils.supported_filters() -@database.run_in_session() -@check_user_admin() -@utils.wrap_to_dict(RESP_FIELDS) -def del_user(user_id, user=None, session=None, **kwargs): - """delete a user and return the deleted user object.""" - del_user = _get_user(user_id, session=session) - return utils.del_db_object(session, del_user) - - -@utils.supported_filters( - optional_support_keys=UPDATED_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@utils.input_validates(email=_check_email) -@database.run_in_session() -@utils.wrap_to_dict(RESP_FIELDS) -def update_user(user_id, user=None, session=None, **kwargs): - """Update a user and return the updated user object.""" - update_user = _get_user( - user_id, session=session, - ) - allowed_fields = set() - if user.is_admin: - allowed_fields |= set(ADMIN_UPDATED_FIELDS) - if user.id == update_user.id: - allowed_fields |= set(SELF_UPDATED_FIELDS) - unsupported_fields = set(kwargs) - allowed_fields - if unsupported_fields: - # The user is not allowed to update a user. - raise exception.Forbidden( - 'User %s has no permission to update user %s fields %s.' % ( - user.email, user.email, unsupported_fields - ) - ) - return utils.update_db_object(session, update_user, **kwargs) - - -@utils.supported_filters(optional_support_keys=PERMISSION_SUPPORTED_FIELDS) -@database.run_in_session() -@check_user_admin_or_owner() -@utils.wrap_to_dict(PERMISSION_RESP_FIELDS) -def get_permissions( - user_id, user=None, exception_when_missing=True, - session=None, **kwargs -): - """List permissions of a user.""" - get_user = _get_user( - user_id, session=session, - exception_when_missing=exception_when_missing - ) - return utils.list_db_objects( - session, models.UserPermission, user_id=get_user.id, **kwargs - ) - - -def _get_permission(user_id, permission_id, session=None, **kwargs): - """Get user permission by user id and permission id.""" - user = _get_user(user_id, session=session) - from compass.db.api import permission as permission_api - permission = permission_api.get_permission_internal( - permission_id, session=session - ) - return utils.get_db_object( - session, models.UserPermission, - user_id=user.id, permission_id=permission.id, - **kwargs - ) - - -@utils.supported_filters() -@database.run_in_session() -@check_user_admin_or_owner() -@utils.wrap_to_dict(PERMISSION_RESP_FIELDS) -def get_permission( - user_id, permission_id, exception_when_missing=True, - user=None, session=None, **kwargs -): - """Get a permission of a user.""" - return _get_permission( - user_id, permission_id, - exception_when_missing=exception_when_missing, - session=session, - **kwargs - ) - - -@utils.supported_filters() -@database.run_in_session() -@check_user_admin_or_owner() -@utils.wrap_to_dict(PERMISSION_RESP_FIELDS) -def del_permission(user_id, permission_id, user=None, session=None, **kwargs): - """Delete a permission from a user.""" - user_permission = _get_permission( - user_id, permission_id, - session=session, **kwargs - ) - return utils.del_db_object(session, user_permission) - - -@utils.supported_filters( - PERMISSION_ADDED_FIELDS, - ignore_support_keys=IGNORE_FIELDS -) -@database.run_in_session() -@check_user_admin() -@utils.wrap_to_dict(PERMISSION_RESP_FIELDS) -def add_permission( - user_id, permission_id=None, exception_when_existing=True, - user=None, session=None -): - """Add a permission to a user.""" - get_user = _get_user(user_id, session=session) - from compass.db.api import permission as permission_api - get_permission = permission_api.get_permission_internal( - permission_id, session=session - ) - return utils.add_db_object( - session, models.UserPermission, exception_when_existing, - get_user.id, get_permission.id - ) - - -def _get_permission_filters(permission_ids): - """Helper function to filter permissions.""" - if permission_ids == 'all': - return {} - else: - return {'id': permission_ids} - - -@utils.supported_filters( - optional_support_keys=[ - 'add_permissions', 'remove_permissions', 'set_permissions' - ] -) -@database.run_in_session() -@check_user_admin() -@utils.wrap_to_dict(PERMISSION_RESP_FIELDS) -def update_permissions( - user_id, add_permissions=[], remove_permissions=[], - set_permissions=None, user=None, session=None, **kwargs -): - """update user permissions.""" - update_user = _get_user(user_id, session=session) - if remove_permissions: - _remove_user_permissions( - update_user, session=session, - **_get_permission_filters(remove_permissions) - ) - if add_permissions: - _add_user_permissions( - update_user, session=session, - **_get_permission_filters(add_permissions) - ) - if set_permissions is not None: - _set_user_permissions( - update_user, session=session, - **_get_permission_filters(set_permissions) - ) - return update_user.user_permissions diff --git a/compass-tasks/db/api/user_log.py b/compass-tasks/db/api/user_log.py deleted file mode 100644 index 70de9db..0000000 --- a/compass-tasks/db/api/user_log.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""UserLog database operations.""" -import logging - -from compass.db.api import database -from compass.db.api import user as user_api -from compass.db.api import utils -from compass.db import exception -from compass.db import models - - -SUPPORTED_FIELDS = ['user_email', 'timestamp'] -USER_SUPPORTED_FIELDS = ['timestamp'] -RESP_FIELDS = ['user_id', 'action', 'timestamp'] - - -@database.run_in_session() -def log_user_action(user_id, action, session=None): - """Log user action.""" - utils.add_db_object( - session, models.UserLog, True, user_id=user_id, action=action - ) - - -@utils.supported_filters(optional_support_keys=USER_SUPPORTED_FIELDS) -@database.run_in_session() -@user_api.check_user_admin_or_owner() -@utils.wrap_to_dict(RESP_FIELDS) -def list_user_actions(user_id, user=None, session=None, **filters): - """list user actions of a user.""" - list_user = user_api.get_user(user_id, user=user, session=session) - return utils.list_db_objects( - session, models.UserLog, order_by=['timestamp'], - user_id=list_user['id'], **filters - ) - - -@utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) -@user_api.check_user_admin() -@database.run_in_session() -@utils.wrap_to_dict(RESP_FIELDS) -def list_actions(user=None, session=None, **filters): - """list actions of all users.""" - return utils.list_db_objects( - session, models.UserLog, order_by=['timestamp'], **filters - ) - - -@utils.supported_filters() -@database.run_in_session() -@user_api.check_user_admin_or_owner() -@utils.wrap_to_dict(RESP_FIELDS) -def del_user_actions(user_id, user=None, session=None, **filters): - """delete actions of a user.""" - del_user = user_api.get_user(user_id, user=user, session=session) - return utils.del_db_objects( - session, models.UserLog, user_id=del_user['id'], **filters - ) - - -@utils.supported_filters() -@database.run_in_session() -@user_api.check_user_admin() -@utils.wrap_to_dict(RESP_FIELDS) -def del_actions(user=None, session=None, **filters): - """delete actions of all users.""" - return utils.del_db_objects( - session, models.UserLog, **filters - ) diff --git a/compass-tasks/db/api/utils.py b/compass-tasks/db/api/utils.py deleted file mode 100644 index a44f26e..0000000 --- a/compass-tasks/db/api/utils.py +++ /dev/null @@ -1,1286 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utils for database usage.""" - -import functools -import inspect -import logging -import netaddr -import re - -from inspect import isfunction -from sqlalchemy import and_ -from sqlalchemy import or_ - -from compass.db import exception -from compass.db import models -from compass.utils import util - - -def model_query(session, model): - """model query. - - Return sqlalchemy query object. - """ - if not issubclass(model, models.BASE): - raise exception.DatabaseException("model should be sublass of BASE!") - - return session.query(model) - - -def _default_list_condition_func(col_attr, value, condition_func): - """The default condition func for a list of data. - - Given the condition func for single item of data, this function - wrap the condition_func and return another condition func using - or_ to merge the conditions of each single item to deal with a - list of data item. - - Args: - col_attr: the colomn name - value: the column value need to be compared. - condition_func: the sqlalchemy condition object like == - - Examples: - col_attr is name, value is ['a', 'b', 'c'] and - condition_func is ==, the returned condition is - name == 'a' or name == 'b' or name == 'c' - """ - conditions = [] - for sub_value in value: - condition = condition_func(col_attr, sub_value) - if condition is not None: - conditions.append(condition) - if conditions: - return or_(*conditions) - else: - return None - - -def _one_item_list_condition_func(col_attr, value, condition_func): - """The wrapper condition func to deal with one item data list. - - For simplification, it is used to reduce generating too complex - sql conditions. - """ - if value: - return condition_func(col_attr, value[0]) - else: - return None - - -def _model_condition_func( - col_attr, value, - item_condition_func, - list_condition_func=_default_list_condition_func -): - """Return sql condition based on value type.""" - if isinstance(value, list): - if not value: - return None - if len(value) == 1: - return item_condition_func(col_attr, value) - return list_condition_func( - col_attr, value, item_condition_func - ) - else: - return item_condition_func(col_attr, value) - - -def _between_condition(col_attr, value): - """Return sql range condition.""" - if value[0] is not None and value[1] is not None: - return col_attr.between(value[0], value[1]) - if value[0] is not None: - return col_attr >= value[0] - if value[1] is not None: - return col_attr <= value[1] - return None - - -def model_order_by(query, model, order_by): - """append order by into sql query model.""" - if not order_by: - return query - order_by_cols = [] - for key in order_by: - if isinstance(key, tuple): - key, is_desc = key - else: - is_desc = False - if isinstance(key, basestring): - if hasattr(model, key): - col_attr = getattr(model, key) - else: - continue - else: - col_attr = key - if is_desc: - order_by_cols.append(col_attr.desc()) - else: - order_by_cols.append(col_attr) - return query.order_by(*order_by_cols) - - -def _model_condition(col_attr, value): - """Generate condition for one column. - - Example for col_attr is name: - value is 'a': name == 'a' - value is ['a']: name == 'a' - value is ['a', 'b']: name == 'a' or name == 'b' - value is {'eq': 'a'}: name == 'a' - value is {'lt': 'a'}: name < 'a' - value is {'le': 'a'}: name <= 'a' - value is {'gt': 'a'}: name > 'a' - value is {'ge': 'a'}: name >= 'a' - value is {'ne': 'a'}: name != 'a' - value is {'in': ['a', 'b']}: name in ['a', 'b'] - value is {'notin': ['a', 'b']}: name not in ['a', 'b'] - value is {'startswith': 'abc'}: name like 'abc%' - value is {'endswith': 'abc'}: name like '%abc' - value is {'like': 'abc'}: name like '%abc%' - value is {'between': ('a', 'c')}: name >= 'a' and name <= 'c' - value is [{'lt': 'a'}]: name < 'a' - value is [{'lt': 'a'}, {'gt': c'}]: name < 'a' or name > 'c' - value is {'lt': 'c', 'gt': 'a'}: name > 'a' and name < 'c' - - If value is a list, the condition is the or relationship among - conditions of each item. - If value is dict and there are multi keys in the dict, the relationship - is and conditions of each key. - Otherwise the condition is to compare the column with the value. - """ - if isinstance(value, list): - basetype_values = [] - composite_values = [] - for item in value: - if isinstance(item, (list, dict)): - composite_values.append(item) - else: - basetype_values.append(item) - conditions = [] - if basetype_values: - if len(basetype_values) == 1: - condition = (col_attr == basetype_values[0]) - else: - condition = col_attr.in_(basetype_values) - conditions.append(condition) - for composite_value in composite_values: - condition = _model_condition(col_attr, composite_value) - if condition is not None: - conditions.append(condition) - if not conditions: - return None - if len(conditions) == 1: - return conditions[0] - return or_(*conditions) - elif isinstance(value, dict): - conditions = [] - if 'eq' in value: - conditions.append(_model_condition_func( - col_attr, value['eq'], - lambda attr, data: attr == data, - lambda attr, data, item_condition_func: attr.in_(data) - )) - if 'lt' in value: - conditions.append(_model_condition_func( - col_attr, value['lt'], - lambda attr, data: attr < data, - _one_item_list_condition_func - )) - if 'gt' in value: - conditions.append(_model_condition_func( - col_attr, value['gt'], - lambda attr, data: attr > data, - _one_item_list_condition_func - )) - if 'le' in value: - conditions.append(_model_condition_func( - col_attr, value['le'], - lambda attr, data: attr <= data, - _one_item_list_condition_func - )) - if 'ge' in value: - conditions.append(_model_condition_func( - col_attr, value['ge'], - lambda attr, data: attr >= data, - _one_item_list_condition_func - )) - if 'ne' in value: - conditions.append(_model_condition_func( - col_attr, value['ne'], - lambda attr, data: attr != data, - lambda attr, data, item_condition_func: attr.notin_(data) - )) - if 'in' in value: - conditions.append(col_attr.in_(value['in'])) - if 'notin' in value: - conditions.append(col_attr.notin_(value['notin'])) - if 'startswith' in value: - conditions.append(_model_condition_func( - col_attr, value['startswith'], - lambda attr, data: attr.like('%s%%' % data) - )) - if 'endswith' in value: - conditions.append(_model_condition_func( - col_attr, value['endswith'], - lambda attr, data: attr.like('%%%s' % data) - )) - if 'like' in value: - conditions.append(_model_condition_func( - col_attr, value['like'], - lambda attr, data: attr.like('%%%s%%' % data) - )) - if 'between' in value: - conditions.append(_model_condition_func( - col_attr, value['between'], - _between_condition - )) - conditions = [ - condition - for condition in conditions - if condition is not None - ] - if not conditions: - return None - if len(conditions) == 1: - return conditions[0] - return and_(conditions) - else: - condition = (col_attr == value) - return condition - - -def model_filter(query, model, **filters): - """Append conditons to query for each possible column.""" - for key, value in filters.items(): - if isinstance(key, basestring): - if hasattr(model, key): - col_attr = getattr(model, key) - else: - continue - else: - col_attr = key - condition = _model_condition(col_attr, value) - if condition is not None: - query = query.filter(condition) - return query - - -def replace_output(**output_mapping): - """Decorator to recursively relace output by output mapping. - - The replacement detail is described in _replace_output. - """ - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - return _replace_output( - func(*args, **kwargs), **output_mapping - ) - return wrapper - return decorator - - -def _replace_output(data, **output_mapping): - """Helper to replace output data. - - Example: - data = {'a': 'hello'} - output_mapping = {'a': 'b'} - returns: {'b': 'hello'} - - data = {'a': {'b': 'hello'}} - output_mapping = {'a': 'b'} - returns: {'b': {'b': 'hello'}} - - data = {'a': {'b': 'hello'}} - output_mapping = {'a': {'b': 'c'}} - returns: {'a': {'c': 'hello'}} - - data = [{'a': 'hello'}, {'a': 'hi'}] - output_mapping = {'a': 'b'} - returns: [{'b': 'hello'}, {'b': 'hi'}] - """ - if isinstance(data, list): - return [ - _replace_output(item, **output_mapping) - for item in data - ] - if not isinstance(data, dict): - raise exception.InvalidResponse( - '%s type is not dict' % data - ) - info = {} - for key, value in data.items(): - if key in output_mapping: - output_key = output_mapping[key] - if isinstance(output_key, basestring): - info[output_key] = value - else: - info[key] = ( - _replace_output(value, **output_key) - ) - else: - info[key] = value - return info - - -def get_wrapped_func(func): - """Get wrapped function instance. - - Example: - @dec1 - @dec2 - myfunc(*args, **kwargs) - - get_wrapped_func(myfunc) returns function object with - following attributes: - __name__: 'myfunc' - args: args - kwargs: kwargs - otherwise myfunc is function object with following attributes: - __name__: partial object ... - args: ... - kwargs: ... - """ - if func.func_closure: - for closure in func.func_closure: - if isfunction(closure.cell_contents): - return get_wrapped_func(closure.cell_contents) - return func - else: - return func - - -def wrap_to_dict(support_keys=[], **filters): - """Decrator to convert returned object to dict. - - The details is decribed in _wrapper_dict. - """ - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - return _wrapper_dict( - func(*args, **kwargs), support_keys, **filters - ) - return wrapper - return decorator - - -def _wrapper_dict(data, support_keys, **filters): - """Helper for warpping db object into dictionary. - - If data is list, convert it to a list of dict - If data is Base model, convert it to dict - for the data as a dict, filter it with the supported keys. - For each filter_key, filter_value in filters, also filter - data[filter_key] by filter_value recursively if it exists. - - Example: - data is models.Switch, it will be converted to - { - 'id': 1, 'ip': '10.0.0.1', 'ip_int': 123456, - 'credentials': {'version': 2, 'password': 'abc'} - } - Then if support_keys are ['id', 'ip', 'credentials'], - it will be filtered to { - 'id': 1, 'ip': '10.0.0.1', - 'credentials': {'version': 2, 'password': 'abc'} - } - Then if filters is {'credentials': ['version']}, - it will be filtered to { - 'id': 1, 'ip': '10.0.0.1', - 'credentials': {'version': 2} - } - """ - logging.debug( - 'wrap dict %s by support_keys=%s filters=%s', - data, support_keys, filters - ) - if isinstance(data, list): - return [ - _wrapper_dict(item, support_keys, **filters) - for item in data - ] - if isinstance(data, models.HelperMixin): - data = data.to_dict() - if not isinstance(data, dict): - raise exception.InvalidResponse( - 'response %s type is not dict' % data - ) - info = {} - try: - for key in support_keys: - if key in data and data[key] is not None: - if key in filters: - filter_keys = filters[key] - if isinstance(filter_keys, dict): - info[key] = _wrapper_dict( - data[key], filter_keys.keys(), - **filter_keys - ) - else: - info[key] = _wrapper_dict( - data[key], filter_keys - ) - else: - info[key] = data[key] - return info - except Exception as error: - logging.exception(error) - raise error - - -def replace_filters(**kwarg_mapping): - """Decorator to replace kwargs. - - Examples: - kwargs: {'a': 'b'}, kwarg_mapping: {'a': 'c'} - replaced kwargs to decorated func: - {'c': 'b'} - - replace_filters is used to replace caller's input - to make it understandable by models.py. - """ - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - replaced_kwargs = {} - for key, value in kwargs.items(): - if key in kwarg_mapping: - replaced_kwargs[kwarg_mapping[key]] = value - else: - replaced_kwargs[key] = value - return func(*args, **replaced_kwargs) - return wrapper - return decorator - - -def supported_filters( - support_keys=[], - optional_support_keys=[], - ignore_support_keys=[], -): - """Decorator to check kwargs keys. - - keys in kwargs and in ignore_support_keys will be removed. - If any unsupported keys found, a InvalidParameter - exception raises. - - Args: - support_keys: keys that must exist. - optional_support_keys: keys that may exist. - ignore_support_keys: keys should be ignored. - - Assumption: args without default value is supposed to exist. - You can add them in support_keys or not but we will make sure - it appears when we call the decorated function. - We do best match on both args and kwargs to make sure if the - key appears or not. - - Examples: - decorated func: func(a, b, c=3, d=4, **kwargs) - - support_keys=['e'] and call func(e=5): - raises: InvalidParameter: missing declared arg - support_keys=['e'] and call func(1,2,3,4,5,e=6): - raises: InvalidParameter: caller sending more args - support_keys=['e'] and call func(1,2): - raises: InvalidParameter: supported keys ['e'] missing - support_keys=['d', 'e'] and call func(1,2,e=3): - raises: InvalidParameter: supported keys ['d'] missing - support_keys=['d', 'e'] and call func(1,2,d=4, e=3): - passed - support_keys=['d'], optional_support_keys=['e'] - and call func(1,2, d=3): - passed - support_keys=['d'], optional_support_keys=['e'] - and call func(1,2, d=3, e=4, f=5): - raises: InvalidParameter: unsupported keys ['f'] - support_keys=['d'], optional_support_keys=['e'], - ignore_support_keys=['f'] - and call func(1,2, d=3, e=4, f=5): - passed to decorated keys: func(1,2, d=3, e=4) - """ - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **filters): - wrapped_func = get_wrapped_func(func) - argspec = inspect.getargspec(wrapped_func) - wrapped_args = argspec.args - args_defaults = argspec.defaults - # wrapped_must_args are positional args caller must pass in. - if args_defaults: - wrapped_must_args = wrapped_args[:-len(args_defaults)] - else: - wrapped_must_args = wrapped_args[:] - # make sure any positional args without default value in - # decorated function should appear in args or filters. - if len(args) < len(wrapped_must_args): - remain_args = wrapped_must_args[len(args):] - for remain_arg in remain_args: - if remain_arg not in filters: - raise exception.InvalidParameter( - 'function missing declared arg %s ' - 'while caller sends args %s' % ( - remain_arg, args - ) - ) - # make sure args should be no more than positional args - # declared in decorated function. - if len(args) > len(wrapped_args): - raise exception.InvalidParameter( - 'function definition args %s while the caller ' - 'sends args %s' % ( - wrapped_args, args - ) - ) - # exist_args are positional args caller has given. - exist_args = dict(zip(wrapped_args, args)).keys() - must_support_keys = set(support_keys) - all_support_keys = must_support_keys | set(optional_support_keys) - wrapped_supported_keys = set(filters) | set(exist_args) - unsupported_keys = ( - set(filters) - set(wrapped_args) - - all_support_keys - set(ignore_support_keys) - ) - # unsupported_keys are the keys that are not in support_keys, - # optional_support_keys, ignore_support_keys and are not passed in - # by positional args. It means the decorated function may - # not understand these parameters. - if unsupported_keys: - raise exception.InvalidParameter( - 'filter keys %s are not supported for %s' % ( - list(unsupported_keys), wrapped_func - ) - ) - # missing_keys are the keys that must exist but missing in - # both positional args or kwargs. - missing_keys = must_support_keys - wrapped_supported_keys - if missing_keys: - raise exception.InvalidParameter( - 'filter keys %s not found for %s' % ( - list(missing_keys), wrapped_func - ) - ) - # We filter kwargs to eliminate ignore_support_keys in kwargs - # passed to decorated function. - filtered_filters = dict([ - (key, value) - for key, value in filters.items() - if key not in ignore_support_keys - ]) - return func(*args, **filtered_filters) - return wrapper - return decorator - - -def input_filters( - **filters -): - """Decorator to filter kwargs. - - For key in kwargs, if the key exists and filters - and the return of call filters[key] is False, the key - will be removed from kwargs. - - The function definition of filters[key] is - func(value, *args, **kwargs) compared with decorated - function func(*args, **kwargs) - - The function is used to filter kwargs in case some - kwargs should be removed conditionally depends on the - related filters. - - Examples: - filters={'a': func(value, *args, **kwargs)} - @input_filters(**filters) - decorated_func(*args, **kwargs) - func returns False. - Then when call decorated_func(a=1, b=2) - it will be actually called the decorated func with - b=2. a=1 will be removed since it does not pass filtering. - """ - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - filtered_kwargs = {} - for key, value in kwargs.items(): - if key in filters: - if filters[key](value, *args, **kwargs): - filtered_kwargs[key] = value - else: - logging.debug( - 'ignore filtered input key %s' % key - ) - else: - filtered_kwargs[key] = value - return func(*args, **filtered_kwargs) - return wrapper - return decorator - - -def _obj_equal_or_subset(check, obj): - """Used by output filter to check if obj is in check.""" - if check == obj: - return True - if not issubclass(obj.__class__, check.__class__): - return False - if isinstance(obj, dict): - return _dict_equal_or_subset(check, obj) - elif isinstance(obj, list): - return _list_equal_or_subset(check, obj) - else: - return False - - -def _list_equal_or_subset(check_list, obj_list): - """Used by output filter to check if obj_list is in check_list""" - if not isinstance(check_list, list): - return False - return set(check_list).issubset(set(obj_list)) - - -def _dict_equal_or_subset(check_dict, obj_dict): - """Used by output filter to check if obj_dict in check_dict.""" - if not isinstance(check_dict, dict): - return False - for key, value in check_dict.items(): - if ( - key not in obj_dict or - not _obj_equal_or_subset(check_dict[key], obj_dict[key]) - ): - return False - return True - - -def general_filter_callback(general_filter, obj): - """General filter function to filter output. - - Since some fields stored in database is json encoded and - we want to do the deep match for the json encoded field to - do the filtering in some cases, we introduces the output_filters - and general_filter_callback to deal with this kind of cases. - - We do special treatment for key 'resp_eq' to check if - obj is the recursively subset of general_filter['resp_eq'] - - - Example: - obj: 'b' - general_filter: {} - returns: True - - obj: 'b' - general_filter: {'resp_in': ['a', 'b']} - returns: True - - obj: 'b' - general_filter: {'resp_in': ['a']} - returns: False - - obj: 'b' - general_filter: {'resp_eq': 'b'} - returns: True - - obj: 'b' - general_filter: {'resp_eq': 'a'} - returns: False - - obj: 'b' - general_filter: {'resp_range': ('a', 'c')} - returns: True - - obj: 'd' - general_filter: {'resp_range': ('a', 'c')} - returns: False - - If there are multi keys in dict, the output is filtered - by and relationship. - - If the general_filter is a list, the output is filtered - by or relationship. - - Supported general filters: [ - 'resp_eq', 'resp_in', 'resp_lt', - 'resp_le', 'resp_gt', 'resp_ge', - 'resp_match', 'resp_range' - ] - """ - if isinstance(general_filter, list): - if not general_filter: - return True - return any([ - general_filter_callback(item, obj) - for item in general_filter - ]) - elif isinstance(general_filter, dict): - if 'resp_eq' in general_filter: - if not _obj_equal_or_subset( - general_filter['resp_eq'], obj - ): - return False - if 'resp_in' in general_filter: - in_filters = general_filter['resp_in'] - if not any([ - _obj_equal_or_subset(in_filer, obj) - for in_filer in in_filters - ]): - return False - if 'resp_lt' in general_filter: - if obj >= general_filter['resp_lt']: - return False - if 'resp_le' in general_filter: - if obj > general_filter['resp_le']: - return False - if 'resp_gt' in general_filter: - if obj <= general_filter['resp_gt']: - return False - if 'resp_ge' in general_filter: - if obj < general_filter['resp_gt']: - return False - if 'resp_match' in general_filter: - if not re.match(general_filter['resp_match'], obj): - return False - if 'resp_range' in general_filter: - resp_range = general_filter['resp_range'] - if not isinstance(resp_range, list): - resp_range = [resp_range] - in_range = False - for range_start, range_end in resp_range: - if range_start <= obj <= range_end: - in_range = True - if not in_range: - return False - return True - else: - return True - - -def filter_output(filter_callbacks, kwargs, obj, missing_ok=False): - """Filter ouput. - - For each key in filter_callbacks, if it exists in kwargs, - kwargs[key] tells what we need to filter. If the call of - filter_callbacks[key] returns False, it tells the obj should be - filtered out of output. - """ - for callback_key, callback_value in filter_callbacks.items(): - if callback_key not in kwargs: - continue - if callback_key not in obj: - if missing_ok: - continue - else: - raise exception.InvalidResponse( - '%s is not in %s' % (callback_key, obj) - ) - if not callback_value( - kwargs[callback_key], obj[callback_key] - ): - return False - return True - - -def output_filters(missing_ok=False, **filter_callbacks): - """Decorator to filter output list. - - Each filter_callback should have the definition like: - func({'resp_eq': 'a'}, 'a') - """ - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - filtered_obj_list = [] - obj_list = func(*args, **kwargs) - for obj in obj_list: - if filter_output( - filter_callbacks, kwargs, obj, missing_ok - ): - filtered_obj_list.append(obj) - return filtered_obj_list - return wrapper - return decorator - - -def _input_validates(args_validators, kwargs_validators, *args, **kwargs): - """Used by input_validators to validate inputs.""" - for i, value in enumerate(args): - if i < len(args_validators) and args_validators[i]: - args_validators[i](value) - for key, value in kwargs.items(): - if kwargs_validators.get(key): - kwargs_validators[key](value) - - -def input_validates(*args_validators, **kwargs_validators): - """Decorator to validate input. - - Each validator should have definition like: - func('00:01:02:03:04:05') - """ - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - _input_validates( - args_validators, kwargs_validators, - *args, **kwargs - ) - return func(*args, **kwargs) - return wrapper - return decorator - - -def _input_validates_with_args( - args_validators, kwargs_validators, *args, **kwargs -): - """Validate input with validators. - - Each validator takes the arguments of the decorated function - as its arguments. The function definition is like: - func(value, *args, **kwargs) compared with the decorated - function func(*args, **kwargs). - """ - for i, value in enumerate(args): - if i < len(args_validators) and args_validators[i]: - args_validators[i](value, *args, **kwargs) - for key, value in kwargs.items(): - if kwargs_validators.get(key): - kwargs_validators[key](value, *args, **kwargs) - - -def input_validates_with_args( - *args_validators, **kwargs_validators -): - """Decorator to validate input.""" - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - _input_validates_with_args( - args_validators, kwargs_validators, - *args, **kwargs - ) - return func(*args, **kwargs) - return wrapper - return decorator - - -def _output_validates_with_args( - kwargs_validators, obj, *args, **kwargs -): - """Validate output with validators. - - Each validator takes the arguments of the decorated function - as its arguments. The function definition is like: - func(value, *args, **kwargs) compared with the decorated - function func(*args, **kwargs). - """ - if isinstance(obj, list): - for item in obj: - _output_validates_with_args( - kwargs_validators, item, *args, **kwargs - ) - return - if isinstance(obj, models.HelperMixin): - obj = obj.to_dict() - if not isinstance(obj, dict): - raise exception.InvalidResponse( - 'response %s type is not dict' % str(obj) - ) - try: - for key, value in obj.items(): - if key in kwargs_validators: - kwargs_validators[key](value, *args, **kwargs) - except Exception as error: - logging.exception(error) - raise error - - -def output_validates_with_args(**kwargs_validators): - """Decorator to validate output. - - The validator can take the arguments of the decorated - function as its arguments. - """ - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - obj = func(*args, **kwargs) - if isinstance(obj, list): - for obj_item in obj: - _output_validates_with_args( - kwargs_validators, obj_item, - *args, **kwargs - ) - else: - _output_validates_with_args( - kwargs_validators, obj, - *args, **kwargs - ) - return obj - return wrapper - return decorator - - -def _output_validates(kwargs_validators, obj): - """Validate output. - - Each validator has following signature: - func(value) - """ - if isinstance(obj, list): - for item in obj: - _output_validates(kwargs_validators, item) - return - if isinstance(obj, models.HelperMixin): - obj = obj.to_dict() - if not isinstance(obj, dict): - raise exception.InvalidResponse( - 'response %s type is not dict' % str(obj) - ) - try: - for key, value in obj.items(): - if key in kwargs_validators: - kwargs_validators[key](value) - except Exception as error: - logging.exception(error) - raise error - - -def output_validates(**kwargs_validators): - """Decorator to validate output.""" - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - obj = func(*args, **kwargs) - if isinstance(obj, list): - for obj_item in obj: - _output_validates(kwargs_validators, obj_item) - else: - _output_validates(kwargs_validators, obj) - return obj - return wrapper - return decorator - - -def get_db_object(session, table, exception_when_missing=True, **kwargs): - """Get db object. - - If not exception_when_missing and the db object can not be found, - return None instead of raising exception. - """ - if not session: - raise exception.DatabaseException('session param is None') - with session.begin(subtransactions=True): - logging.debug( - 'session %s get db object %s from table %s', - id(session), kwargs, table.__name__) - db_object = model_filter( - model_query(session, table), table, **kwargs - ).first() - logging.debug( - 'session %s got db object %s', id(session), db_object - ) - if db_object: - return db_object - - if not exception_when_missing: - return None - - raise exception.RecordNotExists( - 'Cannot find the record in table %s: %s' % ( - table.__name__, kwargs - ) - ) - - -def add_db_object(session, table, exception_when_existing=True, - *args, **kwargs): - """Create db object. - - If not exception_when_existing and the db object exists, - Instead of raising exception, updating the existing db object. - """ - if not session: - raise exception.DatabaseException('session param is None') - with session.begin(subtransactions=True): - logging.debug( - 'session %s add object %s atributes %s to table %s', - id(session), args, kwargs, table.__name__) - argspec = inspect.getargspec(table.__init__) - arg_names = argspec.args[1:] - arg_defaults = argspec.defaults - if not arg_defaults: - arg_defaults = [] - if not ( - len(arg_names) - len(arg_defaults) <= len(args) <= len(arg_names) - ): - raise exception.InvalidParameter( - 'arg names %s does not match arg values %s' % ( - arg_names, args) - ) - db_keys = dict(zip(arg_names, args)) - if db_keys: - db_object = session.query(table).filter_by(**db_keys).first() - else: - db_object = None - - new_object = False - if db_object: - logging.debug( - 'got db object %s: %s', db_keys, db_object - ) - if exception_when_existing: - raise exception.DuplicatedRecord( - '%s exists in table %s' % (db_keys, table.__name__) - ) - else: - db_object = table(**db_keys) - new_object = True - - for key, value in kwargs.items(): - setattr(db_object, key, value) - - if new_object: - session.add(db_object) - session.flush() - db_object.initialize() - db_object.validate() - logging.debug( - 'session %s db object %s added', id(session), db_object - ) - return db_object - - -def list_db_objects(session, table, order_by=[], **filters): - """List db objects. - - If order by given, the db objects should be sorted by the ordered keys. - """ - if not session: - raise exception.DatabaseException('session param is None') - with session.begin(subtransactions=True): - logging.debug( - 'session %s list db objects by filters %s in table %s', - id(session), filters, table.__name__ - ) - db_objects = model_order_by( - model_filter( - model_query(session, table), - table, - **filters - ), - table, - order_by - ).all() - logging.debug( - 'session %s got listed db objects: %s', - id(session), db_objects - ) - return db_objects - - -def del_db_objects(session, table, **filters): - """delete db objects.""" - if not session: - raise exception.DatabaseException('session param is None') - with session.begin(subtransactions=True): - logging.debug( - 'session %s delete db objects by filters %s in table %s', - id(session), filters, table.__name__ - ) - query = model_filter( - model_query(session, table), table, **filters - ) - db_objects = query.all() - query.delete(synchronize_session=False) - logging.debug( - 'session %s db objects %s deleted', id(session), db_objects - ) - return db_objects - - -def update_db_objects(session, table, updates={}, **filters): - """Update db objects.""" - if not session: - raise exception.DatabaseException('session param is None') - with session.begin(subtransactions=True): - logging.debug( - 'session %s update db objects by filters %s in table %s', - id(session), filters, table.__name__) - db_objects = model_filter( - model_query(session, table), table, **filters - ).all() - for db_object in db_objects: - logging.debug('update db object %s: %s', db_object, updates) - update_db_object(session, db_object, **updates) - logging.debug( - 'session %s db objects %s updated', - id(session), db_objects - ) - return db_objects - - -def update_db_object(session, db_object, **kwargs): - """Update db object.""" - if not session: - raise exception.DatabaseException('session param is None') - with session.begin(subtransactions=True): - logging.debug( - 'session %s update db object %s by value %s', - id(session), db_object, kwargs - ) - for key, value in kwargs.items(): - setattr(db_object, key, value) - session.flush() - db_object.update() - db_object.validate() - logging.debug( - 'session %s db object %s updated', - id(session), db_object - ) - return db_object - - -def del_db_object(session, db_object): - """Delete db object.""" - if not session: - raise exception.DatabaseException('session param is None') - with session.begin(subtransactions=True): - logging.debug( - 'session %s delete db object %s', - id(session), db_object - ) - session.delete(db_object) - logging.debug( - 'session %s db object %s deleted', - id(session), db_object - ) - return db_object - - -def check_ip(ip): - """Check ip is ip address formatted.""" - try: - netaddr.IPAddress(ip) - except Exception as error: - logging.exception(error) - raise exception.InvalidParameter( - 'ip address %s format uncorrect' % ip - ) - - -def check_mac(mac): - """Check mac is mac address formatted.""" - try: - netaddr.EUI(mac) - except Exception as error: - logging.exception(error) - raise exception.InvalidParameter( - 'invalid mac address %s' % mac - ) - - -NAME_PATTERN = re.compile(r'[a-zA-Z0-9][a-zA-Z0-9_-]*') - - -def check_name(name): - """Check name meeting name format requirement.""" - if not NAME_PATTERN.match(name): - raise exception.InvalidParameter( - 'name %s does not match the pattern %s' % ( - name, NAME_PATTERN.pattern - ) - ) - - -def _check_ipmi_credentials_ip(ip): - check_ip(ip) - - -def check_ipmi_credentials(ipmi_credentials): - """Check ipmi credentials format is correct.""" - if not ipmi_credentials: - return - if not isinstance(ipmi_credentials, dict): - raise exception.InvalidParameter( - 'invalid ipmi credentials %s' % ipmi_credentials - - ) - for key in ipmi_credentials: - if key not in ['ip', 'username', 'password']: - raise exception.InvalidParameter( - 'unrecognized field %s in ipmi credentials %s' % ( - key, ipmi_credentials - ) - ) - for key in ['ip', 'username', 'password']: - if key not in ipmi_credentials: - raise exception.InvalidParameter( - 'no field %s in ipmi credentials %s' % ( - key, ipmi_credentials - ) - ) - check_ipmi_credential_field = '_check_ipmi_credentials_%s' % key - this_module = globals() - if check_ipmi_credential_field in this_module: - this_module[check_ipmi_credential_field]( - ipmi_credentials[key] - ) - else: - logging.debug( - 'function %s is not defined', check_ipmi_credential_field - ) - - -def _check_switch_credentials_version(version): - if version not in ['1', '2c', '3']: - raise exception.InvalidParameter( - 'unknown snmp version %s' % version - ) - - -def check_switch_credentials(credentials): - """Check switch credentials format is correct.""" - if not credentials: - return - if not isinstance(credentials, dict): - raise exception.InvalidParameter( - 'credentials %s is not dict' % credentials - ) - for key in credentials: - if key not in ['version', 'community']: - raise exception.InvalidParameter( - 'unrecognized key %s in credentials %s' % (key, credentials) - ) - for key in ['version', 'community']: - if key not in credentials: - raise exception.InvalidParameter( - 'there is no %s field in credentials %s' % (key, credentials) - ) - - key_check_func_name = '_check_switch_credentials_%s' % key - this_module = globals() - if key_check_func_name in this_module: - this_module[key_check_func_name]( - credentials[key] - ) - else: - logging.debug( - 'function %s is not defined', - key_check_func_name - ) diff --git a/compass-tasks/db/callback.py b/compass-tasks/db/callback.py deleted file mode 100644 index 35798bc..0000000 --- a/compass-tasks/db/callback.py +++ /dev/null @@ -1,204 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Metadata Callback methods.""" -import logging -import netaddr -import random -import re -import socket - -from compass.db import exception -from compass.utils import setting_wrapper as setting -from compass.utils import util - - -CALLBACK_GLOBALS = globals() -CALLBACK_LOCALS = locals() -CALLBACK_CONFIGS = util.load_configs( - setting.CALLBACK_DIR, - config_name_suffix='.py', - env_globals=CALLBACK_GLOBALS, - env_locals=CALLBACK_LOCALS -) -for callback_config in CALLBACK_CONFIGS: - CALLBACK_LOCALS.update(callback_config) - - -def default_proxy(name, **kwargs): - return setting.COMPASS_SUPPORTED_PROXY - - -def proxy_options(name, **kwargs): - return [setting.COMPASS_SUPPORTED_PROXY] - - -def default_noproxy(name, **kwargs): - return setting.COMPASS_SUPPORTED_DEFAULT_NOPROXY - - -def noproxy_options(name, **kwargs): - return setting.COMPASS_SUPPORTED_DEFAULT_NOPROXY - - -def default_ntp_server(name, **kwargs): - return setting.COMPASS_SUPPORTED_NTP_SERVER - - -def ntp_server_options(name, **kwargs): - return setting.COMPASS_SUPPORTED_NTP_SERVER - - -def default_dns_servers(name, **kwargs): - return setting.COMPASS_SUPPORTED_DNS_SERVERS - - -def dns_servers_options(name, **kwargs): - return setting.COMPASS_SUPPORTED_DNS_SERVERS - - -def default_domain(name, **kwargs): - if setting.COMPASS_SUPPORTED_DOMAINS: - return setting.COMPASS_SUPPORTED_DOMAINS[0] - else: - return None - - -def domain_options(name, **kwargs): - return setting.COMPASS_SUPPORTED_DOMAINS - - -def default_search_path(name, **kwargs): - return setting.COMPASS_SUPPORTED_DOMAINS - - -def search_path_options(name, **kwargs): - return setting.COMPASS_SUPPORTED_DOMAINS - - -def default_gateway(name, **kwargs): - return setting.COMPASS_SUPPORTED_DEFAULT_GATEWAY - - -def default_gateway_options(name, **kwargs): - return [setting.COMPASS_SUPPORTED_DEFAULT_GATEWAY] - - -def default_localrepo(name, **kwargs): - return setting.COMPASS_SUPPORTED_LOCAL_REPO - - -def default_localrepo_options(name, **kwargs): - return [setting.COMPASS_SUPPORTED_LOCAL_REPO] - - -def autofill_callback_default(name, config, **kwargs): - if config is None: - if ( - 'autofill_types' not in kwargs or - not (set(kwargs['autofill_types']) & set(kwargs)) - ): - return None - if 'default_value' not in kwargs: - return None - return kwargs['default_value'] - return config - - -def autofill_callback_random_option(name, config, **kwargs): - if config is None: - if ( - 'autofill_types' not in kwargs or - not (set(kwargs['autofill_types']) & set(kwargs)) - ): - return None - if 'options' not in kwargs or not kwargs['options']: - return None - return random.choice(kwargs['options']) - return config - - -def autofill_no_proxy(name, config, **kwargs): - logging.debug( - 'autofill %s config %s by params %s', - name, config, kwargs - ) - if 'cluster' in kwargs: - if config is None: - config = [] - if 'default_value' in kwargs: - for default_no_proxy in kwargs['default_value']: - if default_no_proxy and default_no_proxy not in config: - config.append(default_no_proxy) - cluster = kwargs['cluster'] - for clusterhost in cluster.clusterhosts: - host = clusterhost.host - hostname = host.name - if hostname not in config: - config.append(hostname) - for host_network in host.host_networks: - if host_network.is_mgmt: - ip = host_network.ip - if ip not in config: - config.append(ip) - if not config: - return config - return [no_proxy for no_proxy in config if no_proxy] - - -def autofill_network_mapping(name, config, **kwargs): - logging.debug( - 'autofill %s config %s by params %s', - name, config, kwargs - ) - if not config: - return config - if isinstance(config, basestring): - config = { - 'interface': config, - 'subnet': None - } - if not isinstance(config, dict): - return config - if 'interface' not in config: - return config - subnet = None - interface = config['interface'] - if 'cluster' in kwargs: - cluster = kwargs['cluster'] - for clusterhost in cluster.clusterhosts: - host = clusterhost.host - for host_network in host.host_networks: - if host_network.interface == interface: - subnet = host_network.subnet.subnet - elif 'clusterhost' in kwargs: - clusterhost = kwargs['clusterhost'] - host = clusterhost.host - for host_network in host.host_networks: - if host_network.interface == interface: - subnet = host_network.subnet.subnet - if not subnet: - raise exception.InvalidParameter( - 'interface %s not found in host(s)' % interface - ) - if 'subnet' not in config or not config['subnet']: - config['subnet'] = subnet - else: - if config['subnet'] != subnet: - raise exception.InvalidParameter( - 'subnet %s in config is not equal to subnet %s in hosts' % ( - config['subnet'], subnet - ) - ) - return config diff --git a/compass-tasks/db/config_validation/__init__.py b/compass-tasks/db/config_validation/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compass-tasks/db/config_validation/default_validator.py b/compass-tasks/db/config_validation/default_validator.py deleted file mode 100644 index 224447f..0000000 --- a/compass-tasks/db/config_validation/default_validator.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Default config validation function.""" - -from sqlalchemy import or_ - -from compass.db.models import OSConfigField -from compass.db.models import OSConfigMetadata -from compass.db import validator - -MAPPER = { - "os_id": { - "metaTable": OSConfigMetadata, - "metaFieldTable": OSConfigField - } - # "adapter_id": { - # "metaTable": AdapterConfigMetadata, - # "metaFieldTable": AdapterConfigField - # } -} - - -def validate_config(session, config, id_name, id_value, patch=True): - """Validates config. - - Validates the given config value according to the config - metadata of the asscoiated os_id or adapter_id. Returns - a tuple (status, message). - """ - if id_name not in MAPPER.keys(): - return (False, "Invalid id type %s" % id_name) - - meta_table = MAPPER[id_name]['metaTable'] - metafield_table = MAPPER[id_name]['metaFieldTable'] - with session.begin(subtransactions=True): - name_col = name_col = getattr(meta_table, 'name') - id_col = getattr(meta_table, id_name) - - return _validate_config_helper(session, config, - name_col, id_col, id_value, - meta_table, metafield_table, - patch) - - -def _validate_config_helper(session, config, - name_col, id_col, id_value, - meta_table, metafield_table, patch=True): - - with session.begin(subtransactions=True): - for elem in config: - - obj = session.query(meta_table).filter(name_col == elem)\ - .filter(or_(id_col is None, - id_col == id_value)).first() - - if not obj and "_type" not in config[elem]: - return (False, "Invalid metadata '%s'!" % elem) - - if "_type" in config[elem]: - # Metadata is a variable - metadata_name = config[elem]['_type'] - obj = session.query(meta_table).filter_by(name=metadata_name)\ - .first() - - if not obj: - err_msg = ("Invalid metatdata '%s' or missing '_type'" - "to indicate this is a variable metatdata." - % elem) - return (False, err_msg) - - # TODO(Grace): validate metadata here - del config[elem]['_type'] - - fields = obj.fields - - if not fields: - is_valid, message = _validate_config_helper(session, - config[elem], - name_col, id_col, - id_value, - meta_table, - metafield_table, - patch) - if not is_valid: - return (False, message) - - else: - field_config = config[elem] - for key in field_config: - field = session.query(metafield_table)\ - .filter_by(field=key).first() - if not field: - # The field is not in schema - return (False, "Invalid field '%s'!" % key) - - value = field_config[key] - if field.is_required and value is None: - # The value of this field is required - # and cannot be none - err = "The value of field '%s' cannot be null!" % key - return (False, err) - - if field.validator: - func = getattr(validator, field.validator) - if not func or not func(value): - err_msg = ("The value of the field '%s' is " - "invalid format or None!" % key) - return (False, err_msg) - - # This is a PUT request. We need to check presence of all - # required fields. - if not patch: - for field in fields: - name = field.field - if field.is_required and name not in field_config: - return (False, - "Missing required field '%s'" % name) - - return (True, None) diff --git a/compass-tasks/db/config_validation/extension/__init__.py b/compass-tasks/db/config_validation/extension/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compass-tasks/db/config_validation/extension/openstack.py b/compass-tasks/db/config_validation/extension/openstack.py deleted file mode 100644 index 6b3af69..0000000 --- a/compass-tasks/db/config_validation/extension/openstack.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def validate_cluster_config(): - # TODO(xiaodong): Add openstack specific validation here. - pass diff --git a/compass-tasks/db/exception.py b/compass-tasks/db/exception.py deleted file mode 100644 index 44556c9..0000000 --- a/compass-tasks/db/exception.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Custom exception""" -import traceback - - -class DatabaseException(Exception): - """Base class for all database exceptions.""" - def __init__(self, message): - super(DatabaseException, self).__init__(message) - self.traceback = traceback.format_exc() - self.status_code = 400 - - def to_dict(self): - return {'message': str(self)} - - -class RecordNotExists(DatabaseException): - """Define the exception for referring non-existing object in DB.""" - def __init__(self, message): - super(RecordNotExists, self).__init__(message) - self.status_code = 404 - - -class DuplicatedRecord(DatabaseException): - """Define the exception for trying to insert an existing object in DB.""" - def __init__(self, message): - super(DuplicatedRecord, self).__init__(message) - self.status_code = 409 - - -class Unauthorized(DatabaseException): - """Define the exception for invalid user login.""" - def __init__(self, message): - super(Unauthorized, self).__init__(message) - self.status_code = 401 - - -class UserDisabled(DatabaseException): - """Define the exception that a disabled user tries to do some operations. - - """ - def __init__(self, message): - super(UserDisabled, self).__init__(message) - self.status_code = 403 - - -class Forbidden(DatabaseException): - """Define the exception that a user is trying to make some action - - without the right permission. - - """ - def __init__(self, message): - super(Forbidden, self).__init__(message) - self.status_code = 403 - - -class NotAcceptable(DatabaseException): - """The data is not acceptable.""" - def __init__(self, message): - super(NotAcceptable, self).__init__(message) - self.status_code = 406 - - -class InvalidParameter(DatabaseException): - """Define the exception that the request has invalid or missing parameters. - - """ - def __init__(self, message): - super(InvalidParameter, self).__init__(message) - self.status_code = 400 - - -class InvalidResponse(DatabaseException): - """Define the exception that the response is invalid. - - """ - def __init__(self, message): - super(InvalidResponse, self).__init__(message) - self.status_code = 400 - - -class MultiDatabaseException(DatabaseException): - """Define the exception composites with multi exceptions.""" - def __init__(self, exceptions): - super(MultiDatabaseException, self).__init__('multi exceptions') - self.exceptions = exceptions - self.status_code = 400 - - @property - def traceback(self): - tracebacks = [] - for exception in self.exceptions: - tracebacks.append(exception.trackback) - - def to_dict(self): - dict_info = super(MultiDatabaseException, self).to_dict() - dict_info.update({ - 'exceptions': [ - exception.to_dict() for exception in self.exceptions - ] - }) - return dict_info diff --git a/compass-tasks/db/models.py b/compass-tasks/db/models.py deleted file mode 100644 index d4b0324..0000000 --- a/compass-tasks/db/models.py +++ /dev/null @@ -1,1924 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Database model""" -import copy -import datetime -import logging -import netaddr -import re -import simplejson as json - -from sqlalchemy import BigInteger -from sqlalchemy import Boolean -from sqlalchemy import Column -from sqlalchemy import ColumnDefault -from sqlalchemy import DateTime -from sqlalchemy import Enum -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy import Float -from sqlalchemy import ForeignKey -from sqlalchemy import Integer -from sqlalchemy.orm import relationship, backref -from sqlalchemy import String -from sqlalchemy import Table -from sqlalchemy import Text -from sqlalchemy.types import TypeDecorator -from sqlalchemy import UniqueConstraint - -from compass.db import exception -from compass.utils import util - - -BASE = declarative_base() - - -class JSONEncoded(TypeDecorator): - """Represents an immutable structure as a json-encoded string.""" - - impl = Text - - def process_bind_param(self, value, dialect): - if value is not None: - value = json.dumps(value) - return value - - def process_result_value(self, value, dialect): - if value is not None: - value = json.loads(value) - return value - - -class TimestampMixin(object): - """Provides table fields for each row created/updated timestamp.""" - created_at = Column(DateTime, default=lambda: datetime.datetime.now()) - updated_at = Column(DateTime, default=lambda: datetime.datetime.now(), - onupdate=lambda: datetime.datetime.now()) - - -class HelperMixin(object): - """Provides general fuctions for all compass table models.""" - - def initialize(self): - self.update() - - def update(self): - pass - - @staticmethod - def type_compatible(value, column_type): - """Check if value type is compatible with the column type.""" - if value is None: - return True - if not hasattr(column_type, 'python_type'): - return True - column_python_type = column_type.python_type - if isinstance(value, column_python_type): - return True - if issubclass(column_python_type, basestring): - return isinstance(value, basestring) - if column_python_type in [int, long]: - return type(value) in [int, long] - if column_python_type in [float]: - return type(value) in [float] - if column_python_type in [bool]: - return type(value) in [bool] - return False - - def validate(self): - """Generate validate function to make sure the record is legal.""" - columns = self.__mapper__.columns - for key, column in columns.items(): - value = getattr(self, key) - if not self.type_compatible(value, column.type): - raise exception.InvalidParameter( - 'column %s value %r type is unexpected: %s' % ( - key, value, column.type - ) - ) - - def to_dict(self): - """General function to convert record to dict. - - Convert all columns not starting with '_' to - {: } - """ - keys = self.__mapper__.columns.keys() - dict_info = {} - for key in keys: - if key.startswith('_'): - continue - value = getattr(self, key) - if value is not None: - if isinstance(value, datetime.datetime): - value = util.format_datetime(value) - dict_info[key] = value - return dict_info - - -class StateMixin(TimestampMixin, HelperMixin): - """Provides general fields and functions for state related table.""" - - state = Column( - Enum( - 'UNINITIALIZED', 'INITIALIZED', 'UPDATE_PREPARING', - 'INSTALLING', 'SUCCESSFUL', 'ERROR' - ), - ColumnDefault('UNINITIALIZED') - ) - percentage = Column(Float, default=0.0) - message = Column(Text, default='') - severity = Column( - Enum('INFO', 'WARNING', 'ERROR'), - ColumnDefault('INFO') - ) - ready = Column(Boolean, default=False) - - def update(self): - # In state table, some field information is redundant. - # The update function to make sure all related fields - # are set to correct state. - if self.ready: - self.state = 'SUCCESSFUL' - if self.state in ['UNINITIALIZED', 'INITIALIZED']: - self.percentage = 0.0 - self.severity = 'INFO' - self.message = '' - if self.state == 'INSTALLING': - if self.severity == 'ERROR': - self.state = 'ERROR' - elif self.percentage >= 1.0: - self.state = 'SUCCESSFUL' - self.percentage = 1.0 - if self.state == 'SUCCESSFUL': - self.percentage = 1.0 - super(StateMixin, self).update() - - -class LogHistoryMixin(TimestampMixin, HelperMixin): - """Provides general fields and functions for LogHistory related tables.""" - position = Column(Integer, default=0) - partial_line = Column(Text, default='') - percentage = Column(Float, default=0.0) - message = Column(Text, default='') - severity = Column( - Enum('ERROR', 'WARNING', 'INFO'), - ColumnDefault('INFO') - ) - line_matcher_name = Column( - String(80), default='start' - ) - - def validate(self): - # TODO(xicheng): some validation can be moved to column. - if not self.filename: - raise exception.InvalidParameter( - 'filename is not set in %s' % self.id - ) - - -class HostNetwork(BASE, TimestampMixin, HelperMixin): - """Host network table.""" - __tablename__ = 'host_network' - - id = Column(Integer, primary_key=True) - host_id = Column( - Integer, - ForeignKey('host.id', onupdate='CASCADE', ondelete='CASCADE') - ) - interface = Column( - String(80), nullable=False) - subnet_id = Column( - Integer, - ForeignKey('subnet.id', onupdate='CASCADE', ondelete='CASCADE') - ) - user_id = Column(Integer, ForeignKey('user.id')) - ip_int = Column(BigInteger, nullable=False) - is_mgmt = Column(Boolean, default=False) - is_promiscuous = Column(Boolean, default=False) - - __table_args__ = ( - UniqueConstraint('host_id', 'interface', name='interface_constraint'), - UniqueConstraint('ip_int', 'user_id', name='ip_constraint') - ) - - def __init__(self, host_id, interface, user_id, **kwargs): - self.host_id = host_id - self.interface = interface - self.user_id = user_id - super(HostNetwork, self).__init__(**kwargs) - - def __str__(self): - return 'HostNetwork[%s=%s]' % (self.interface, self.ip) - - @property - def ip(self): - return str(netaddr.IPAddress(self.ip_int)) - - @ip.setter - def ip(self, value): - self.ip_int = int(netaddr.IPAddress(value)) - - @property - def netmask(self): - return str(netaddr.IPNetwork(self.subnet.subnet).netmask) - - def update(self): - self.host.config_validated = False - - def validate(self): - # TODO(xicheng): some validation can be moved to column. - super(HostNetwork, self).validate() - if not self.subnet: - raise exception.InvalidParameter( - 'subnet is not set in %s interface %s' % ( - self.host_id, self.interface - ) - ) - if not self.ip_int: - raise exception.InvalidParameter( - 'ip is not set in %s interface %s' % ( - self.host_id, self.interface - ) - ) - ip = netaddr.IPAddress(self.ip_int) - subnet = netaddr.IPNetwork(self.subnet.subnet) - if ip not in subnet: - raise exception.InvalidParameter( - 'ip %s is not in subnet %s' % ( - str(ip), str(subnet) - ) - ) - - def to_dict(self): - dict_info = super(HostNetwork, self).to_dict() - dict_info['ip'] = self.ip - dict_info['interface'] = self.interface - dict_info['netmask'] = self.netmask - dict_info['subnet'] = self.subnet.subnet - dict_info['user_id'] = self.user_id - return dict_info - - -class ClusterHostLogHistory(BASE, LogHistoryMixin): - """clusterhost installing log history for each file. - - """ - __tablename__ = 'clusterhost_log_history' - - clusterhost_id = Column( - 'id', Integer, - ForeignKey('clusterhost.id', onupdate='CASCADE', ondelete='CASCADE'), - primary_key=True - ) - filename = Column(String(80), primary_key=True, nullable=False) - cluster_id = Column( - Integer, - ForeignKey('cluster.id') - ) - host_id = Column( - Integer, - ForeignKey('host.id') - ) - - def __init__(self, clusterhost_id, filename, **kwargs): - self.clusterhost_id = clusterhost_id - self.filename = filename - super(ClusterHostLogHistory, self).__init__(**kwargs) - - def __str__(self): - return 'ClusterHostLogHistory[%s:%s]' % ( - self.clusterhost_id, self.filename - ) - - def initialize(self): - self.cluster_id = self.clusterhost.cluster_id - self.host_id = self.clusterhost.host_id - super(ClusterHostLogHistory, self).initialize() - - -class HostLogHistory(BASE, LogHistoryMixin): - """host installing log history for each file. - - """ - __tablename__ = 'host_log_history' - - id = Column( - Integer, - ForeignKey('host.id', onupdate='CASCADE', ondelete='CASCADE'), - primary_key=True) - filename = Column(String(80), primary_key=True, nullable=False) - - def __init__(self, id, filename, **kwargs): - self.id = id - self.filename = filename - super(HostLogHistory, self).__init__(**kwargs) - - def __str__(self): - return 'HostLogHistory[%s:%s]' % (self.id, self.filename) - - -class ClusterHostState(BASE, StateMixin): - """ClusterHost state table.""" - __tablename__ = 'clusterhost_state' - - id = Column( - Integer, - ForeignKey( - 'clusterhost.id', - onupdate='CASCADE', ondelete='CASCADE' - ), - primary_key=True - ) - - def __str__(self): - return 'ClusterHostState[%s state %s percentage %s]' % ( - self.id, self.state, self.percentage - ) - - def update(self): - """Update clusterhost state. - - When clusterhost state is updated, the underlying host state - may be updated accordingly. - """ - super(ClusterHostState, self).update() - host_state = self.clusterhost.host.state - if self.state == 'INITIALIZED': - if host_state.state in ['UNINITIALIZED', 'UPDATE_PREPARING']: - host_state.state = 'INITIALIZED' - host_state.update() - elif self.state == 'INSTALLING': - if host_state.state in [ - 'UNINITIALIZED', 'UPDATE_PREPARING', 'INITIALIZED' - ]: - host_state.state = 'INSTALLING' - host_state.update() - elif self.state == 'SUCCESSFUL': - if host_state.state != 'SUCCESSFUL': - host_state.state = 'SUCCESSFUL' - host_state.update() - - -class ClusterHost(BASE, TimestampMixin, HelperMixin): - """ClusterHost table.""" - __tablename__ = 'clusterhost' - - clusterhost_id = Column('id', Integer, primary_key=True) - cluster_id = Column( - Integer, - ForeignKey('cluster.id', onupdate='CASCADE', ondelete='CASCADE') - ) - host_id = Column( - Integer, - ForeignKey('host.id', onupdate='CASCADE', ondelete='CASCADE') - ) - # the list of role names. - _roles = Column('roles', JSONEncoded, default=[]) - _patched_roles = Column('patched_roles', JSONEncoded, default=[]) - config_step = Column(String(80), default='') - package_config = Column(JSONEncoded, default={}) - config_validated = Column(Boolean, default=False) - deployed_package_config = Column(JSONEncoded, default={}) - - log_histories = relationship( - ClusterHostLogHistory, - passive_deletes=True, passive_updates=True, - cascade='all, delete-orphan', - backref=backref('clusterhost') - ) - - __table_args__ = ( - UniqueConstraint('cluster_id', 'host_id', name='constraint'), - ) - - state = relationship( - ClusterHostState, - uselist=False, - passive_deletes=True, passive_updates=True, - cascade='all, delete-orphan', - backref=backref('clusterhost') - ) - - def __init__(self, cluster_id, host_id, **kwargs): - self.cluster_id = cluster_id - self.host_id = host_id - self.state = ClusterHostState() - super(ClusterHost, self).__init__(**kwargs) - - def __str__(self): - return 'ClusterHost[%s:%s]' % (self.clusterhost_id, self.name) - - def update(self): - if self.host.reinstall_os: - if self.state in ['SUCCESSFUL', 'ERROR']: - if self.config_validated: - self.state.state = 'INITIALIZED' - else: - self.state.state = 'UNINITIALIZED' - self.cluster.update() - self.host.update() - self.state.update() - super(ClusterHost, self).update() - - @property - def name(self): - return '%s.%s' % (self.host.name, self.cluster.name) - - @property - def patched_package_config(self): - return self.package_config - - @patched_package_config.setter - def patched_package_config(self, value): - package_config = copy.deepcopy(self.package_config) - self.package_config = util.merge_dict(package_config, value) - logging.debug( - 'patch clusterhost %s package_config: %s', - self.clusterhost_id, value - ) - self.config_validated = False - - @property - def put_package_config(self): - return self.package_config - - @put_package_config.setter - def put_package_config(self, value): - package_config = copy.deepcopy(self.package_config) - package_config.update(value) - self.package_config = package_config - logging.debug( - 'put clusterhost %s package_config: %s', - self.clusterhost_id, value - ) - self.config_validated = False - - @property - def patched_os_config(self): - return self.host.os_config - - @patched_os_config.setter - def patched_os_config(self, value): - host = self.host - host.patched_os_config = value - - @property - def put_os_config(self): - return self.host.os_config - - @put_os_config.setter - def put_os_config(self, value): - host = self.host - host.put_os_config = value - - @property - def deployed_os_config(self): - return self.host.deployed_os_config - - @deployed_os_config.setter - def deployed_os_config(self, value): - host = self.host - host.deployed_os_config = value - - @hybrid_property - def os_name(self): - return self.host.os_name - - @os_name.expression - def os_name(cls): - return cls.host.os_name - - @hybrid_property - def clustername(self): - return self.cluster.name - - @clustername.expression - def clustername(cls): - return cls.cluster.name - - @hybrid_property - def hostname(self): - return self.host.hostname - - @hostname.expression - def hostname(cls): - return Host.hostname - - @property - def distributed_system_installed(self): - return self.state.state == 'SUCCESSFUL' - - @property - def resintall_os(self): - return self.host.reinstall_os - - @property - def reinstall_distributed_system(self): - return self.cluster.reinstall_distributed_system - - @property - def os_installed(self): - return self.host.os_installed - - @property - def roles(self): - # only the role exists in flavor roles will be returned. - # the role will be sorted as the order defined in flavor - # roles. - # duplicate role names will be removed. - # The returned value is a list of dict like - # [{'name': 'allinone', 'optional': False}] - role_names = list(self._roles) - if not role_names: - return [] - cluster_roles = self.cluster.flavor['roles'] - if not cluster_roles: - return [] - roles = [] - for cluster_role in cluster_roles: - if cluster_role['name'] in role_names: - roles.append(cluster_role) - return roles - - @roles.setter - def roles(self, value): - """value should be a list of role name.""" - self._roles = list(value) - self.config_validated = False - - @property - def patched_roles(self): - patched_role_names = list(self._patched_roles) - if not patched_role_names: - return [] - cluster_roles = self.cluster.flavor['roles'] - if not cluster_roles: - return [] - roles = [] - for cluster_role in cluster_roles: - if cluster_role['name'] in patched_role_names: - roles.append(cluster_role) - return roles - - @patched_roles.setter - def patched_roles(self, value): - """value should be a list of role name.""" - # if value is an empty list, we empty the field - if value: - roles = list(self._roles) - roles.extend(value) - self._roles = roles - patched_roles = list(self._patched_roles) - patched_roles.extend(value) - self._patched_roles = patched_roles - self.config_validated = False - else: - self._patched_roles = list(value) - self.config_validated = False - - @hybrid_property - def owner(self): - return self.cluster.owner - - @owner.expression - def owner(cls): - return cls.cluster.owner - - def state_dict(self): - """Get clusterhost state dict. - - The clusterhost state_dict is different from - clusterhost.state.to_dict. The main difference is state_dict - show the progress of both installing os on host and installing - distributed system on clusterhost. While clusterhost.state.to_dict - only shows the progress of installing distributed system on - clusterhost. - """ - cluster = self.cluster - host = self.host - host_state = host.state_dict() - if not cluster.flavor_name: - return host_state - clusterhost_state = self.state.to_dict() - if clusterhost_state['state'] in ['ERROR', 'SUCCESSFUL']: - return clusterhost_state - if ( - clusterhost_state['state'] in 'INSTALLING' and - clusterhost_state['percentage'] > 0 - ): - clusterhost_state['percentage'] = min( - 1.0, ( - 0.5 + clusterhost_state['percentage'] / 2 - ) - ) - return clusterhost_state - - host_state['percentage'] = host_state['percentage'] / 2 - if host_state['state'] == 'SUCCESSFUL': - host_state['state'] = 'INSTALLING' - return host_state - - def to_dict(self): - dict_info = self.host.to_dict() - dict_info.update(super(ClusterHost, self).to_dict()) - state_dict = self.state_dict() - dict_info.update({ - 'distributed_system_installed': self.distributed_system_installed, - 'reinstall_distributed_system': self.reinstall_distributed_system, - 'owner': self.owner, - 'clustername': self.clustername, - 'name': self.name, - 'state': state_dict['state'] - }) - dict_info['roles'] = self.roles - dict_info['patched_roles'] = self.patched_roles - return dict_info - - -class HostState(BASE, StateMixin): - """Host state table.""" - __tablename__ = 'host_state' - - id = Column( - Integer, - ForeignKey('host.id', onupdate='CASCADE', ondelete='CASCADE'), - primary_key=True - ) - - def __str__(self): - return 'HostState[%s state %s percentage %s]' % ( - self.id, self.state, self.percentage - ) - - def update(self): - """Update host state. - - When host state is updated, all clusterhosts on the - host will update their state if necessary. - """ - super(HostState, self).update() - host = self.host - if self.state == 'INSTALLING': - host.reinstall_os = False - for clusterhost in self.host.clusterhosts: - if clusterhost.state in [ - 'SUCCESSFUL', 'ERROR' - ]: - clusterhost.state = 'INSTALLING' - clusterhost.state.update() - elif self.state == 'UNINITIALIZED': - for clusterhost in self.host.clusterhosts: - if clusterhost.state in [ - 'INITIALIZED', 'INSTALLING', 'SUCCESSFUL', 'ERROR' - ]: - clusterhost.state = 'UNINITIALIZED' - clusterhost.state.update() - elif self.state == 'UPDATE_PREPARING': - for clusterhost in self.host.clusterhosts: - if clusterhost.state in [ - 'INITIALIZED', 'INSTALLING', 'SUCCESSFUL', 'ERROR' - ]: - clusterhost.state = 'UPDATE_PREPARING' - clusterhost.state.update() - elif self.state == 'INITIALIZED': - for clusterhost in self.host.clusterhosts: - if clusterhost.state in [ - 'INSTALLING', 'SUCCESSFUL', 'ERROR' - ]: - clusterhost.state = 'INITIALIZED' - clusterhost.state.update() - - -class Host(BASE, TimestampMixin, HelperMixin): - """Host table.""" - __tablename__ = 'host' - - name = Column(String(80), nullable=True) - config_step = Column(String(80), default='') - os_config = Column(JSONEncoded, default={}) - config_validated = Column(Boolean, default=False) - deployed_os_config = Column(JSONEncoded, default={}) - os_name = Column(String(80)) - creator_id = Column(Integer, ForeignKey('user.id')) - owner = Column(String(80)) - os_installer = Column(JSONEncoded, default={}) - - __table_args__ = ( - UniqueConstraint('name', 'owner', name='constraint'), - ) - - id = Column( - Integer, - ForeignKey('machine.id', onupdate='CASCADE', ondelete='CASCADE'), - primary_key=True - ) - reinstall_os = Column(Boolean, default=True) - - host_networks = relationship( - HostNetwork, - passive_deletes=True, passive_updates=True, - cascade='all, delete-orphan', - backref=backref('host') - ) - clusterhosts = relationship( - ClusterHost, - passive_deletes=True, passive_updates=True, - cascade='all, delete-orphan', - backref=backref('host') - ) - state = relationship( - HostState, - uselist=False, - passive_deletes=True, passive_updates=True, - cascade='all, delete-orphan', - backref=backref('host') - ) - log_histories = relationship( - HostLogHistory, - passive_deletes=True, passive_updates=True, - cascade='all, delete-orphan', - backref=backref('host') - ) - - def __str__(self): - return 'Host[%s:%s]' % (self.id, self.name) - - @hybrid_property - def mac(self): - machine = self.machine - if machine: - return machine.mac - else: - return None - - @property - def os_id(self): - return self.os_name - - @os_id.setter - def os_id(self, value): - self.os_name = value - - @hybrid_property - def hostname(self): - return self.name - - @hostname.expression - def hostname(cls): - return cls.name - - @property - def patched_os_config(self): - return self.os_config - - @patched_os_config.setter - def patched_os_config(self, value): - os_config = copy.deepcopy(self.os_config) - self.os_config = util.merge_dict(os_config, value) - logging.debug('patch host os config in %s: %s', self.id, value) - self.config_validated = False - - @property - def put_os_config(self): - return self.os_config - - @put_os_config.setter - def put_os_config(self, value): - os_config = copy.deepcopy(self.os_config) - os_config.update(value) - self.os_config = os_config - logging.debug('put host os config in %s: %s', self.id, value) - self.config_validated = False - - def __init__(self, id, **kwargs): - self.id = id - self.state = HostState() - super(Host, self).__init__(**kwargs) - - def update(self): - creator = self.creator - if creator: - self.owner = creator.email - if self.reinstall_os: - if self.state in ['SUCCESSFUL', 'ERROR']: - if self.config_validated: - self.state.state = 'INITIALIZED' - else: - self.state.state = 'UNINITIALIZED' - self.state.update() - self.state.update() - super(Host, self).update() - - def validate(self): - # TODO(xicheng): some validation can be moved to the column in future. - super(Host, self).validate() - creator = self.creator - if not creator: - raise exception.InvalidParameter( - 'creator is not set in host %s' % self.id - ) - os_name = self.os_name - if not os_name: - raise exception.InvalidParameter( - 'os is not set in host %s' % self.id - ) - os_installer = self.os_installer - if not os_installer: - raise exception.Invalidparameter( - 'os_installer is not set in host %s' % self.id - ) - - @property - def os_installed(self): - return self.state.state == 'SUCCESSFUL' - - @property - def clusters(self): - return [clusterhost.cluster for clusterhost in self.clusterhosts] - - def state_dict(self): - return self.state.to_dict() - - def to_dict(self): - """Host dict contains its underlying machine dict.""" - dict_info = self.machine.to_dict() - dict_info.update(super(Host, self).to_dict()) - state_dict = self.state_dict() - ip = None - for host_network in self.host_networks: - if host_network.is_mgmt: - ip = host_network.ip - dict_info.update({ - 'machine_id': self.machine.id, - 'os_installed': self.os_installed, - 'hostname': self.hostname, - 'ip': ip, - 'networks': [ - host_network.to_dict() - for host_network in self.host_networks - ], - 'os_id': self.os_id, - 'clusters': [cluster.to_dict() for cluster in self.clusters], - 'state': state_dict['state'] - }) - return dict_info - - -class ClusterState(BASE, StateMixin): - """Cluster state table.""" - __tablename__ = 'cluster_state' - - id = Column( - Integer, - ForeignKey('cluster.id', onupdate='CASCADE', ondelete='CASCADE'), - primary_key=True - ) - total_hosts = Column( - Integer, - default=0 - ) - installing_hosts = Column( - Integer, - default=0 - ) - completed_hosts = Column( - Integer, - default=0 - ) - failed_hosts = Column( - Integer, - default=0 - ) - - def __init__(self, **kwargs): - super(ClusterState, self).__init__(**kwargs) - - def __str__(self): - return 'ClusterState[%s state %s percentage %s]' % ( - self.id, self.state, self.percentage - ) - - def to_dict(self): - dict_info = super(ClusterState, self).to_dict() - dict_info['status'] = { - 'total_hosts': self.total_hosts, - 'installing_hosts': self.installing_hosts, - 'completed_hosts': self.completed_hosts, - 'failed_hosts': self.failed_hosts - } - return dict_info - - def update(self): - # all fields of cluster state should be calculated by - # its each underlying clusterhost state. - cluster = self.cluster - clusterhosts = cluster.clusterhosts - self.total_hosts = len(clusterhosts) - self.installing_hosts = 0 - self.failed_hosts = 0 - self.completed_hosts = 0 - if not cluster.flavor_name: - for clusterhost in clusterhosts: - host = clusterhost.host - host_state = host.state.state - if host_state == 'INSTALLING': - self.installing_hosts += 1 - elif host_state == 'ERROR': - self.failed_hosts += 1 - elif host_state == 'SUCCESSFUL': - self.completed_hosts += 1 - else: - for clusterhost in clusterhosts: - clusterhost_state = clusterhost.state.state - if clusterhost_state == 'INSTALLING': - self.installing_hosts += 1 - elif clusterhost_state == 'ERROR': - self.failed_hosts += 1 - elif clusterhost_state == 'SUCCESSFUL': - self.completed_hosts += 1 - if self.total_hosts: - if self.completed_hosts == self.total_hosts: - self.percentage = 1.0 - else: - self.percentage = ( - float(self.completed_hosts) - / - float(self.total_hosts) - ) - if self.state == 'SUCCESSFUL': - self.state = 'INSTALLING' - self.ready = False - self.message = ( - 'total %s, installing %s, completed: %s, error %s' - ) % ( - self.total_hosts, self.installing_hosts, - self.completed_hosts, self.failed_hosts - ) - if self.failed_hosts: - self.severity = 'ERROR' - - super(ClusterState, self).update() - if self.state == 'INSTALLING': - cluster.reinstall_distributed_system = False - - -class Cluster(BASE, TimestampMixin, HelperMixin): - """Cluster table.""" - __tablename__ = 'cluster' - - id = Column(Integer, primary_key=True) - name = Column(String(80), nullable=False) - reinstall_distributed_system = Column(Boolean, default=True) - config_step = Column(String(80), default='') - os_name = Column(String(80)) - flavor_name = Column(String(80), nullable=True) - # flavor dict got from flavor id. - flavor = Column(JSONEncoded, default={}) - os_config = Column(JSONEncoded, default={}) - package_config = Column(JSONEncoded, default={}) - deployed_os_config = Column(JSONEncoded, default={}) - deployed_package_config = Column(JSONEncoded, default={}) - config_validated = Column(Boolean, default=False) - adapter_name = Column(String(80)) - creator_id = Column(Integer, ForeignKey('user.id')) - owner = Column(String(80)) - clusterhosts = relationship( - ClusterHost, - passive_deletes=True, passive_updates=True, - cascade='all, delete-orphan', - backref=backref('cluster') - ) - state = relationship( - ClusterState, - uselist=False, - passive_deletes=True, passive_updates=True, - cascade='all, delete-orphan', - backref=backref('cluster') - ) - __table_args__ = ( - UniqueConstraint('name', 'creator_id', name='constraint'), - ) - - def __init__(self, name, creator_id, **kwargs): - self.name = name - self.creator_id = creator_id - self.state = ClusterState() - super(Cluster, self).__init__(**kwargs) - - def __str__(self): - return 'Cluster[%s:%s]' % (self.id, self.name) - - def update(self): - creator = self.creator - if creator: - self.owner = creator.email - if self.reinstall_distributed_system: - if self.state in ['SUCCESSFUL', 'ERROR']: - if self.config_validated: - self.state.state = 'INITIALIZED' - else: - self.state.state = 'UNINITIALIZED' - self.state.update() - self.state.update() - super(Cluster, self).update() - - def validate(self): - # TODO(xicheng): some validation can be moved to column. - super(Cluster, self).validate() - creator = self.creator - if not creator: - raise exception.InvalidParameter( - 'creator is not set in cluster %s' % self.id - ) - os_name = self.os_name - if not os_name: - raise exception.InvalidParameter( - 'os is not set in cluster %s' % self.id - ) - adapter_name = self.adapter_name - if not adapter_name: - raise exception.InvalidParameter( - 'adapter is not set in cluster %s' % self.id - ) - flavor_name = self.flavor_name - if flavor_name: - if 'name' not in self.flavor: - raise exception.InvalidParameter( - 'key name does not exist in flavor %s' % ( - self.flavor - ) - ) - if flavor_name != self.flavor['name']: - raise exception.InvalidParameter( - 'flavor name %s is not match ' - 'the name key in flavor %s' % ( - flavor_name, self.flavor - ) - ) - else: - if self.flavor: - raise exception.InvalidParameter( - 'flavor %s is not empty' % self.flavor - ) - - @property - def os_id(self): - return self.os_name - - @os_id.setter - def os_id(self, value): - self.os_name = value - - @property - def adapter_id(self): - return self.adapter_name - - @adapter_id.setter - def adapter_id(self, value): - self.adapter_name = value - - @property - def flavor_id(self): - if self.flavor_name: - return '%s:%s' % (self.adapter_name, self.flavor_name) - else: - return None - - @flavor_id.setter - def flavor_id(self, value): - if value: - _, flavor_name = value.split(':', 1) - self.flavor_name = flavor_name - else: - self.flavor_name = value - - @property - def patched_os_config(self): - return self.os_config - - @patched_os_config.setter - def patched_os_config(self, value): - os_config = copy.deepcopy(self.os_config) - self.os_config = util.merge_dict(os_config, value) - logging.debug('patch cluster %s os config: %s', self.id, value) - self.config_validated = False - - @property - def put_os_config(self): - return self.os_config - - @put_os_config.setter - def put_os_config(self, value): - os_config = copy.deepcopy(self.os_config) - os_config.update(value) - self.os_config = os_config - logging.debug('put cluster %s os config: %s', self.id, value) - self.config_validated = False - - @property - def patched_package_config(self): - return self.package_config - - @patched_package_config.setter - def patched_package_config(self, value): - package_config = copy.deepcopy(self.package_config) - self.package_config = util.merge_dict(package_config, value) - logging.debug('patch cluster %s package config: %s', self.id, value) - self.config_validated = False - - @property - def put_package_config(self): - return self.package_config - - @put_package_config.setter - def put_package_config(self, value): - package_config = dict(self.package_config) - package_config.update(value) - self.package_config = package_config - logging.debug('put cluster %s package config: %s', self.id, value) - self.config_validated = False - - @property - def distributed_system_installed(self): - return self.state.state == 'SUCCESSFUL' - - def state_dict(self): - return self.state.to_dict() - - def to_dict(self): - dict_info = super(Cluster, self).to_dict() - dict_info['distributed_system_installed'] = ( - self.distributed_system_installed - ) - dict_info['os_id'] = self.os_id - dict_info['adapter_id'] = self.adapter_id - dict_info['flavor_id'] = self.flavor_id - return dict_info - - -# User, Permission relation table -class UserPermission(BASE, HelperMixin, TimestampMixin): - """User permission table.""" - __tablename__ = 'user_permission' - id = Column(Integer, primary_key=True) - user_id = Column( - Integer, - ForeignKey('user.id', onupdate='CASCADE', ondelete='CASCADE') - ) - permission_id = Column( - Integer, - ForeignKey('permission.id', onupdate='CASCADE', ondelete='CASCADE') - ) - __table_args__ = ( - UniqueConstraint('user_id', 'permission_id', name='constraint'), - ) - - def __init__(self, user_id, permission_id, **kwargs): - self.user_id = user_id - self.permission_id = permission_id - - def __str__(self): - return 'UserPermission[%s:%s]' % (self.id, self.name) - - @hybrid_property - def name(self): - return self.permission.name - - def to_dict(self): - dict_info = self.permission.to_dict() - dict_info.update(super(UserPermission, self).to_dict()) - return dict_info - - -class Permission(BASE, HelperMixin, TimestampMixin): - """Permission table.""" - __tablename__ = 'permission' - - id = Column(Integer, primary_key=True) - name = Column(String(80), unique=True, nullable=False) - alias = Column(String(100)) - description = Column(Text) - user_permissions = relationship( - UserPermission, - passive_deletes=True, passive_updates=True, - cascade='all, delete-orphan', - backref=backref('permission') - ) - - def __init__(self, name, **kwargs): - self.name = name - super(Permission, self).__init__(**kwargs) - - def __str__(self): - return 'Permission[%s:%s]' % (self.id, self.name) - - -class UserToken(BASE, HelperMixin): - """user token table.""" - __tablename__ = 'user_token' - - id = Column(Integer, primary_key=True) - user_id = Column( - Integer, - ForeignKey('user.id', onupdate='CASCADE', ondelete='CASCADE') - ) - token = Column(String(256), unique=True, nullable=False) - expire_timestamp = Column(DateTime, nullable=True) - - def __init__(self, token, **kwargs): - self.token = token - super(UserToken, self).__init__(**kwargs) - - def validate(self): - # TODO(xicheng): some validation can be moved to column. - super(UserToken, self).validate() - if not self.user: - raise exception.InvalidParameter( - 'user is not set in token: %s' % self.token - ) - - -class UserLog(BASE, HelperMixin): - """User log table.""" - __tablename__ = 'user_log' - - id = Column(Integer, primary_key=True) - user_id = Column( - Integer, - ForeignKey('user.id', onupdate='CASCADE', ondelete='CASCADE') - ) - action = Column(Text) - timestamp = Column(DateTime, default=lambda: datetime.datetime.now()) - - @hybrid_property - def user_email(self): - return self.user.email - - def validate(self): - # TODO(xicheng): some validation can be moved to column. - super(UserLog, self).validate() - if not self.user: - raise exception.InvalidParameter( - 'user is not set in user log: %s' % self.id - ) - - -class User(BASE, HelperMixin, TimestampMixin): - """User table.""" - __tablename__ = 'user' - - id = Column(Integer, primary_key=True) - email = Column(String(80), unique=True, nullable=False) - crypted_password = Column('password', String(225)) - firstname = Column(String(80)) - lastname = Column(String(80)) - is_admin = Column(Boolean, default=False) - active = Column(Boolean, default=True) - user_permissions = relationship( - UserPermission, - passive_deletes=True, passive_updates=True, - cascade='all, delete-orphan', - backref=backref('user') - ) - user_logs = relationship( - UserLog, - passive_deletes=True, passive_updates=True, - cascade='all, delete-orphan', - backref=backref('user') - ) - user_tokens = relationship( - UserToken, - passive_deletes=True, passive_updates=True, - cascade='all, delete-orphan', - backref=backref('user') - ) - clusters = relationship( - Cluster, - backref=backref('creator') - ) - hosts = relationship( - Host, - backref=backref('creator') - ) - - def __init__(self, email, **kwargs): - self.email = email - super(User, self).__init__(**kwargs) - - def __str__(self): - return 'User[%s]' % self.email - - def validate(self): - # TODO(xicheng): some validation can be moved to column. - super(User, self).validate() - if not self.crypted_password: - raise exception.InvalidParameter( - 'password is not set in user : %s' % self.email - ) - - @property - def password(self): - return '***********' - - @password.setter - def password(self, password): - # password stored in database is crypted. - self.crypted_password = util.encrypt(password) - - @hybrid_property - def permissions(self): - permissions = [] - for user_permission in self.user_permissions: - permissions.append(user_permission.permission) - - return permissions - - def to_dict(self): - dict_info = super(User, self).to_dict() - dict_info['permissions'] = [ - permission.to_dict() - for permission in self.permissions - ] - return dict_info - - -class SwitchMachine(BASE, HelperMixin, TimestampMixin): - """Switch Machine table.""" - __tablename__ = 'switch_machine' - switch_machine_id = Column( - 'id', Integer, primary_key=True - ) - switch_id = Column( - Integer, - ForeignKey('switch.id', onupdate='CASCADE', ondelete='CASCADE') - ) - machine_id = Column( - Integer, - ForeignKey('machine.id', onupdate='CASCADE', ondelete='CASCADE') - ) - owner_id = Column(Integer, ForeignKey('user.id')) - port = Column(String(80), nullable=True) - vlans = Column(JSONEncoded, default=[]) - __table_args__ = ( - UniqueConstraint('switch_id', 'machine_id', name='constraint'), - ) - - def __init__(self, switch_id, machine_id, **kwargs): - self.switch_id = switch_id - self.machine_id = machine_id - super(SwitchMachine, self).__init__(**kwargs) - - def __str__(self): - return 'SwitchMachine[%s port %s]' % ( - self.switch_machine_id, self.port - ) - - def validate(self): - # TODO(xicheng): some validation can be moved to column. - super(SwitchMachine, self).validate() - if not self.switch: - raise exception.InvalidParameter( - 'switch is not set in %s' % self.id - ) - if not self.machine: - raise exception.Invalidparameter( - 'machine is not set in %s' % self.id - ) - if not self.port: - raise exception.InvalidParameter( - 'port is not set in %s' % self.id - ) - - @hybrid_property - def mac(self): - return self.machine.mac - - @hybrid_property - def tag(self): - return self.machine.tag - - @property - def switch_ip(self): - return self.switch.ip - - @hybrid_property - def switch_ip_int(self): - return self.switch.ip_int - - @switch_ip_int.expression - def switch_ip_int(cls): - return Switch.ip_int - - @hybrid_property - def switch_vendor(self): - return self.switch.vendor - - @switch_vendor.expression - def switch_vendor(cls): - return Switch.vendor - - @property - def patched_vlans(self): - return self.vlans - - @patched_vlans.setter - def patched_vlans(self, value): - if not value: - return - vlans = list(self.vlans) - for item in value: - if item not in vlans: - vlans.append(item) - self.vlans = vlans - - @property - def filtered(self): - """Check if switch machine should be filtered. - - port should be composed with - For each filter in switch machine filters, - if filter_type is allow and port match the pattern, the switch - machine is allowed to be got by api. If filter_type is deny and - port match the pattern, the switch machine is not allowed to be got - by api. - If not filter is matched, if the last filter is allow, deny all - unmatched switch machines, if the last filter is deny, allow all - unmatched switch machines. - If no filter defined, allow all switch machines. - if ports defined in filter and 'all' in ports, the switch machine is - matched. if ports defined in filter and 'all' not in ports, - the switch machine with the port name in ports will be matched. - If the port pattern matches - < and port number is in the - range of [port_start, port_end], the switch machine is matched. - """ - filters = self.switch.machine_filters - port = self.port - unmatched_allowed = True - ports_pattern = re.compile(r'(\D*)(\d+)-(\d+)(\D*)') - port_pattern = re.compile(r'(\D*)(\d+)(\D*)') - port_match = port_pattern.match(port) - if port_match: - port_prefix = port_match.group(1) - port_number = int(port_match.group(2)) - port_suffix = port_match.group(3) - else: - port_prefix = '' - port_number = 0 - port_suffix = '' - for port_filter in filters: - filter_type = port_filter.get('filter_type', 'allow') - denied = filter_type != 'allow' - unmatched_allowed = denied - if 'ports' in port_filter: - if 'all' in port_filter['ports']: - return denied - if port in port_filter['ports']: - return denied - if port_match: - for port_or_ports in port_filter['ports']: - ports_match = ports_pattern.match(port_or_ports) - if ports_match: - filter_port_prefix = ports_match.group(1) - filter_port_start = int(ports_match.group(2)) - filter_port_end = int(ports_match.group(3)) - filter_port_suffix = ports_match.group(4) - if ( - filter_port_prefix == port_prefix and - filter_port_suffix == port_suffix and - filter_port_start <= port_number and - port_number <= filter_port_end - ): - return denied - else: - filter_port_prefix = port_filter.get('port_prefix', '') - filter_port_suffix = port_filter.get('port_suffix', '') - if ( - port_match and - port_prefix == filter_port_prefix and - port_suffix == filter_port_suffix - ): - if ( - 'port_start' not in port_filter or - port_number >= port_filter['port_start'] - ) and ( - 'port_end' not in port_filter or - port_number <= port_filter['port_end'] - ): - return denied - return not unmatched_allowed - - def to_dict(self): - dict_info = self.machine.to_dict() - dict_info.update(super(SwitchMachine, self).to_dict()) - dict_info['switch_ip'] = self.switch.ip - return dict_info - - -class Machine(BASE, HelperMixin, TimestampMixin): - """Machine table.""" - __tablename__ = 'machine' - id = Column(Integer, primary_key=True) - mac = Column(String(24), unique=True, nullable=False) - ipmi_credentials = Column(JSONEncoded, default={}) - tag = Column(JSONEncoded, default={}) - location = Column(JSONEncoded, default={}) - owner_id = Column(Integer, ForeignKey('user.id')) - machine_attributes = Column(JSONEncoded, default={}) - - switch_machines = relationship( - SwitchMachine, - passive_deletes=True, passive_updates=True, - cascade='all, delete-orphan', - backref=backref('machine') - ) - host = relationship( - Host, - uselist=False, - passive_deletes=True, passive_updates=True, - cascade='all, delete-orphan', - backref=backref('machine') - ) - - def __init__(self, mac, **kwargs): - self.mac = mac - super(Machine, self).__init__(**kwargs) - - def __str__(self): - return 'Machine[%s:%s]' % (self.id, self.mac) - - def validate(self): - # TODO(xicheng): some validation can be moved to column. - super(Machine, self).validate() - try: - netaddr.EUI(self.mac) - except Exception: - raise exception.InvalidParameter( - 'mac address %s format uncorrect' % self.mac - ) - - @property - def patched_ipmi_credentials(self): - return self.ipmi_credentials - - @patched_ipmi_credentials.setter - def patched_ipmi_credentials(self, value): - if not value: - return - ipmi_credentials = copy.deepcopy(self.ipmi_credentials) - self.ipmi_credentials = util.merge_dict(ipmi_credentials, value) - - @property - def patched_tag(self): - return self.tag - - @patched_tag.setter - def patched_tag(self, value): - if not value: - return - tag = copy.deepcopy(self.tag) - tag.update(value) - self.tag = value - - @property - def patched_location(self): - return self.location - - @patched_location.setter - def patched_location(self, value): - if not value: - return - location = copy.deepcopy(self.location) - location.update(value) - self.location = location - - def to_dict(self): - # TODO(xicheng): move the filling of switches - # to db/api. - dict_info = {} - dict_info['switches'] = [ - { - 'switch_ip': switch_machine.switch_ip, - 'port': switch_machine.port, - 'vlans': switch_machine.vlans - } - for switch_machine in self.switch_machines - if not switch_machine.filtered - ] - if dict_info['switches']: - dict_info.update(dict_info['switches'][0]) - dict_info.update(super(Machine, self).to_dict()) - return dict_info - - -class Switch(BASE, HelperMixin, TimestampMixin): - """Switch table.""" - __tablename__ = 'switch' - id = Column(Integer, primary_key=True) - ip_int = Column('ip', BigInteger, unique=True, nullable=False) - credentials = Column(JSONEncoded, default={}) - vendor = Column(String(256), nullable=True) - state = Column(Enum('initialized', 'unreachable', 'notsupported', - 'repolling', 'error', 'under_monitoring', - name='switch_state'), - ColumnDefault('initialized')) - # filters is json formatted list, each element has following format: - # keys: ['filter_type', 'ports', 'port_prefix', 'port_suffix', - # 'port_start', 'port_end']. - # each port name is divided into - # filter_type is one of ['allow', 'deny'], default is 'allow' - # ports is a list of port name. - # port_prefix is the prefix that filtered port should start with. - # port_suffix is the suffix that filtered posrt should end with. - # port_start is integer that the port number should start with. - # port_end is the integer that the port number should end with. - _filters = Column('filters', JSONEncoded, default=[]) - switch_machines = relationship( - SwitchMachine, - passive_deletes=True, passive_updates=True, - cascade='all, delete-orphan', - backref=backref('switch') - ) - - def __str__(self): - return 'Switch[%s:%s]' % (self.id, self.ip) - - @classmethod - def parse_filters(cls, filters): - """parse filters set from outside to standard format. - - api can set switch filters with the flexible format, this - function will parse the flexible format filters. - - Supported format: - as string: - allow ports ae10,ae20 - allow port_prefix ae port_start 30 port_end 40 - deny ports all - as python object: - [{ - 'filter_type': 'allow', - 'ports': ['ae10', 'ae20'] - },{ - 'filter_type': 'allow', - 'port_prefix': 'ae', - 'port_suffix': '', - 'port_start': 30, - 'port_end': 40 - },{ - 'filter_type': 'deny', - 'ports': ['all'] - }] - """ - if isinstance(filters, basestring): - filters = filters.replace('\r\n', '\n').replace('\n', ';') - filters = [ - machine_filter for machine_filter in filters.split(';') - if machine_filter - ] - if not isinstance(filters, list): - filters = [filters] - machine_filters = [] - for machine_filter in filters: - if not machine_filter: - continue - if isinstance(machine_filter, basestring): - filter_dict = {} - filter_items = [ - item for item in machine_filter.split() if item - ] - if filter_items[0] in ['allow', 'deny']: - filter_dict['filter_type'] = filter_items[0] - filter_items = filter_items[1:] - elif filter_items[0] not in [ - 'ports', 'port_prefix', 'port_suffix', - 'port_start', 'port_end' - ]: - raise exception.InvalidParameter( - 'unrecognized filter type %s' % filter_items[0] - ) - while filter_items: - if len(filter_items) >= 2: - filter_dict[filter_items[0]] = filter_items[1] - filter_items = filter_items[2:] - else: - filter_dict[filter_items[0]] = '' - filter_items = filter_items[1:] - machine_filter = filter_dict - if not isinstance(machine_filter, dict): - raise exception.InvalidParameter( - 'filter %s is not dict' % machine_filter - ) - if 'filter_type' in machine_filter: - if machine_filter['filter_type'] not in ['allow', 'deny']: - raise exception.InvalidParameter( - 'filter_type should be `allow` or `deny` in %s' % ( - machine_filter - ) - ) - if 'ports' in machine_filter: - if isinstance(machine_filter['ports'], basestring): - machine_filter['ports'] = [ - port_or_ports - for port_or_ports in machine_filter['ports'].split(',') - if port_or_ports - ] - if not isinstance(machine_filter['ports'], list): - raise exception.InvalidParameter( - '`ports` type is not list in filter %s' % ( - machine_filter - ) - ) - for port_or_ports in machine_filter['ports']: - if not isinstance(port_or_ports, basestring): - raise exception.InvalidParameter( - '%s type is not basestring in `ports` %s' % ( - port_or_ports, machine_filter['ports'] - ) - ) - for key in ['port_start', 'port_end']: - if key in machine_filter: - if isinstance(machine_filter[key], basestring): - if machine_filter[key].isdigit(): - machine_filter[key] = int(machine_filter[key]) - if not isinstance(machine_filter[key], (int, long)): - raise exception.InvalidParameter( - '`%s` type is not int in filer %s' % ( - key, machine_filter - ) - ) - machine_filters.append(machine_filter) - return machine_filters - - @classmethod - def format_filters(cls, filters): - """format json formatted filters to string.""" - filter_strs = [] - for machine_filter in filters: - filter_properties = [] - filter_properties.append( - machine_filter.get('filter_type', 'allow') - ) - if 'ports' in machine_filter: - filter_properties.append( - 'ports ' + ','.join(machine_filter['ports']) - ) - if 'port_prefix' in machine_filter: - filter_properties.append( - 'port_prefix ' + machine_filter['port_prefix'] - ) - if 'port_suffix' in machine_filter: - filter_properties.append( - 'port_suffix ' + machine_filter['port_suffix'] - ) - if 'port_start' in machine_filter: - filter_properties.append( - 'port_start ' + str(machine_filter['port_start']) - ) - if 'port_end' in machine_filter: - filter_properties.append( - 'port_end ' + str(machine_filter['port_end']) - ) - filter_strs.append(' '.join(filter_properties)) - return ';'.join(filter_strs) - - def __init__(self, ip_int, **kwargs): - self.ip_int = ip_int - super(Switch, self).__init__(**kwargs) - - @property - def ip(self): - return str(netaddr.IPAddress(self.ip_int)) - - @ip.setter - def ip(self, ipaddr): - self.ip_int = int(netaddr.IPAddress(ipaddr)) - - @property - def patched_credentials(self): - return self.credentials - - @patched_credentials.setter - def patched_credentials(self, value): - if not value: - return - credentials = copy.deepcopy(self.credentials) - self.credentials = util.merge_dict(credentials, value) - - @property - def machine_filters(self): - return self._filters - - @machine_filters.setter - def machine_filters(self, value): - if not value: - return - self._filters = self.parse_filters(value) - - @property - def put_machine_filters(self): - return self._filters - - @put_machine_filters.setter - def put_machine_filters(self, value): - if not value: - return - self._filters = self.parse_filters(value) - - @property - def patched_machine_filters(self): - return self._filters - - @patched_machine_filters.setter - def patched_machine_filters(self, value): - if not value: - return - filters = list(self.machine_filters) - self._filters = self.parse_filters(value) + filters - - def to_dict(self): - dict_info = super(Switch, self).to_dict() - dict_info['ip'] = self.ip - dict_info['filters'] = self.format_filters(self._filters) - return dict_info - - -class Subnet(BASE, TimestampMixin, HelperMixin): - """network table.""" - __tablename__ = 'subnet' - - id = Column(Integer, primary_key=True) - name = Column(String(80), unique=True, nullable=True) - subnet = Column(String(80), unique=True, nullable=False) - - host_networks = relationship( - HostNetwork, - passive_deletes=True, passive_updates=True, - cascade='all, delete-orphan', - backref=backref('subnet') - ) - - def __init__(self, subnet, **kwargs): - self.subnet = subnet - super(Subnet, self).__init__(**kwargs) - - def __str__(self): - return 'Subnet[%s:%s]' % (self.id, self.subnet) - - def to_dict(self): - dict_info = super(Subnet, self).to_dict() - if not self.name: - dict_info['name'] = self.subnet - return dict_info - - -# TODO(grace): move this global variable into HealthCheckReport. -HEALTH_REPORT_STATES = ('verifying', 'success', 'finished', 'error') - - -class HealthCheckReport(BASE, HelperMixin): - """Health check report table.""" - __tablename__ = 'health_check_report' - - cluster_id = Column( - Integer, - ForeignKey('cluster.id', onupdate='CASCADE', ondelete='CASCADE'), - primary_key=True - ) - name = Column(String(80), nullable=False, primary_key=True) - display_name = Column(String(100)) - report = Column(JSONEncoded, default={}) - category = Column(String(80), default='') - state = Column( - Enum(*HEALTH_REPORT_STATES, name='report_state'), - ColumnDefault('verifying'), - nullable=False - ) - error_message = Column(Text, default='') - - def __init__(self, cluster_id, name, **kwargs): - self.cluster_id = cluster_id - self.name = name - if 'state' in kwargs and kwargs['state'] not in HEALTH_REPORT_STATES: - err_msg = 'State value %s is not accepted.' % kwargs['state'] - raise exception.InvalidParameter(err_msg) - - super(HealthCheckReport, self).__init__(**kwargs) - - def __str__(self): - return 'HealthCheckReport[cluster_id: %s, name: %s]' % ( - self.cluster_id, self.name - ) diff --git a/compass-tasks/db/v1/model.py b/compass-tasks/db/v1/model.py deleted file mode 100644 index d74e355..0000000 --- a/compass-tasks/db/v1/model.py +++ /dev/null @@ -1,724 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""database model.""" -from datetime import datetime -from hashlib import md5 -import logging -import simplejson as json -import uuid - -from sqlalchemy import Column, ColumnDefault, Integer, String -from sqlalchemy import Float, Enum, DateTime, ForeignKey, Text, Boolean -from sqlalchemy import UniqueConstraint -from sqlalchemy.orm import relationship, backref -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.ext.hybrid import hybrid_property - -from compass.utils import util - -from flask.ext.login import UserMixin -from itsdangerous import URLSafeTimedSerializer - -BASE = declarative_base() -# TODO(grace) SECRET_KEY should be generated when installing compass -# and save to a config file or DB -SECRET_KEY = "abcd" - -# This is used for generating a token by user's ID and -# decode the ID from this token -login_serializer = URLSafeTimedSerializer(SECRET_KEY) - - -class User(BASE, UserMixin): - """User table.""" - __tablename__ = 'user' - id = Column(Integer, primary_key=True) - email = Column(String(80), unique=True) - password = Column(String(225), default='') - active = Column(Boolean, default=True) - - def __init__(self, email, password, **kwargs): - self.email = email - self.password = self._set_password(password) - - def __repr__(self): - return '' % self.email - - def _set_password(self, password): - return self._hash_password(password) - - def get_password(self): - return self.password - - def valid_password(self, password): - return self.password == self._hash_password(password) - - def get_auth_token(self): - return login_serializer.dumps(self.id) - - def is_active(self): - return self.active - - def _hash_password(self, password): - return md5(password).hexdigest() - - -class SwitchConfig(BASE): - """Swtich Config table. - - :param id: The unique identifier of the switch config. - :param ip: The IP address of the switch. - :param filter_port: The port of the switch which need to be filtered. - """ - __tablename__ = 'switch_config' - id = Column(Integer, primary_key=True) - ip = Column(String(80)) - filter_port = Column(String(16)) - __table_args__ = (UniqueConstraint('ip', 'filter_port', name='filter1'), ) - - def __init__(self, **kwargs): - super(SwitchConfig, self).__init__(**kwargs) - - -class Switch(BASE): - """Switch table. - - :param id: the unique identifier of the switch. int as primary key. - :param ip: the IP address of the switch. - :param vendor_info: the name of the vendor - :param credential_data: used for accessing and retrieving information - from the switch. Store json format as string. - :param state: Enum.'initialized/repolling': polling switch not complete to - learn all MAC addresses of devices connected to the switch; - 'unreachable': one of the final state, indicates that the - switch is unreachable at this time, no MAC address could be - retrieved from the switch. - 'notsupported': one of the final state, indicates that the - vendor found is not supported yet, no MAC address will be - retrieved from the switch. - 'error': one of the final state, indicates that something - wrong happend. - 'under_monitoring': one of the final state, indicates that - MAC addresses has been learned successfully from the switch. - :param err_msg: Error message when polling switch failed. - :param machines: refer to list of Machine connected to the switch. - """ - __tablename__ = 'switch' - - id = Column(Integer, primary_key=True) - ip = Column(String(80), unique=True) - credential_data = Column(Text) - vendor_info = Column(String(256), nullable=True) - state = Column(Enum('initialized', 'unreachable', 'notsupported', - 'repolling', 'error', 'under_monitoring', - name='switch_state'), - default='initialized') - err_msg = Column(Text) - - def __init__(self, **kwargs): - super(Switch, self).__init__(**kwargs) - - def __repr__(self): - return ''\ - % (self.ip, self.credential, self.vendor, self.state) - - @hybrid_property - def vendor(self): - """vendor property getter""" - return self.vendor_info - - @vendor.setter - def vendor(self, value): - """vendor property setter""" - self.vendor_info = value - - @property - def credential(self): - """credential data getter. - - :returns: python primitive dictionary object. - """ - if self.credential_data: - try: - credential = json.loads(self.credential_data) - return credential - except Exception as error: - logging.error('failed to load credential data %s: %s', - self.id, self.credential_data) - logging.exception(error) - raise error - else: - return {} - - @credential.setter - def credential(self, value): - """credential property setter - - :param value: dict of configuration data needed to update. - """ - if value: - try: - credential = {} - if self.credential_data: - credential = json.loads(self.credential_data) - - credential.update(value) - self.credential_data = json.dumps(credential) - - except Exception as error: - logging.error('failed to dump credential data %s: %s', - self.id, value) - logging.exception(error) - raise error - - else: - self.credential_data = json.dumps({}) - - logging.debug('switch now is %s', self) - - -class Machine(BASE): - """Machine table. - - .. note:: - currently, we are taking care of management plane. - Therefore, we assume one machine is connected to one switch. - - :param id: int, identity as primary key - :param mac: string, the MAC address of the machine. - :param switch_id: switch id that this machine connected on to. - :param port: nth port of the switch that this machine connected. - :param vlan: vlan id that this machine connected on to. - :param update_timestamp: last time this entry got updated. - :param switch: refer to the Switch the machine connects to. - """ - __tablename__ = 'machine' - - id = Column(Integer, primary_key=True) - mac = Column(String(24), default='') - port = Column(String(16), default='') - vlan = Column(Integer, default=0) - update_timestamp = Column(DateTime, default=datetime.now, - onupdate=datetime.now) - switch_id = Column(Integer, ForeignKey('switch.id', - onupdate='CASCADE', - ondelete='SET NULL')) - __table_args__ = (UniqueConstraint('mac', 'switch_id', - name='unique_machine'),) - switch = relationship('Switch', backref=backref('machines', - lazy='dynamic')) - - def __init__(self, **kwargs): - super(Machine, self).__init__(**kwargs) - - def __repr__(self): - return '' % ( - self.mac, self.port, self.vlan, self.switch) - - -class HostState(BASE): - """The state of the ClusterHost. - - :param id: int, identity as primary key. - :param state: Enum. 'UNINITIALIZED': the host is ready to setup. - 'INSTALLING': the host is not installing. - 'READY': the host is setup. - 'ERROR': the host has error. - :param progress: float, the installing progress from 0 to 1. - :param message: the latest installing message. - :param severity: Enum, the installing message severity. - ('INFO', 'WARNING', 'ERROR') - :param update_timestamp: the lastest timestamp the entry got updated. - :param host: refer to ClusterHost. - :param os_progress: float, the installing progress of OS from 0 to 1. - """ - __tablename__ = "host_state" - - id = Column(Integer, ForeignKey('cluster_host.id', - onupdate='CASCADE', - ondelete='CASCADE'), - primary_key=True) - state = Column(Enum('UNINITIALIZED', 'INSTALLING', 'READY', 'ERROR'), - ColumnDefault('UNINITIALIZED')) - progress = Column(Float, ColumnDefault(0.0)) - message = Column(Text) - severity = Column(Enum('INFO', 'WARNING', 'ERROR'), ColumnDefault('INFO')) - update_timestamp = Column(DateTime, default=datetime.now, - onupdate=datetime.now) - host = relationship('ClusterHost', backref=backref('state', - uselist=False)) - - os_progress = Column(Float, ColumnDefault(0.0)) - os_message = Column(Text) - os_severity = Column( - Enum('INFO', 'WARNING', 'ERROR'), - ColumnDefault('INFO') - ) - """ - this is added by Lei for separating os and package progress purposes - os_state = Column(Enum('UNINITIALIZED', 'INSTALLING', 'OS_READY', 'ERROR'), - ColumnDefault('UNINITIALIZED')) - """ - - def __init__(self, **kwargs): - super(HostState, self).__init__(**kwargs) - - @hybrid_property - def hostname(self): - """hostname getter""" - return self.host.hostname - - @hybrid_property - def fullname(self): - """fullname getter""" - return self.host.fullname - - def __repr__(self): - return ( - '' - ) % ( - self.hostname, self.state, self.progress, - self.message, self.severity, self.os_progress - ) - - -class ClusterState(BASE): - """The state of the Cluster. - - :param id: int, identity as primary key. - :param state: Enum, 'UNINITIALIZED': the cluster is ready to setup. - 'INSTALLING': the cluster is not installing. - 'READY': the cluster is setup. - 'ERROR': the cluster has error. - :param progress: float, the installing progress from 0 to 1. - :param message: the latest installing message. - :param severity: Enum, the installing message severity. - ('INFO', 'WARNING', 'ERROR'). - :param update_timestamp: the lastest timestamp the entry got updated. - :param cluster: refer to Cluster. - """ - __tablename__ = 'cluster_state' - id = Column(Integer, ForeignKey('cluster.id', - onupdate='CASCADE', - ondelete='CASCADE'), - primary_key=True) - state = Column(Enum('UNINITIALIZED', 'INSTALLING', 'READY', 'ERROR'), - ColumnDefault('UNINITIALIZED')) - progress = Column(Float, ColumnDefault(0.0)) - message = Column(Text) - severity = Column(Enum('INFO', 'WARNING', 'ERROR'), ColumnDefault('INFO')) - update_timestamp = Column(DateTime, default=datetime.now, - onupdate=datetime.now) - cluster = relationship('Cluster', backref=backref('state', - uselist=False)) - - def __init__(self, **kwargs): - super(ClusterState, self).__init__(**kwargs) - - @hybrid_property - def clustername(self): - """clustername getter""" - return self.cluster.name - - def __repr__(self): - return ( - '' - ) % ( - self.clustername, self.state, self.progress, - self.message, self.severity - ) - - -class Cluster(BASE): - """Cluster configuration information. - - :param id: int, identity as primary key. - :param name: str, cluster name. - :param mutable: bool, if the Cluster is mutable. - :param security_config: str stores json formatted security information. - :param networking_config: str stores json formatted networking information. - :param partition_config: string stores json formatted parition information. - :param adapter_id: the refer id in the Adapter table. - :param raw_config: str stores json formatted other cluster information. - :param adapter: refer to the Adapter. - :param state: refer to the ClusterState. - """ - __tablename__ = 'cluster' - - id = Column(Integer, primary_key=True) - name = Column(String(80), unique=True) - mutable = Column(Boolean, default=True) - security_config = Column(Text) - networking_config = Column(Text) - partition_config = Column(Text) - adapter_id = Column(Integer, ForeignKey('adapter.id', - onupdate='CASCADE', - ondelete='SET NULL'), - nullable=True) - raw_config = Column(Text) - adapter = relationship("Adapter", backref=backref('clusters', - lazy='dynamic')) - - def __init__(self, **kwargs): - if 'name' not in kwargs or not kwargs['name']: - kwargs['name'] = str(uuid.uuid4()) - - super(Cluster, self).__init__(**kwargs) - - def __repr__(self): - return '' % (self.name, self.config) - - @property - def partition(self): - """partition getter""" - if self.partition_config: - try: - return json.loads(self.partition_config) - except Exception as error: - logging.error('failed to load security config %s: %s', - self.id, self.partition_config) - logging.exception(error) - raise error - else: - return {} - - @partition.setter - def partition(self, value): - """partition setter""" - logging.debug('cluster %s set partition %s', self.id, value) - if value: - try: - self.partition_config = json.dumps(value) - except Exception as error: - logging.error('failed to dump partition config %s: %s', - self.id, value) - logging.exception(error) - raise error - else: - self.partition_config = None - - @property - def security(self): - """security getter""" - if self.security_config: - try: - return json.loads(self.security_config) - except Exception as error: - logging.error('failed to load security config %s: %s', - self.id, self.security_config) - logging.exception(error) - raise error - else: - return {} - - @security.setter - def security(self, value): - """security setter""" - logging.debug('cluster %s set security %s', self.id, value) - if value: - try: - self.security_config = json.dumps(value) - except Exception as error: - logging.error('failed to dump security config %s: %s', - self.id, value) - logging.exception(error) - raise error - else: - self.security_config = None - - @property - def networking(self): - """networking getter""" - if self.networking_config: - try: - return json.loads(self.networking_config) - except Exception as error: - logging.error('failed to load networking config %s: %s', - self.id, self.networking_config) - logging.exception(error) - raise error - else: - return {} - - @networking.setter - def networking(self, value): - """networking setter.""" - logging.debug('cluster %s set networking %s', self.id, value) - if value: - try: - self.networking_config = json.dumps(value) - except Exception as error: - logging.error('failed to dump networking config %s: %s', - self.id, value) - logging.exception(error) - raise error - else: - self.networking_config = None - - @hybrid_property - def config(self): - """get config from security, networking, partition.""" - config = {} - if self.raw_config: - try: - config = json.loads(self.raw_config) - except Exception as error: - logging.error('failed to load raw config %s: %s', - self.id, self.raw_config) - logging.exception(error) - raise error - - util.merge_dict(config, {'security': self.security}) - util.merge_dict(config, {'networking': self.networking}) - util.merge_dict(config, {'partition': self.partition}) - util.merge_dict(config, {'clusterid': self.id, - 'clustername': self.name}) - return config - - @config.setter - def config(self, value): - """set config to security, networking, partition.""" - logging.debug('cluster %s set config %s', self.id, value) - if not value: - self.security = None - self.networking = None - self.partition = None - self.raw_config = None - return - - self.security = value.get('security') - self.networking = value.get('networking') - self.partition = value.get('partition') - - try: - self.raw_config = json.dumps(value) - except Exception as error: - logging.error('failed to dump raw config %s: %s', - self.id, value) - logging.exception(error) - raise error - - -class ClusterHost(BASE): - """ClusterHost information. - - :param id: int, identity as primary key. - :param machine_id: int, the id of the Machine. - :param cluster_id: int, the id of the Cluster. - :param mutable: if the ClusterHost information is mutable. - :param hostname: str, host name. - :param config_data: string, json formatted config data. - :param cluster: refer to Cluster the host in. - :param machine: refer to the Machine the host on. - :param state: refer to HostState indicates the host state. - """ - __tablename__ = 'cluster_host' - - id = Column(Integer, primary_key=True) - - machine_id = Column(Integer, ForeignKey('machine.id', - onupdate='CASCADE', - ondelete='CASCADE'), - nullable=True, unique=True) - - cluster_id = Column(Integer, ForeignKey('cluster.id', - onupdate='CASCADE', - ondelete='SET NULL'), - nullable=True) - - hostname = Column(String(80)) - config_data = Column(Text) - mutable = Column(Boolean, default=True) - __table_args__ = (UniqueConstraint('cluster_id', 'hostname', - name='unique_host'),) - - cluster = relationship("Cluster", - backref=backref('hosts', lazy='dynamic')) - machine = relationship("Machine", - backref=backref('host', uselist=False)) - - def __init__(self, **kwargs): - if 'hostname' not in kwargs or not kwargs['hostname']: - kwargs['hostname'] = str(uuid.uuid4()) - - super(ClusterHost, self).__init__(**kwargs) - - def __repr__(self): - return '' % ( - self.hostname, self.cluster, self.machine) - - @hybrid_property - def fullname(self): - return '%s.%s' % (self.hostname, self.cluster.id) - - @property - def config(self): - """config getter.""" - config = {} - try: - if self.config_data: - config.update(json.loads(self.config_data)) - - config.update({ - 'hostid': self.id, - 'hostname': self.hostname, - }) - if self.cluster: - config.update({ - 'clusterid': self.cluster.id, - 'clustername': self.cluster.name, - 'fullname': self.fullname, - }) - - if self.machine: - util.merge_dict( - config, { - 'networking': { - 'interfaces': { - 'management': { - 'mac': self.machine.mac - } - } - }, - 'switch_port': self.machine.port, - 'vlan': self.machine.vlan, - }) - if self.machine.switch: - util.merge_dict( - config, {'switch_ip': self.machine.switch.ip}) - - except Exception as error: - logging.error('failed to load config %s: %s', - self.hostname, self.config_data) - logging.exception(error) - raise error - - return config - - @config.setter - def config(self, value): - """config setter""" - if not self.config_data: - config = { - } - self.config_data = json.dumps(config) - - if value: - try: - config = json.loads(self.config_data) - util.merge_dict(config, value) - - self.config_data = json.dumps(config) - except Exception as error: - logging.error('failed to dump config %s: %s', - self.hostname, value) - logging.exception(error) - raise error - - -class LogProgressingHistory(BASE): - """host installing log history for each file. - - :param id: int, identity as primary key. - :param pathname: str, the full path of the installing log file. unique. - :param position: int, the position of the log file it has processed. - :param partial_line: str, partial line of the log. - :param progressing: float, indicate the installing progress between 0 to 1. - :param message: str, str, the installing message. - :param severity: Enum, the installing message severity. - ('ERROR', 'WARNING', 'INFO') - :param line_matcher_name: str, the line matcher name of the log processor. - :param update_timestamp: datetime, the latest timestamp the entry updated. - """ - __tablename__ = 'log_progressing_history' - id = Column(Integer, primary_key=True) - pathname = Column(String(80), unique=True) - position = Column(Integer, ColumnDefault(0)) - partial_line = Column(Text) - progress = Column(Float, ColumnDefault(0.0)) - message = Column(Text) - severity = Column(Enum('ERROR', 'WARNING', 'INFO'), ColumnDefault('INFO')) - line_matcher_name = Column(String(80), ColumnDefault('start')) - update_timestamp = Column(DateTime, default=datetime.now, - onupdate=datetime.now) - - def __init__(self, **kwargs): - super(LogProgressingHistory, self).__init__(**kwargs) - - def __repr__(self): - return ( - 'LogProgressingHistory[%r: position %r,' - 'partial_line %r,progress %r,message %r,' - 'severity %r]' - ) % ( - self.pathname, self.position, - self.partial_line, - self.progress, - self.message, - self.severity - ) - - -class Adapter(BASE): - """Table stores ClusterHost installing Adapter information. - - :param id: int, identity as primary key. - :param name: string, adapter name, unique. - :param os: string, os name for installing the host. - :param target_system: string, target system to be installed on the host. - :param clusters: refer to the list of Cluster. - """ - __tablename__ = 'adapter' - id = Column(Integer, primary_key=True) - name = Column(String(80), unique=True) - os = Column(String(80)) - target_system = Column(String(80)) - __table_args__ = ( - UniqueConstraint('os', 'target_system', name='unique_adapter'),) - - def __init__(self, **kwargs): - super(Adapter, self).__init__(**kwargs) - - def __repr__(self): - return '' % ( - self.name, self.os, self.target_system - ) - - -class Role(BASE): - """The Role table stores avaiable roles of one target system. - - .. note:: - the host can be deployed to one or several roles in the cluster. - - :param id: int, identity as primary key. - :param name: role name. - :param target_system: str, the target_system. - :param description: str, the description of the role. - """ - __tablename__ = 'role' - id = Column(Integer, primary_key=True) - name = Column(String(80), unique=True) - target_system = Column(String(80)) - description = Column(Text) - - def __init__(self, **kwargs): - super(Role, self).__init__(**kwargs) - - def __repr__(self): - return '' % ( - self.name, self.target_system, self.description) diff --git a/compass-tasks/db/validator.py b/compass-tasks/db/validator.py deleted file mode 100644 index 730bb52..0000000 --- a/compass-tasks/db/validator.py +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Validator methods.""" -import logging -import netaddr -import re -import socket - -from compass.utils import setting_wrapper as setting -from compass.utils import util - - -def is_valid_ip(name, ip_addr, **kwargs): - """Valid the format of an IP address.""" - if isinstance(ip_addr, list): - return all([ - is_valid_ip(name, item, **kwargs) for item in ip_addr - ]) - try: - netaddr.IPAddress(ip_addr) - except Exception: - logging.debug('%s invalid ip addr %s', name, ip_addr) - return False - return True - - -def is_valid_network(name, ip_network, **kwargs): - """Valid the format of an Ip network.""" - if isinstance(ip_network, list): - return all([ - is_valid_network(name, item, **kwargs) for item in ip_network - ]) - try: - netaddr.IPNetwork(ip_network) - except Exception: - logging.debug('%s invalid network %s', name, ip_network) - return False - return True - - -def is_valid_netmask(name, ip_addr, **kwargs): - """Valid the format of a netmask.""" - if isinstance(ip_addr, list): - return all([ - is_valid_netmask(name, item, **kwargs) for item in ip_addr - ]) - if not is_valid_ip(ip_addr): - return False - ip = netaddr.IPAddress(ip_addr) - if ip.is_netmask(): - return True - logging.debug('%s invalid netmask %s', name, ip_addr) - return False - - -def is_valid_gateway(name, ip_addr, **kwargs): - """Valid the format of gateway.""" - if isinstance(ip_addr, list): - return all([ - is_valid_gateway(name, item, **kwargs) for item in ip_addr - ]) - if not is_valid_ip(ip_addr): - return False - ip = netaddr.IPAddress(ip_addr) - if ip.is_private() or ip.is_public(): - return True - logging.debug('%s invalid gateway %s', name, ip_addr) - return False - - -def is_valid_dns(name, dns, **kwargs): - """Valid the format of DNS.""" - if isinstance(dns, list): - return all([is_valid_dns(name, item, **kwargs) for item in dns]) - if is_valid_ip(dns): - return True - try: - socket.gethostbyname_ex(dns) - except Exception: - logging.debug('%s invalid dns name %s', name, dns) - return False - return True - - -def is_valid_url(name, url, **kwargs): - """Valid the format of url.""" - if isinstance(url, list): - return all([ - is_valid_url(name, item, **kwargs) for item in url - ]) - if re.match( - r'^(http|https|ftp)://([0-9A-Za-z_-]+)(\.[0-9a-zA-Z_-]+)*' - r'(:\d+)?(/[0-9a-zA-Z_-]+)*$', - url - ): - return True - logging.debug( - '%s invalid url %s', name, url - ) - return False - - -def is_valid_domain(name, domain, **kwargs): - """Validate the format of domain.""" - if isinstance(domain, list): - return all([ - is_valid_domain(name, item, **kwargs) for item in domain - ]) - if re.match( - r'^([0-9a-zA-Z_-]+)(\.[0-9a-zA-Z_-]+)*$', - domain - ): - return True - logging.debug( - '%s invalid domain %s', name, domain - ) - return False - - -def is_valid_username(name, username, **kwargs): - """Valid the format of username.""" - if bool(username): - return True - logging.debug( - '%s username is empty', name - ) - - -def is_valid_password(name, password, **kwargs): - """Valid the format of password.""" - if bool(password): - return True - logging.debug('%s password is empty', name) - return False - - -def is_valid_partition(name, partition, **kwargs): - """Valid the format of partition name.""" - if name != 'swap' and not name.startswith('/'): - logging.debug( - '%s is not started with / or swap', name - ) - return False - if 'size' not in partition and 'percentage' not in partition: - logging.debug( - '%s partition does not contain sie or percentage', - name - ) - return False - return True - - -def is_valid_percentage(name, percentage, **kwargs): - """Valid the percentage.""" - if 0 <= percentage <= 100: - return True - logging.debug('%s invalid percentage %s', name, percentage) - - -def is_valid_port(name, port, **kwargs): - """Valid the format of port.""" - if 0 < port < 65536: - return True - logging.debug('%s invalid port %s', name, port) - - -def is_valid_size(name, size, **kwargs): - if re.match(r'^(\d+)(K|M|G|T)$', size): - return True - logging.debug('%s invalid size %s', name, size) - return False - - -VALIDATOR_GLOBALS = globals() -VALIDATOR_LOCALS = locals() -VALIDATOR_CONFIGS = util.load_configs( - setting.VALIDATOR_DIR, - config_name_suffix='.py', - env_globals=VALIDATOR_GLOBALS, - env_locals=VALIDATOR_LOCALS -) -for validator_config in VALIDATOR_CONFIGS: - VALIDATOR_LOCALS.update(validator_config) diff --git a/compass-tasks/deployment/__init__.py b/compass-tasks/deployment/__init__.py deleted file mode 100644 index cbd36e0..0000000 --- a/compass-tasks/deployment/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = "Grace Yu (grace.yu@huawei.com)" diff --git a/compass-tasks/deployment/deploy_manager.py b/compass-tasks/deployment/deploy_manager.py deleted file mode 100644 index baf7cd6..0000000 --- a/compass-tasks/deployment/deploy_manager.py +++ /dev/null @@ -1,237 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = "Grace Yu (grace.yu@huawei.com)" - -"""Module to get configs from provider and isntallers and update - them to provider and installers. -""" -from compass.deployment.installers.installer import OSInstaller -from compass.deployment.installers.installer import PKInstaller -from compass.deployment.utils import constants as const -from compass.utils import util - - -import logging - - -class DeployManager(object): - """Deploy manager module.""" - def __init__(self, adapter_info, cluster_info, hosts_info): - """Init deploy manager.""" - self.os_installer = None - self.pk_installer = None - - # Get OS installer - os_installer_name = adapter_info[const.OS_INSTALLER][const.NAME] - self.os_installer = DeployManager._get_installer(OSInstaller, - os_installer_name, - adapter_info, - cluster_info, - hosts_info) - - # Get package installer - pk_info = adapter_info.setdefault(const.PK_INSTALLER, {}) - if pk_info: - pk_installer_name = pk_info[const.NAME] - self.pk_installer = DeployManager._get_installer(PKInstaller, - pk_installer_name, - adapter_info, - cluster_info, - hosts_info) - - @staticmethod - def _get_installer(installer_type, name, adapter_info, cluster_info, - hosts_info): - """Get installer instance.""" - callback = getattr(installer_type, 'get_installer') - installer = callback(name, adapter_info, cluster_info, hosts_info) - - return installer - - def deploy(self): - """Deploy the cluster.""" - deployed_config = self.deploy_os() - package_deployed_config = self.deploy_target_system() - - util.merge_dict(deployed_config, package_deployed_config) - - return deployed_config - - def check_cluster_health(self, callback_url): - logging.info("DeployManager check_cluster_health...........") - self.pk_installer.check_cluster_health(callback_url) - - def clean_progress(self): - """Clean previous installation log and progress.""" - self.clean_os_installtion_progress() - self.clean_package_installation_progress() - - def clean_os_installtion_progress(self): - # OS installer cleans previous installing progress. - if self.os_installer: - self.os_installer.clean_progress() - - def clean_package_installation_progress(self): - # Package installer cleans previous installing progress. - if self.pk_installer: - self.pk_installer.clean_progress() - - def prepare_for_deploy(self): - self.clean_progress() - - def deploy_os(self): - """Deploy OS to hosts which need to in the cluster. - - Return OS deployed config. - """ - if not self.os_installer: - return {} - - pk_installer_config = {} - if self.pk_installer: - # generate target system config which will be installed by OS - # installer right after OS installation is completed. - pk_installer_config = self.pk_installer.generate_installer_config() - logging.debug('[DeployManager]package installer config is %s', - pk_installer_config) - - # Send package installer config info to OS installer. - self.os_installer.set_package_installer_config(pk_installer_config) - - # start to deploy OS - return self.os_installer.deploy() - - def deploy_target_system(self): - """Deploy target system to all hosts in the cluster. - - Return package deployed config. - """ - if not self.pk_installer: - return {} - - return self.pk_installer.deploy() - - def redeploy_os(self): - """Redeploy OS for this cluster without changing configurations.""" - if not self.os_installer: - logging.info("Redeploy_os: No OS installer found!") - return - - self.os_installer.redeploy() - logging.info("Start to redeploy OS for cluster.") - - def redeploy_target_system(self): - """Redeploy target system for the cluster without changing config.""" - if not self.pk_installer: - logging.info("Redeploy_target_system: No installer found!") - return - - self.pk_installer.deploy() - logging.info("Start to redeploy target system.") - - def redeploy(self): - """Redeploy the cluster without changing configurations.""" - self.redeploy_os() - self.redeploy_target_system() - - def remove_hosts(self, package_only=False, delete_cluster=False): - """Remove hosts from both OS and/or package installlers server side.""" - if self.os_installer and not package_only: - self.os_installer.delete_hosts() - - if self.pk_installer: - self.pk_installer.delete_hosts(delete_cluster=delete_cluster) - - def os_installed(self): - if self.os_installer: - self.os_installer.ready() - if self.pk_installer: - self.pk_installer.os_ready() - - def cluster_os_installed(self): - if self.os_installer: - self.os_installer.cluster_ready() - if self.pk_installer: - self.pk_installer.cluster_os_ready() - - def package_installed(self): - if self.pk_installer: - self.pk_installer.ready() - - def cluster_installed(self): - if self.pk_installer: - self.pk_installer.cluster_ready() - - -class Patcher(DeployManager): - """Patcher Module.""" - def __init__(self, adapter_info, cluster_info, hosts_info, cluster_hosts): - self.pk_installer = None - self.cluster_info = cluster_info - registered_roles = cluster_info['flavor']['roles'] - - pk_info = adapter_info.setdefault(const.PK_INSTALLER, {}) - if pk_info: - pk_installer_name = pk_info[const.NAME] - self.pk_installer = Patcher._get_installer(PKInstaller, - pk_installer_name, - adapter_info, - cluster_info, - hosts_info) - - patched_role_mapping = {} - for role in registered_roles: - patched_role_mapping[role] = [] - for host in cluster_hosts: - if len(host['patched_roles']) == 0: - continue - for role in host['patched_roles']: - patched_role_mapping[role['name']].append(host) - self.patched_role_mapping = patched_role_mapping - - def patch(self): - patched_config = self.pk_installer.patch(self.patched_role_mapping) - - return patched_config - - -class PowerManager(object): - """Manage host to power on, power off, and reset.""" - - def __init__(self, adapter_info, cluster_info, hosts_info): - os_installer_name = adapter_info[const.OS_INSTALLER][const.NAME] - self.os_installer = DeployManager._get_installer(OSInstaller, - os_installer_name, - adapter_info, - cluster_info, - hosts_info) - - def poweron(self): - if not self.os_installer: - logging.info("No OS installer found, cannot power on machine!") - return - self.os_installer.poweron() - - def poweroff(self): - if not self.os_installer: - logging.info("No OS installer found, cannot power on machine!") - return - self.os_installer.poweroff() - - def reset(self): - if not self.os_installer: - logging.info("No OS installer found, cannot power on machine!") - return - self.os_installer.reset() diff --git a/compass-tasks/deployment/installers/__init__.py b/compass-tasks/deployment/installers/__init__.py deleted file mode 100644 index 0296be5..0000000 --- a/compass-tasks/deployment/installers/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = "Grace Yu (grace.yu@huawei.com)" - - -"""modules to read/write cluster/host config from installers. - - .. moduleauthor:: Grace Yu -""" diff --git a/compass-tasks/deployment/installers/config_manager.py b/compass-tasks/deployment/installers/config_manager.py deleted file mode 100644 index 597c3a6..0000000 --- a/compass-tasks/deployment/installers/config_manager.py +++ /dev/null @@ -1,527 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = "baigk baiguoku@huawei.com)" - -from collections import defaultdict -from copy import deepcopy -import json -import logging -import netaddr - -from compass.deployment.utils import constants as const - -ip_generator_map = {} - - -def get_ip_addr(ip_ranges): - def _get_ip_addr(): - for ip_range in ip_ranges: - for ip in netaddr.iter_iprange(*ip_range): - yield str(ip) - - s = json.dumps(ip_ranges) - if s not in ip_generator_map: - ip_generator_map[s] = _get_ip_addr() - return ip_generator_map[s] - else: - return ip_generator_map[s] - - -class AdapterInfo(object): - def __init__(self, adapter_info): - self.adapter_info = adapter_info - self.name = self.adapter_info.get(const.NAME) - self.dist_system_name = self.name - self.health_check_cmd = self.adapter_info.get(const.HEALTH_CHECK_CMD) - - self.os_installer = self.adapter_info.setdefault( - const.OS_INSTALLER, {} - ) - self.os_installer.setdefault(const.INSTALLER_SETTINGS, {}) - - self.package_installer = self.adapter_info.setdefault( - const.PK_INSTALLER, {} - ) - self.package_installer.setdefault(const.INSTALLER_SETTINGS, {}) - - self.metadata = self.adapter_info.setdefault(const.METADATA, {}) - self.os_metadata = self.metadata.setdefault(const.OS_CONFIG, {}) - self.package_metadata = self.metadata.setdefault(const.PK_CONFIG, {}) - - self.flavors = dict([(f[const.FLAVOR_NAME], f) - for f in self.adapter_info.get(const.FLAVOR, [])]) - - @property - def flavor_list(self): - return self.flavors.values() - - def get_flavor(self, flavor_name): - return self.flavors.get(flavor_name) - - -class ClusterInfo(object): - def __init__(self, cluster_info): - self.cluster_info = cluster_info - self.id = self.cluster_info.get(const.ID) - self.name = self.cluster_info.get(const.NAME) - self.os_version = self.cluster_info.get(const.OS_VERSION) - self.flavor = self.cluster_info.setdefault( - const.FLAVOR, {} - ) - self.os_config = self.cluster_info.setdefault( - const.OS_CONFIG, {} - ) - self.package_config = self.cluster_info.setdefault( - const.PK_CONFIG, {} - ) - self.deployed_os_config = self.cluster_info.setdefault( - const.DEPLOYED_OS_CONFIG, {} - ) - self.deployed_package_config = self.cluster_info.setdefault( - const.DEPLOYED_PK_CONFIG, {} - ) - self.network_mapping = self.package_config.setdefault( - const.NETWORK_MAPPING, {} - ) - - os_config_general = self.os_config.setdefault( - const.OS_CONFIG_GENERAL, {} - ) - self.domain = os_config_general.setdefault(const.DOMAIN, None) - self.hosts = [] - - def add_host(self, host): - self.hosts.append(host) - - @property - def roles_mapping(self): - deploy_config = self.deployed_package_config - return deploy_config.setdefault( - const.ROLES_MAPPING, self._get_cluster_roles_mapping() - ) - - def _get_cluster_roles_mapping(self): - """The ouput format will be as below, for example: - - { - "controller": [{ - "hostname": "xxx", - "management": { - "interface": "eth0", - "ip": "192.168.1.10", - "netmask": "255.255.255.0", - "subnet": "192.168.1.0/24", - "is_mgmt": True, - "is_promiscuous": False - }, - ... - }], - ... - } - """ - mapping = defaultdict(list) - for host in self.hosts: - for role, value in host.roles_mapping.iteritems(): - mapping[role].append(value) - - return dict(mapping) - - def _get_cluster_patched_roles_mapping(self): - mapping = defaultdict(list) - for host in self.hosts: - for role, value in host.patched_roles_mapping.iteritems(): - mapping[role].append(value) - - return dict(mapping) - - @property - def base_info(self): - return { - const.ID: self.id, - const.NAME: self.name, - const.OS_VERSION: self.os_version - } - - -class HostInfo(object): - def __init__(self, host_info, cluster_info): - self.host_info = host_info - self.cluster_info = cluster_info - self.id = self.host_info.get(const.ID) - self.name = self.host_info.get(const.NAME) - self.mac = self.host_info.get(const.MAC_ADDR) - self.hostname = self.host_info.get(const.HOSTNAME) - self.networks = self.host_info.setdefault(const.NETWORKS, {}) - self.os_config = self.host_info.setdefault(const.OS_CONFIG, {}) - - self.package_config = self.host_info.setdefault(const.PK_CONFIG, {}) - self.roles = self.host_info.setdefault(const.ROLES, []) - self.patched_roles = self.host_info.setdefault(const.PATCHED_ROLES, []) - self.ipmi = deepcopy(self.host_info.setdefault(const.IPMI, {})) - self.reinstall_os_flag = self.host_info.get(const.REINSTALL_OS_FLAG) - self.deployed_os_config = self.host_info.setdefault( - const.DEPLOYED_OS_CONFIG, {} - ) - self.deployed_package_config = self.host_info.setdefault( - const.DEPLOYED_PK_CONFIG, {} - ) - - os_general_config = self.os_config.setdefault( - const.OS_CONFIG_GENERAL, {} - ) - domain = os_general_config.setdefault(const.DOMAIN, None) - if domain is None: - self.domain = self.cluster_info.domain - else: - self.domain = domain - - if const.DNS in host_info: - self.dns = host_info[const.DNS] - else: - self.dns = '.'.join((self.hostname, self.domain)) - - if const.NETWORK_MAPPING not in self.package_config: - self.network_mapping = self.cluster_info.network_mapping - else: - self.network_mapping = self.package_config[const.NETWORK_MAPPING] - - if const.ROLES_MAPPING not in self.deployed_package_config: - self.roles_mapping = self._get_host_roles_mapping() - self.deployed_package_config[ - const.ROLES_MAPPING - ] = self.roles_mapping - else: - self.roles_mapping = \ - self.deployed_package_config[const.ROLES_MAPPING] - - self.patched_roles_mapping = self._get_host_patched_roles_mapping() - - self.cluster_info.add_host(self) - - def valid_interface(self, interface): - if interface not in self.networks: - raise RuntimeError("interface %s is invalid" % interface) - - def get_interface(self, interface): - self.valid_interface(interface) - return self.networks[interface] - - def get_interface_ip(self, interface): - return self.get_interface(interface).get(const.IP_ADDR) - - def get_interface_netmask(self, interface): - return self.get_interface(interface).get(const.NETMASK) - - def get_interface_subnet(self, interface): - return self.get_interface(interface).get(const.SUBNET) - - def is_interface_promiscuous(self, interface): - return self.get_interface(interface).get(const.PROMISCUOUS_FLAG) - - def is_interface_mgmt(self, interface): - return self.get_interface(interface).get(const.MGMT_NIC_FLAG) - - def _get_host_roles_mapping(self): - if not self.network_mapping: - return {} - - net_info = {const.HOSTNAME: self.hostname} - for k, v in self.network_mapping.items(): - try: - net_info[k] = self.networks[v[const.NIC]] - net_info[k][const.NIC] = v[const.NIC] - except Exception: - pass - - mapping = {} - for role in self.roles: - role = role.replace("-", "_") - mapping[role] = net_info - - return mapping - - def _get_host_patched_roles_mapping(self): - if not self.network_mapping: - return {} - - net_info = {const.HOSTNAME: self.hostname} - for k, v in self.network_mapping.items(): - try: - net_info[k] = self.networks[v[const.NIC]] - net_info[k][const.NIC] = v[const.NIC] - except Exception: - pass - - mapping = {} - for role in self.patched_roles: - role = role['name'].replace("-", "_") - mapping[role] = net_info - - return mapping - - @property - def baseinfo(self): - return { - const.REINSTALL_OS_FLAG: self.reinstall_os_flag, - const.MAC_ADDR: self.mac, - const.NAME: self.name, - const.HOSTNAME: self.hostname, - const.DNS: self.dns, - const.NETWORKS: deepcopy(self.networks) - } - - -class BaseConfigManager(object): - def __init__(self, adapter_info={}, cluster_info={}, hosts_info={}): - assert(adapter_info and isinstance(adapter_info, dict)) - assert(cluster_info and isinstance(cluster_info, dict)) - assert(hosts_info and isinstance(hosts_info, dict)) - - self.adapter_info = AdapterInfo(adapter_info) - self.cluster_info = ClusterInfo(cluster_info) - self.hosts_info = dict([(k, HostInfo(v, self.cluster_info)) - for k, v in hosts_info.iteritems()]) - - def get_adapter_name(self): - return self.adapter_info.name - - def get_dist_system_name(self): - return self.adapter_info.dist_system_name - - def get_adapter_health_check_cmd(self): - return self.adapter_info.health_check_cmd - - def get_os_installer_settings(self): - return self.adapter_info.os_installer[const.INSTALLER_SETTINGS] - - def get_pk_installer_settings(self): - return self.adapter_info.package_installer[const.INSTALLER_SETTINGS] - - def get_os_config_metadata(self): - return self.adapter_info.metadata[const.OS_CONFIG] - - def get_pk_config_meatadata(self): - return self.adapter_info.metadata[const.PK_CONFIG] - - def get_adapter_all_flavors(self): - return self.adapter_info.flavor_list - - def get_adapter_flavor(self, flavor_name): - return self.adapter_info.get_flavor(flavor_name) - - def get_cluster_id(self): - return self.cluster_info.id - - def get_clustername(self): - return self.cluster_info.name - - def get_os_version(self): - return self.cluster_info.os_version - - def get_cluster_os_config(self): - return self.cluster_info.os_config - - def get_cluster_baseinfo(self): - return self.cluster_info.base_info - - def get_cluster_flavor_name(self): - return self.cluster_info.flavor.get(const.FLAVOR_NAME) - - def get_cluster_flavor_roles(self): - return self.cluster_info.flavor.get(const.ROLES, []) - - def get_cluster_flavor_template(self): - return self.cluster_info.flavor.get(const.TMPL) - - def get_cluster_package_config(self): - return self.cluster_info.package_config - - def get_cluster_network_mapping(self): - mapping = self.cluster_info.network_mapping - logging.info("Network mapping in the config is '%s'!", mapping) - return mapping - - def get_cluster_deployed_os_config(self): - return self.cluster_info.deployed_os_config - - def get_cluster_deployed_package_config(self): - return self.cluster_info.deployed_package_config - - def get_cluster_roles_mapping(self): - return self.cluster_info.roles_mapping - - def get_cluster_patched_roles_mapping(self): - return self.cluster_info._get_cluster_patched_roles_mapping() - - def validate_host(self, host_id): - if host_id not in self.hosts_info: - raise RuntimeError("host_id %s is invalid" % host_id) - - def get_host_id_list(self): - return self.hosts_info.keys() - - def get_hosts_id_list_for_os_installation(self): - """Get info of hosts which need to install/reinstall OS.""" - return [ - id for id, info in self.hosts_info.items() - if info.reinstall_os_flag - ] - - def get_server_credentials(self): - cluster_os_config = self.get_cluster_os_config() - if not cluster_os_config: - logging.info("cluster os_config is None!") - return () - - username = cluster_os_config[const.SERVER_CREDS][const.USERNAME] - password = cluster_os_config[const.SERVER_CREDS][const.PASSWORD] - return (username, password) - - def _get_host_info(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id] - - def get_host_baseinfo(self, host_id): - self.validate_host(host_id) - host_info = self.hosts_info[host_id] - return host_info.baseinfo - - def get_host_fullname(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].name - - def get_host_dns(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].dns - - def get_host_mac_address(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].mac - - def get_hostname(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].hostname - - def get_host_networks(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].networks - - def get_host_interfaces(self, host_id): - # get interface names - return self.get_host_networks(host_id).keys() - - def get_host_interface_ip(self, host_id, interface): - self.validate_host(host_id) - return self.hosts_info[host_id].get_interface_ip(interface) - - def get_host_interface_netmask(self, host_id, interface): - self.validate_host(host_id) - return self.hosts_info[host_id].get_interface_netmask(interface) - - def get_host_interface_subnet(self, host_id, interface): - self.validate_host(host_id) - return self.hosts_info[host_id].get_interface_subnet(interface) - - def is_interface_promiscuous(self, host_id, interface): - self.validate_host(host_id) - return self.hosts_info[host_id].is_interface_promiscuous(interface) - - def is_interface_mgmt(self, host_id, interface): - self.validate_host(host_id) - return self.hosts_info[host_id].is_interface_mgmt(interface) - - def get_host_os_config(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].os_config - - def get_host_domain(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].domain - - def get_host_network_mapping(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].network_mapping - - def get_host_package_config(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].package_config - - def get_host_deployed_os_config(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].deployed_os_config - - def get_host_deployed_package_config(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].deployed_package_config - - def get_host_roles(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].roles - - def get_all_hosts_roles(self, hosts_id_list=None): - roles = [] - for host_id, host_info in self.hosts_info.iteritems(): - roles.extend(host_info.roles) - - return list(set(roles)) - - def get_hosts_ip_settings(self, ip_settings, sys_intf_mappings): - logging.info( - "get_hosts_ip_settings:ip_settings=%s, sys_intf_mappings=%s" % - (ip_settings, sys_intf_mappings) - ) - - intf_alias = {} - for m in sys_intf_mappings: - if "vlan_tag" in m: - intf_alias[m["name"]] = m["name"] - else: - intf_alias[m["name"]] = m["interface"] - - mappings = {} - hosts_id_list = self.get_host_id_list() - for host_id in hosts_id_list: - hostname = self.get_hostname(host_id) - mappings[hostname] = [] - for ip_info in ip_settings: - logging.info("ip_info=%s" % ip_info) - new_ip_info = deepcopy(ip_info) - del new_ip_info["ip_ranges"] - - ip_ranges = ip_info["ip_ranges"] - new_ip_info["netmask"] = netaddr.IPNetwork( - ip_info["cidr"] - ).netmask.bin.count("1") - new_ip_info["ip"] = get_ip_addr(ip_ranges).next() - new_ip_info["alias"] = intf_alias[ip_info["name"]] - mappings[hostname].append(new_ip_info) - - return {"ip_settings": mappings} - - def get_host_roles_mapping(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].roles_mapping - - def get_host_ipmi_info(self, host_id): - self.validate_host(host_id) - if self.hosts_info[host_id].ipmi: - return ( - self.hosts_info[host_id].ipmi[const.IP_ADDR], - self.hosts_info[host_id].ipmi - [const.IPMI_CREDS][const.USERNAME], - self.hosts_info[host_id].ipmi - [const.IPMI_CREDS][const.USERNAME]) - else: - return (None, None, None) diff --git a/compass-tasks/deployment/installers/installer.py b/compass-tasks/deployment/installers/installer.py deleted file mode 100644 index cfeb9e8..0000000 --- a/compass-tasks/deployment/installers/installer.py +++ /dev/null @@ -1,291 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = "Grace Yu (grace.yu@huawei.com)" - - -"""Module to provider installer interface. -""" -from Cheetah.Template import Template -from copy import deepcopy -import imp -import logging -import os -import simplejson as json - -from compass.deployment.installers.config_manager import BaseConfigManager -from compass.utils import setting_wrapper as compass_setting -from compass.utils import util - - -CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) - - -class BaseInstaller(object): - """Interface for installer.""" - NAME = 'installer' - - def __repr__(self): - return '%r[%r]' % (self.__class__.__name__, self.NAME) - - def deploy(self, **kwargs): - """virtual method to start installing process.""" - raise NotImplementedError - - def clean_progress(self, **kwargs): - raise NotImplementedError - - def delete_hosts(self, **kwargs): - """Delete hosts from installer server.""" - raise NotImplementedError - - def redeploy(self, **kwargs): - raise NotImplementedError - - def ready(self, **kwargs): - pass - - def cluster_ready(self, **kwargs): - pass - - def get_tmpl_vars_from_metadata(self, metadata, config): - """Get variables dictionary for rendering templates from metadata. - - :param dict metadata: The metadata dictionary. - :param dict config: The - """ - template_vars = {} - self._get_tmpl_vars_helper(metadata, config, template_vars) - - return template_vars - - def _get_key_mapping(self, metadata, key, is_regular_key): - """Get the keyword which the input key maps to. - - This keyword will be added to dictionary used to render templates. - - If the key in metadata has a mapping to another keyword which is - used for templates, then return this keyword. If the key is started - with '$', which is a variable in metadata, return the key itself as - the mapping keyword. If the key has no mapping, return None. - - :param dict metadata: metadata/submetadata dictionary. - :param str key: The keyword defined in metadata. - :param bool is_regular_key: False when the key defined in metadata - is a variable(starting with '$'). - """ - mapping_to = key - if is_regular_key: - try: - mapping_to = metadata['_self']['mapping_to'] - except Exception: - mapping_to = None - - return mapping_to - - def _get_submeta_by_key(self, metadata, key): - """Get submetadata dictionary. - - Based on current metadata key. And - determines the input key is a regular string keyword or a variable - keyword defined in metadata, which starts with '$'. - - :param dict metadata: The metadata dictionary. - :param str key: The keyword defined in the metadata. - """ - if key in metadata: - return (True, metadata[key]) - - temp = deepcopy(metadata) - if '_self' in temp: - del temp['_self'] - meta_key = temp.keys()[0] - if meta_key.startswith("$"): - return (False, metadata[meta_key]) - - raise KeyError("'%s' is invalid in metadata '%s'!" % (key, metadata)) - - def _get_tmpl_vars_helper(self, metadata, config, output): - for key, config_value in sorted(config.iteritems()): - is_regular_key, sub_meta = self._get_submeta_by_key(metadata, key) - mapping_to = self._get_key_mapping(sub_meta, key, is_regular_key) - - if isinstance(config_value, dict): - if mapping_to: - new_output = output[mapping_to] = {} - else: - new_output = output - - self._get_tmpl_vars_helper(sub_meta, config_value, new_output) - - elif mapping_to: - output[mapping_to] = config_value - - def get_config_from_template(self, tmpl_path, vars_dict): - logging.debug("template path is %s", tmpl_path) - logging.debug("vars_dict is %s", vars_dict) - - if not os.path.exists(tmpl_path) or not vars_dict: - logging.info("Template dir or vars_dict is None!") - return {} - - searchList = [] - copy_vars_dict = deepcopy(vars_dict) - for key, value in vars_dict.iteritems(): - if isinstance(value, dict): - temp = copy_vars_dict[key] - del copy_vars_dict[key] - searchList.append(temp) - searchList.append(copy_vars_dict) - - # Load base template first if it exists - base_config = {} - base_tmpl_path = os.path.join(os.path.dirname(tmpl_path), 'base.tmpl') - if os.path.isfile(base_tmpl_path) and base_tmpl_path != tmpl_path: - base_tmpl = Template(file=base_tmpl_path, searchList=searchList) - base_config = json.loads(base_tmpl.respond(), encoding='utf-8') - base_config = json.loads(json.dumps(base_config), encoding='utf-8') - - # Load specific template for current adapter - tmpl = Template(file=open(tmpl_path, "r"), searchList=searchList) - config = json.loads(tmpl.respond(), encoding='utf-8') - config = json.loads(json.dumps(config), encoding='utf-8') - - # Merge the two outputs - config = util.merge_dict(base_config, config) - - logging.debug("get_config_from_template resulting %s", config) - return config - - @classmethod - def get_installer(cls, name, path, adapter_info, cluster_info, hosts_info): - try: - mod_file, path, descr = imp.find_module(name, [path]) - if mod_file: - mod = imp.load_module(name, mod_file, path, descr) - config_manager = BaseConfigManager(adapter_info, cluster_info, - hosts_info) - return getattr(mod, mod.NAME)(config_manager) - - except ImportError as exc: - logging.error('No such module found: %s', name) - logging.exception(exc) - - return None - - -class OSInstaller(BaseInstaller): - """Interface for os installer.""" - NAME = 'OSInstaller' - INSTALLER_BASE_DIR = os.path.join(CURRENT_DIR, 'os_installers') - - def get_oses(self): - """virtual method to get supported oses. - - :returns: list of str, each is the supported os version. - """ - return [] - - @classmethod - def get_installer(cls, name, adapter_info, cluster_info, hosts_info): - if name is None: - logging.info("Installer name is None! No OS installer loaded!") - return None - - path = os.path.join(cls.INSTALLER_BASE_DIR, name) - installer = super(OSInstaller, cls).get_installer(name, path, - adapter_info, - cluster_info, - hosts_info) - - if not isinstance(installer, OSInstaller): - logging.info("Installer '%s' is not an OS installer!" % name) - return None - - return installer - - def poweron(self, host_id): - pass - - def poweroff(self, host_id): - pass - - def reset(self, host_id): - pass - - -class PKInstaller(BaseInstaller): - """Interface for package installer.""" - NAME = 'PKInstaller' - INSTALLER_BASE_DIR = os.path.join(CURRENT_DIR, 'pk_installers') - - def generate_installer_config(self): - raise NotImplementedError( - 'generate_installer_config is not defined in %s', - self.__class__.__name__ - ) - - def get_target_systems(self): - """virtual method to get available target_systems for each os. - - :param oses: supported os versions. - :type oses: list of st - - :returns: dict of os_version to target systems as list of str. - """ - return {} - - def get_roles(self, target_system): - """virtual method to get all roles of given target system. - - :param target_system: target distributed system such as OpenStack. - :type target_system: str - - :returns: dict of role to role description as str. - """ - return {} - - def os_ready(self, **kwargs): - pass - - def cluster_os_ready(self, **kwargs): - pass - - def serialize_config(self, config, destination): - with open(destination, "w") as f: - f.write(config) - - @classmethod - def get_installer(cls, name, adapter_info, cluster_info, hosts_info): - if name is None: - logging.info("Install name is None. No package installer loaded!") - return None - - path = os.path.join(cls.INSTALLER_BASE_DIR, name) - if not os.path.exists(path): - path = os.path.join(os.path.join(os.path.join( - compass_setting.PLUGINS_DIR, name), "implementation"), name) - if not os.path.exists(path): - logging.info("Installer '%s' does not exist!" % name) - return None - installer = super(PKInstaller, cls).get_installer(name, path, - adapter_info, - cluster_info, - hosts_info) - - if not isinstance(installer, PKInstaller): - logging.info("Installer '%s' is not a package installer!" % name) - return None - - return installer diff --git a/compass-tasks/deployment/installers/os_installers/__init__.py b/compass-tasks/deployment/installers/os_installers/__init__.py deleted file mode 100644 index 5e42ae9..0000000 --- a/compass-tasks/deployment/installers/os_installers/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/deployment/installers/os_installers/cobbler/__init__.py b/compass-tasks/deployment/installers/os_installers/cobbler/__init__.py deleted file mode 100644 index 5e42ae9..0000000 --- a/compass-tasks/deployment/installers/os_installers/cobbler/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/deployment/installers/os_installers/cobbler/cobbler.py b/compass-tasks/deployment/installers/os_installers/cobbler/cobbler.py deleted file mode 100644 index 9c2a935..0000000 --- a/compass-tasks/deployment/installers/os_installers/cobbler/cobbler.py +++ /dev/null @@ -1,449 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""os installer cobbler plugin. -""" -import logging -import os -import shutil -import xmlrpclib - -from compass.deployment.installers.installer import OSInstaller -from compass.deployment.utils import constants as const -from compass.utils import setting_wrapper as compass_setting -from compass.utils import util -from copy import deepcopy - - -NAME = 'CobblerInstaller' - - -class CobblerInstaller(OSInstaller): - """cobbler installer""" - CREDENTIALS = "credentials" - USERNAME = 'username' - PASSWORD = 'password' - - INSTALLER_URL = "cobbler_url" - TMPL_DIR = 'tmpl_dir' - SYS_TMPL = 'system.tmpl' - SYS_TMPL_NAME = 'system.tmpl' - SYS_PROFILE_NAME = 'profile.tmpl' - PROFILE = 'profile' - - POWER_TYPE = 'power_type' - POWER_ADDR = 'power_address' - POWER_USER = 'power_user' - POWER_PASS = 'power_pass' - - def __init__(self, config_manager): - super(CobblerInstaller, self).__init__() - - self.config_manager = config_manager - installer_settings = self.config_manager.get_os_installer_settings() - try: - username = installer_settings[self.CREDENTIALS][self.USERNAME] - password = installer_settings[self.CREDENTIALS][self.PASSWORD] - cobbler_url = installer_settings[self.INSTALLER_URL] - self.tmpl_dir = CobblerInstaller.get_tmpl_path() - - except KeyError as ex: - raise KeyError(ex.message) - - # The connection is created when cobbler installer is initialized. - self.remote = self._get_cobbler_server(cobbler_url) - self.token = self._get_token(username, password) - self.pk_installer_config = None - - logging.debug('%s instance created', 'CobblerInstaller') - - @classmethod - def get_tmpl_path(cls): - return os.path.join(compass_setting.TMPL_DIR, 'cobbler') - - def __repr__(self): - return '%r[remote=%r,token=%r' % ( - self.__class__.__name__, self.remote, self.token) - - def _get_cobbler_server(self, cobbler_url): - if not cobbler_url: - logging.error("Cobbler URL is None!") - raise Exception("Cobbler URL cannot be None!") - - return xmlrpclib.Server(cobbler_url) - - def _get_token(self, username, password): - if self.remote is None: - raise Exception("Cobbler remote instance is None!") - return self.remote.login(username, password) - - def get_supported_oses(self): - """get supported os versions. - - note:: - In cobbler, we treat profile name as the indicator - of os version. It is just a simple indicator - and not accurate. - """ - profiles = self.remote.get_profiles() - oses = [] - for profile in profiles: - oses.append(profile['name']) - return oses - - def deploy(self): - """Sync cobbler to catch up the latest update config and start to - - install OS. Return both cluster and hosts deploy configs. The return - format: - { - "cluster": { - "id": 1, - "deployed_os_config": {}, - }, - "hosts": { - 1($clusterhost_id): { - "deployed_os_config": {...}, - }, - .... - } - } - """ - host_ids = self.config_manager.get_hosts_id_list_for_os_installation() - if not host_ids: - # No hosts need to install OS - logging.info("Cobbler: No host needs to install OS.") - return {} - - os_version = self.config_manager.get_os_version() - profile = self._get_profile_from_server(os_version) - - global_vars_dict = self._get_cluster_tmpl_vars_dict() - - self.update_profile_config_to_cobbler(profile, global_vars_dict) - - hosts_deploy_config = {} - - for host_id in host_ids: - hostname = self.config_manager.get_hostname(host_id) - vars_dict = self._get_host_tmpl_vars_dict(host_id, - global_vars_dict, - hostname=hostname, - profile=profile) - - self.update_host_config_to_cobbler(host_id, hostname, vars_dict) - - # set host deploy config - host_config = {} - host_config[const.DEPLOYED_OS_CONFIG] = vars_dict[const.OS_CONFIG] - hosts_deploy_config[host_id] = host_config - - # sync to cobbler and trigger installtion. - self._sync() - - cluster_config = global_vars_dict.setdefault(const.OS_CONFIG, {}) - - return { - const.CLUSTER: { - const.ID: self.config_manager.get_cluster_id(), - const.DEPLOYED_OS_CONFIG: cluster_config - }, - const.HOSTS: hosts_deploy_config - } - - def clean_progress(self): - """clean log files and config for hosts which to deploy.""" - clusterhost_list = self.config_manager.get_host_id_list() - log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[NAME] - - for host_id in clusterhost_list: - hostname = self.config_manager.get_hostname(host_id) - self._clean_log(log_dir_prefix, hostname) - - def redeploy(self): - """redeploy hosts.""" - host_ids = self.config_manager.get_host_id_list() - if not host_ids: - logging.info("Cobbler: hostlist is None, no host is redeployed") - return - for host_id in host_ids: - hostname = self.config_manager.get_hostname(host_id) - sys_id = self._get_create_system(hostname) - if sys_id: - # enable netboot for this host - self._netboot_enabled(sys_id) - - self._sync() - - def set_package_installer_config(self, package_configs): - """Cobbler can install and configure package installer right after - - OS installation compelets by setting package_config info provided - by package installer. - - :param dict package_configs: The dict of config generated by package - installer for each clusterhost. The IDs - of clusterhosts are the keys of - package_configs. - """ - self.pk_installer_config = package_configs - - def _sync(self): - """Sync the updated config to cobbler and trigger installation.""" - try: - self.remote.sync(self.token) - os.system('sudo service rsyslog restart') - except Exception as ex: - logging.debug("Failed to sync cobbler server! Error: %s" % ex) - raise ex - - def dump_system_info(self, host_id): - - hostname = self.config_manager.get_hostname(host_id) - if self.remote is None or not hostname: - logging.info("[dump_system_info]Remote or hostname is None.") - return {} - - return self.remote.get_system_as_rendered(hostname) - - def _generate_system_config(self, host_id, host_vars_dict): - """Generate updated system config from the template. - - :param host_vars_dict: dict of variables for the system template to - generate system config dict for each host. - """ - os_version = self.config_manager.get_os_version() - - tmpl_path = os.path.join( - os.path.join(self.tmpl_dir, os_version), self.SYS_TMPL_NAME - ) - if not os.path.exists(tmpl_path): - err_msg = "Template '%s' does not exists!" % tmpl_path - logging.error(err_msg) - raise Exception(err_msg) - host_vars_dict[const.BASEINFO]['host_id'] = host_id - system_config = self.get_config_from_template(tmpl_path, - host_vars_dict) - - # update package config info to cobbler ksmeta - if self.pk_installer_config and host_id in self.pk_installer_config: - pk_config = self.pk_installer_config[host_id] - ksmeta = system_config.setdefault("ksmeta", {}) - util.merge_dict(ksmeta, pk_config) - system_config["ksmeta"] = ksmeta - - return system_config - - def _generate_profile_config(self, cluster_vars_dict): - os_version = self.config_manager.get_os_version() - tmpl_path = os.path.join( - os.path.join(self.tmpl_dir, os_version), self.SYS_PROFILE_NAME - ) - - return self.get_config_from_template(tmpl_path, cluster_vars_dict) - - def _get_profile_from_server(self, os_version): - """Get profile from cobbler server.""" - result = self.remote.find_profile({'name': os_version}) - if not result: - raise Exception("Cannot find profile for '%s'", os_version) - - profile = result[0] - return profile - - def _get_create_system(self, hostname): - """get system reference id for the host.""" - sys_name = hostname - sys_id = None - system_info = self.remote.find_system({"name": hostname}) - - if not system_info: - # Create a new system - sys_id = self.remote.new_system(self.token) - self.remote.modify_system(sys_id, "name", hostname, self.token) - logging.debug('create new system %s for %s', sys_id, sys_name) - else: - sys_id = self.remote.get_system_handle(sys_name, self.token) - - return sys_id - - def _get_profile_id(self, profilename): - """get profile reference id for the cluster.""" - return self.remote.get_profile_handle(profilename, self.token) - - def _clean_system(self, hostname): - """clean system.""" - sys_name = hostname - try: - self.remote.remove_system(sys_name, self.token) - logging.debug('system %s is removed', sys_name) - except Exception: - logging.debug('no system %s found to remove', sys_name) - - def _update_system_config(self, sys_id, system_config): - """update modify system.""" - for key, value in system_config.iteritems(): - self.remote.modify_system(sys_id, str(key), value, self.token) - - self.remote.save_system(sys_id, self.token) - - def _update_profile_config(self, profile_id, profile_config): - for key, value in profile_config.iteritems(): - self.remote.modify_profile(profile_id, str(key), value, self.token) - - self.remote.save_profile(profile_id, self.token) - - def _netboot_enabled(self, sys_id): - """enable netboot.""" - self.remote.modify_system(sys_id, 'netboot_enabled', True, self.token) - self.remote.save_system(sys_id, self.token) - - def _clean_log(self, log_dir_prefix, system_name): - """clean log.""" - log_dir = os.path.join(log_dir_prefix, system_name) - shutil.rmtree(log_dir, True) - - def update_host_config_to_cobbler(self, host_id, hostname, host_vars_dict): - """update host config and upload to cobbler server.""" - sys_id = self._get_create_system(hostname) - - system_config = self._generate_system_config(host_id, host_vars_dict) - logging.debug('%s system config to update: %s', host_id, system_config) - - self._update_system_config(sys_id, system_config) - self._netboot_enabled(sys_id) - - def update_profile_config_to_cobbler(self, profilename, cluster_vars_dict): - """update profile config and upload to cobbler server.""" - - profile_id = self._get_profile_id(profilename) - - profile_config = self._generate_profile_config(cluster_vars_dict) - logging.debug( - '%s profile config to update: %s', profilename, profile_config - ) - - self._update_profile_config(profile_id, profile_config) - - def delete_hosts(self): - hosts_id_list = self.config_manager.get_host_id_list() - logging.debug('delete hosts %s', hosts_id_list) - for host_id in hosts_id_list: - self.delete_single_host(host_id) - self._sync() - - def delete_single_host(self, host_id): - """Delete the host from cobbler server and clean up the installation - - progress. - """ - hostname = self.config_manager.get_hostname(host_id) - try: - log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[NAME] - self._clean_system(hostname) - self._clean_log(log_dir_prefix, hostname) - except Exception as ex: - logging.error("Deleting host got exception: %s", ex) - logging.exception(ex) - - def _get_host_tmpl_vars_dict(self, host_id, global_vars_dict, **kwargs): - """Generate template variables dictionary.""" - vars_dict = {} - if global_vars_dict: - # Set cluster template vars_dict from cluster os_config. - vars_dict = deepcopy(global_vars_dict) - - # Set hostname, MAC address and hostname, networks, dns and so on. - host_baseinfo = self.config_manager.get_host_baseinfo(host_id) - vars_dict[const.BASEINFO] = host_baseinfo - - # Set profile - if self.PROFILE in kwargs: - profile = kwargs[self.PROFILE] - else: - os_version = self.config_manager.get_os_version() - profile = self._get_profile_from_server(os_version) - - vars_dict[const.BASEINFO][self.PROFILE] = profile - - metadata = self.config_manager.get_os_config_metadata() - os_config = self.config_manager.get_host_os_config(host_id) - - # Get template variables values from host os_config - host_vars_dict = self.get_tmpl_vars_from_metadata(metadata, os_config) - util.merge_dict( - vars_dict.setdefault(const.OS_CONFIG, {}), host_vars_dict - ) - return vars_dict - - def _get_cluster_tmpl_vars_dict(self): - metadata = self.config_manager.get_os_config_metadata() - os_config = self.config_manager.get_cluster_os_config() - - cluster_vas_dict = {} - cluster_vas_dict[const.OS_CONFIG] = \ - self.get_tmpl_vars_from_metadata(metadata, os_config) - - return cluster_vas_dict - - def _check_and_set_system_impi(self, host_id, sys_id): - if not sys_id: - logging.info("System is None!") - return False - - system = self.dump_system_info(host_id) - if system[self.POWER_TYPE] != 'ipmilan' or not system[self.POWER_USER]: - # Set sytem power type to ipmilan if needs and set IPMI info - ipmi_info = self.config_manager.get_host_ipmi_info(host_id) - if not ipmi_info: - logging.info('No IPMI information found! Failed power on.') - return False - - ipmi_ip, ipmi_user, ipmi_pass = ipmi_info - power_opts = {} - power_opts[self.POWER_TYPE] = 'ipmilan' - power_opts[self.POWER_ADDR] = ipmi_ip - power_opts[self.POWER_USER] = ipmi_user - power_opts[self.POWER_PASS] = ipmi_pass - - self._update_system_config(sys_id, power_opts) - - return True - - def poweron(self, host_id): - hostname = self.config_manager.get_hostname(host_id) - sys_id = self._get_create_system(hostname) - if not self._check_and_set_system_impi(sys_id): - return - - self.remote.power_system(sys_id, self.token, power='on') - logging.info("Host with ID=%d starts to power on!" % host_id) - - def poweroff(self, host_id): - hostname = self.config_manager.get_hostname(host_id) - sys_id = self._get_create_system(hostname) - if not self._check_and_set_system_impi(sys_id): - return - - self.remote.power_system(sys_id, self.token, power='off') - logging.info("Host with ID=%d starts to power off!" % host_id) - - def reset(self, host_id): - hostname = self.config_manager.get_hostname(host_id) - sys_id = self._get_create_system(hostname) - if not self._check_and_set_system_impi(sys_id): - return - - self.remote.power_system(sys_id, self.token, power='reboot') - logging.info("Host with ID=%d starts to reboot!" % host_id) diff --git a/compass-tasks/deployment/installers/pk_installers/__init__.py b/compass-tasks/deployment/installers/pk_installers/__init__.py deleted file mode 100644 index 5e42ae9..0000000 --- a/compass-tasks/deployment/installers/pk_installers/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/deployment/installers/pk_installers/ansible_installer/__init__.py b/compass-tasks/deployment/installers/pk_installers/ansible_installer/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compass-tasks/deployment/installers/pk_installers/ansible_installer/ansible_installer.py b/compass-tasks/deployment/installers/pk_installers/ansible_installer/ansible_installer.py deleted file mode 100644 index 0a86be4..0000000 --- a/compass-tasks/deployment/installers/pk_installers/ansible_installer/ansible_installer.py +++ /dev/null @@ -1,441 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__auther__ = "Compass Dev Team (dev-team@syscompass.org)" - -"""package installer: ansible plugin.""" - -from Cheetah.Template import Template -from copy import deepcopy -import json -import logging -import os -import re -import shutil -import subprocess - -from compass.deployment.installers.installer import PKInstaller -from compass.deployment.utils import constants as const -from compass.utils import setting_wrapper as compass_setting -from compass.utils import util - -NAME = "AnsibleInstaller" - - -def byteify(input): - if isinstance(input, dict): - return dict([(byteify(key), byteify(value)) - for key, value in input.iteritems()]) - elif isinstance(input, list): - return [byteify(element) for element in input] - elif isinstance(input, unicode): - return input.encode('utf-8') - else: - return input - - -class AnsibleInstaller(PKInstaller): - INVENTORY_TMPL_DIR = 'inventories' - GROUPVARS_TMPL_DIR = 'vars' - INVENTORY_PATCH_TEMPALTE_DIR = 'inventories' - - # keywords in package installer settings - ANSIBLE_DIR = 'ansible_dir' - ANSIBLE_RUN_DIR = 'ansible_run_dir' - LOG_FILE = 'ansible_log_file' - ANSIBLE_CONFIG = 'ansible_config' - INVENTORY = 'inventory_file' - INVENTORY_JSON = 'inventory_json_file' - INVENTORY_GROUP = 'inventory_group' - GROUP_VARIABLE = 'group_variable' - HOSTS_PATH = 'etc_hosts_path' - RUNNER_DIRS = 'runner_dirs' - - def __init__(self, config_manager): - super(AnsibleInstaller, self).__init__() - - self.config_manager = config_manager - self.tmpl_name = self.config_manager.get_cluster_flavor_template() - self.installer_settings = ( - self.config_manager.get_pk_installer_settings() - ) - settings = self.installer_settings - self.ansible_dir = settings.setdefault(self.ANSIBLE_DIR, None) - self.ansible_run_dir = ( - settings.setdefault(self.ANSIBLE_RUN_DIR, None) - ) - self.log_file = settings.setdefault(self.LOG_FILE, None) - self.ansible_config = ( - settings.setdefault(self.ANSIBLE_CONFIG, None) - ) - self.inventory = settings.setdefault(self.INVENTORY, None) - self.inventory_json = settings.setdefault(self.INVENTORY_JSON, None) - self.inventory_group = settings.setdefault(self.INVENTORY_GROUP, None) - self.group_variable = ( - settings.setdefault(self.GROUP_VARIABLE, None) - ) - self.hosts_path = ( - settings.setdefault(self.HOSTS_PATH, None) - ) - self.runner_dirs = ( - settings.setdefault(self.RUNNER_DIRS, None) - ) - self.playbook = self.tmpl_name.replace('tmpl', 'yml') - self.runner_files = [self.playbook] - - adapter_name = self.config_manager.get_dist_system_name() - self.tmpl_dir = AnsibleInstaller.get_tmpl_path(adapter_name) - self.adapter_dir = os.path.join(self.ansible_dir, adapter_name) - logging.debug('%s instance created', self) - - @classmethod - def get_tmpl_path(cls, adapter_name): - tmpl_path = os.path.join( - os.path.join(compass_setting.TMPL_DIR, 'ansible_installer'), - adapter_name - ) - return tmpl_path - - def __repr__(self): - return '%s[name=%s,installer_url=%s]' % ( - self.__class__.__name__, self.NAME, self.installer_url) - - def dump_inventory(self, data, inventory): - with open(inventory, "w") as f: - json.dump(data, f, indent=4) - - def _generate_inventory_data(self, global_vars_dict): - vars_dict = global_vars_dict['roles_mapping'] - inventory_data = {} - inventory_data['_meta'] = {'hostvars': {}} - for item in self.inventory_group: - if item in vars_dict: - inventory_data[item] = {'hosts': []} - for host in vars_dict[item]: - hostname = host['hostname'] - if hostname not in inventory_data['_meta']['hostvars']: - host_dict = {} - host_dict['ansible_ssh_host'] = host['install']['ip'] - host_dict['ansible_ssh_user'] = 'root' - host_dict['ansible_ssh_pass'] = 'root' - inventory_data['_meta']['hostvars'].update( - {hostname: host_dict}) - inventory_data[item]['hosts'].append(hostname) - - inventory_data['ceph'] = {'children': - ['ceph_adm', 'ceph_mon', 'ceph_osd']} - return inventory_data - - def generate_installer_config(self): - """Render ansible config file by OS installing. - - The output format: - { - '1'($host_id/clusterhost_id):{ - 'tool': 'ansible', - }, - ..... - } - """ - host_ids = self.config_manager.get_host_id_list() - os_installer_configs = {} - for host_id in host_ids: - temp = { - "tool": "ansible", - } - os_installer_configs[host_id] = temp - - return os_installer_configs - - def get_env_name(self, dist_sys_name, cluster_name): - return "-".join((dist_sys_name, cluster_name)) - - def _get_cluster_tmpl_vars(self): - """Generate template variables dict - - Generates based on cluster level config. - The vars_dict will be: - { - "baseinfo": { - "id":1, - "name": "cluster01", - ... - }, - "package_config": { - .... //mapped from original package config based on metadata - }, - "role_mapping": { - .... - } - } - """ - cluster_vars_dict = {} - # set cluster basic information to vars_dict - cluster_baseinfo = self.config_manager.get_cluster_baseinfo() - cluster_vars_dict[const.BASEINFO] = cluster_baseinfo - - # get and set template variables from cluster package config. - pk_metadata = self.config_manager.get_pk_config_meatadata() - pk_config = self.config_manager.get_cluster_package_config() - - # get os config as ansible needs them - os_metadata = self.config_manager.get_os_config_metadata() - os_config = self.config_manager.get_cluster_os_config() - - pk_meta_dict = self.get_tmpl_vars_from_metadata(pk_metadata, pk_config) - os_meta_dict = self.get_tmpl_vars_from_metadata(os_metadata, os_config) - util.merge_dict(pk_meta_dict, os_meta_dict) - - cluster_vars_dict[const.PK_CONFIG] = pk_meta_dict - - # get and set roles_mapping to vars_dict - mapping = self.config_manager.get_cluster_roles_mapping() - logging.info("cluster role mapping is %s", mapping) - cluster_vars_dict[const.ROLES_MAPPING] = mapping - - # get ip settings to vars_dict - hosts_ip_settings = self.config_manager.get_hosts_ip_settings( - pk_meta_dict["network_cfg"]["ip_settings"], - pk_meta_dict["network_cfg"]["sys_intf_mappings"] - ) - logging.info("hosts_ip_settings is %s", hosts_ip_settings) - cluster_vars_dict["ip_settings"] = hosts_ip_settings - - return byteify(cluster_vars_dict) - - def _generate_inventory_attributes(self, global_vars_dict): - inventory_tmpl_path = os.path.join( - os.path.join(self.tmpl_dir, self.INVENTORY_TMPL_DIR), - self.tmpl_name - ) - if not os.path.exists(inventory_tmpl_path): - logging.error( - "Inventory template '%s' does not exist", self.tmpl_name - ) - raise Exception("Template '%s' does not exist!" % self.tmpl_name) - inventory_dir = os.path.join(global_vars_dict['run_dir'], 'inventories') - inventory_json = os.path.join(inventory_dir, self.inventory_json) - vars_dict = {'inventory_json': inventory_json} - return self.get_config_from_template( - inventory_tmpl_path, vars_dict - ) - - def _generate_group_vars_attributes(self, global_vars_dict): - logging.info("global vars dict is %s", global_vars_dict) - group_vars_tmpl_path = os.path.join( - os.path.join(self.tmpl_dir, self.GROUPVARS_TMPL_DIR), - self.tmpl_name - ) - if not os.path.exists(group_vars_tmpl_path): - logging.error("Vars template '%s' does not exist", - self.tmpl_name) - raise Exception("Template '%s' does not exist!" % self.tmpl_name) - - return self.get_config_from_template( - group_vars_tmpl_path, global_vars_dict - ) - - def _generate_hosts_attributes(self, global_vars_dict): - hosts_tmpl_path = os.path.join( - os.path.join(self.tmpl_dir, 'hosts'), self.tmpl_name - ) - if not os.path.exists(hosts_tmpl_path): - logging.error("Hosts template '%s' does not exist", self.tmpl_name) - raise Exception("Template '%s' does not exist!" % self.tmpl_name) - - return self.get_config_from_template(hosts_tmpl_path, global_vars_dict) - - def _generate_ansible_cfg_attributes(self, global_vars_dict): - ansible_cfg_tmpl_path = os.path.join( - os.path.join(self.tmpl_dir, 'ansible_cfg'), self.tmpl_name - ) - if not os.path.exists(ansible_cfg_tmpl_path): - logging.error("cfg template '%s' does not exist", self.tmpl_name) - raise Exception("Template '%s' does not exist!" % self.tmpl_name) - - return self.get_config_from_template( - ansible_cfg_tmpl_path, - global_vars_dict - ) - - def get_config_from_template(self, tmpl_path, vars_dict): - logging.debug("vars_dict is %s", vars_dict) - - if not os.path.exists(tmpl_path) or not vars_dict: - logging.info("Template dir or vars_dict is None!") - return {} - - searchList = [] - copy_vars_dict = deepcopy(vars_dict) - for key, value in vars_dict.iteritems(): - if isinstance(value, dict): - temp = copy_vars_dict[key] - del copy_vars_dict[key] - searchList.append(temp) - searchList.append(copy_vars_dict) - - # Load specific template for current adapter - tmpl = Template(file=open(tmpl_path, "r"), searchList=searchList) - return tmpl.respond() - - def _create_ansible_run_env(self, env_name, ansible_run_destination): - if os.path.exists(ansible_run_destination): - shutil.rmtree(ansible_run_destination, True) - - os.mkdir(ansible_run_destination) - - # copy roles to run env - dirs = self.runner_dirs - files = self.runner_files - for dir in dirs: - if not os.path.exists(os.path.join(self.ansible_dir, dir)): - continue - os.system( - "cp -rf %s %s" % ( - os.path.join(self.ansible_dir, dir), - ansible_run_destination - ) - ) - for file in files: - logging.info('file is %s', file) - shutil.copy( - os.path.join(self.adapter_dir, file), - os.path.join( - ansible_run_destination, - file - ) - ) - - def prepare_ansible(self, env_name, global_vars_dict): - ansible_run_destination = os.path.join(self.ansible_run_dir, env_name) - if os.path.exists(ansible_run_destination): - ansible_run_destination += "-expansion" - self._create_ansible_run_env(env_name, ansible_run_destination) - global_vars_dict.update({'run_dir': ansible_run_destination}) - - inv_config = self._generate_inventory_attributes(global_vars_dict) - inventory_dir = os.path.join(ansible_run_destination, 'inventories') - - vars_config = self._generate_group_vars_attributes(global_vars_dict) - vars_dir = os.path.join(ansible_run_destination, 'group_vars') - - hosts_config = self._generate_hosts_attributes(global_vars_dict) - hosts_destination = os.path.join( - ansible_run_destination, self.hosts_path - ) - - cfg_config = self._generate_ansible_cfg_attributes(global_vars_dict) - cfg_destination = os.path.join( - ansible_run_destination, - self.ansible_config - ) - - inventory_data = self._generate_inventory_data(global_vars_dict) - inventory_json_destination = os.path.join(inventory_dir, - self.inventory_json) - - os.mkdir(inventory_dir) - os.mkdir(vars_dir) - - inventory_destination = os.path.join(inventory_dir, self.inventory) - group_vars_destination = os.path.join(vars_dir, self.group_variable) - self.dump_inventory(inventory_data, inventory_json_destination) - self.serialize_config(inv_config, inventory_destination) - self.serialize_config(vars_config, group_vars_destination) - self.serialize_config(hosts_config, hosts_destination) - self.serialize_config(cfg_config, cfg_destination) - - def deploy(self): - """Start to deploy a distributed system. - - Return both cluster and hosts deployed configs. - The return format: - { - "cluster": { - "id": 1, - "deployed_package_config": { - "roles_mapping": {...}, - "service_credentials": {...}, - .... - } - }, - "hosts": { - 1($clusterhost_id): { - "deployed_package_config": {...} - }, - .... - } - } - """ - host_list = self.config_manager.get_host_id_list() - if not host_list: - return {} - - adapter_name = self.config_manager.get_adapter_name() - cluster_name = self.config_manager.get_clustername() - env_name = self.get_env_name(adapter_name, cluster_name) - - global_vars_dict = self._get_cluster_tmpl_vars() - logging.info( - '%s var dict: %s', self.__class__.__name__, global_vars_dict - ) - # Create ansible related files - self.prepare_ansible(env_name, global_vars_dict) - - def patch(self, patched_role_mapping): - adapter_name = self.config_manager.get_adapter_name() - cluster_name = self.config_manager.get_clustername() - env_name = self.get_env_name(adapter_name, cluster_name) - ansible_run_destination = os.path.join(self.ansible_run_dir, env_name) - inventory_dir = os.path.join(ansible_run_destination, 'inventories') - patched_global_vars_dict = self._get_cluster_tmpl_vars() - mapping = self.config_manager.get_cluster_patched_roles_mapping() - patched_global_vars_dict['roles_mapping'] = mapping - patched_inv = self._generate_inventory_attributes( - patched_global_vars_dict) - inv_file = os.path.join(inventory_dir, 'patched_inventory.yml') - self.serialize_config(patched_inv, inv_file) - config_file = os.path.join( - ansible_run_destination, self.ansible_config - ) - playbook_file = os.path.join(ansible_run_destination, self.playbook) - log_file = os.path.join(ansible_run_destination, 'patch.log') - cmd = "ANSIBLE_CONFIG=%s ansible-playbook -i %s %s" % (config_file, - inv_file, - playbook_file) - with open(log_file, 'w') as logfile: - subprocess.Popen(cmd, shell=True, stdout=logfile, stderr=logfile) - return patched_role_mapping - - def cluster_os_ready(self): - adapter_name = self.config_manager.get_adapter_name() - cluster_name = self.config_manager.get_clustername() - env_name = self.get_env_name(adapter_name, cluster_name) - ansible_run_destination = os.path.join(self.ansible_run_dir, env_name) - expansion_dir = ansible_run_destination + "-expansion" - if os.path.exists(expansion_dir): - ansible_run_destination = expansion_dir - inventory_dir = os.path.join(ansible_run_destination, 'inventories') - inventory_file = os.path.join(inventory_dir, self.inventory) - playbook_file = os.path.join(ansible_run_destination, self.playbook) - log_file = os.path.join(ansible_run_destination, 'run.log') - config_file = os.path.join( - ansible_run_destination, self.ansible_config - ) - os.system("chmod +x %s" % inventory_file) - cmd = "ANSIBLE_CONFIG=%s ansible-playbook -i %s %s" % (config_file, - inventory_file, - playbook_file) - with open(log_file, 'w') as logfile: - subprocess.Popen(cmd, shell=True, stdout=logfile, stderr=logfile) diff --git a/compass-tasks/deployment/utils/__init__.py b/compass-tasks/deployment/utils/__init__.py deleted file mode 100644 index cbd36e0..0000000 --- a/compass-tasks/deployment/utils/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = "Grace Yu (grace.yu@huawei.com)" diff --git a/compass-tasks/deployment/utils/constants.py b/compass-tasks/deployment/utils/constants.py deleted file mode 100644 index e90b1b2..0000000 --- a/compass-tasks/deployment/utils/constants.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = "Grace Yu (grace.yu@huawei.com)" - - -"""All keywords variables in deployment are defined in this module.""" - - -# General keywords -BASEINFO = 'baseinfo' -CLUSTER = 'cluster' -HOST = 'host' -HOSTS = 'hosts' -ID = 'id' -NAME = 'name' -PASSWORD = 'password' -USERNAME = 'username' - - -# Adapter info related keywords -FLAVOR = 'flavor' -FLAVORS = 'flavors' -PLAYBOOK = 'playbook' -FLAVOR_NAME = 'flavor_name' -HEALTH_CHECK_CMD = 'health_check_cmd' -TMPL = 'template' -INSTALLER_SETTINGS = 'settings' -METADATA = 'metadata' -OS_INSTALLER = 'os_installer' -PK_INSTALLER = 'package_installer' -SUPPORT_OSES = 'supported_oses' - - -# Cluster info related keywords -ADAPTER_ID = 'adapter_id' -OS_VERSION = 'os_name' - - -# Host info related keywords -DNS = 'dns' -DOMAIN = 'domain' -HOST_ID = 'host_id' -HOSTNAME = 'hostname' -IP_ADDR = 'ip' -IPMI = 'ipmi' -IPMI_CREDS = 'ipmi_credentials' -MAC_ADDR = 'mac' -MGMT_NIC_FLAG = 'is_mgmt' -NETMASK = 'netmask' -NETWORKS = 'networks' -NIC = 'interface' -CLUSTER_ID = 'cluster_id' -ORIGIN_CLUSTER_ID = 'origin_cluster_id' -PROMISCUOUS_FLAG = 'is_promiscuous' -REINSTALL_OS_FLAG = 'reinstall_os' -SUBNET = 'subnet' - - -# Cluster/host config related keywords -COMPLETED_PK_CONFIG = 'completed_package_config' -COMPLETED_OS_CONFIG = 'completed_os_config' -DEPLOYED_OS_CONFIG = 'deployed_os_config' -DEPLOYED_PK_CONFIG = 'deployed_package_config' -NETWORK_MAPPING = 'network_mapping' -OS_CONFIG = 'os_config' -OS_CONFIG_GENERAL = 'general' -PK_CONFIG = 'package_config' -ROLES = 'roles' -PATCHED_ROLES = 'patched_roles' -ROLES_MAPPING = 'roles_mapping' -SERVER_CREDS = 'server_credentials' -TMPL_VARS_DICT = 'vars_dict' diff --git a/compass-tasks/hdsdiscovery/SNMP_CONFIG.md b/compass-tasks/hdsdiscovery/SNMP_CONFIG.md deleted file mode 100644 index 337b542..0000000 --- a/compass-tasks/hdsdiscovery/SNMP_CONFIG.md +++ /dev/null @@ -1,33 +0,0 @@ -Install & Config Prerequisite Packages: - -1. Net-Snmp: - a. #apt-get install -y snmpd snmp libsnmp-python - b. #apt-get install -y snmp-mibs-downloader - For Centos: - # yum install net-snmp net-snmp-utils - - c. create vendor's mibs directory(for example): - - #mkdir -p /root/.snmp/mibs/huawei - - #vim /etc/snmp/snmp.conf (if not exists, create snmp.conf file) - * add vendor;s mibs directory: - mibdirs +/root/.snmp/mibs/huawei - * comment the line: - #mibs: - d. copy vendor's mibs to that directory - e. #vim /etc/default/snmpd - * modify the directive from - TRAPDRUN=no --> TRAPDRUN=yes - For Centos: - # vim /etc/sysconfig/snmpd - * modify into or add the directive - TRAPDRUN=yes - - f. #vim /etc/snmp/snmpd.conf - * add the following line, where $ip is the ip address of manager machine: - com2sec mynetwork $ip/24 public - g. #service snmpd restart - - Note: net-snmp-config is used to see default configuration - -2. paramiko: - #apt-get install python-paramiko diff --git a/compass-tasks/hdsdiscovery/__init__.py b/compass-tasks/hdsdiscovery/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/hdsdiscovery/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/hdsdiscovery/base.py b/compass-tasks/hdsdiscovery/base.py deleted file mode 100644 index 77b3b0b..0000000 --- a/compass-tasks/hdsdiscovery/base.py +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Base class extended by specific vendor in vendors directory. -A vendor needs to implement abstract methods of base class. -""" -import logging -import re - -from abc import ABCMeta - -from compass.hdsdiscovery.error import TimeoutError -from compass.hdsdiscovery import utils - - -class BaseVendor(object): - """Basic Vendor object.""" - __metaclass__ = ABCMeta - - def is_this_vendor(self, sys_info, **kwargs): - """Determine if the host is associated with this vendor. - - This function must be implemented by vendor itself - """ - raise NotImplementedError - - -class BaseSnmpVendor(BaseVendor): - """Base SNMP-based vendor plugin. - - .. note:: - It uses MIB-II sysDescr value to determine the vendor of the switch. - """ - - def __init__(self, matched_names): - super(BaseSnmpVendor, self).__init__() - self._matched_names = matched_names - - def is_this_vendor(self, sys_info, **kwargs): - """Determine if the switch belongs to this vendor. - - Matching the system information retrieved from the switch. - :param str sys_info: the system information retrieved from a switch - Return True - """ - if sys_info: - for name in self._matched_names: - if re.search(r"\b" + re.escape(name) + r"\b", sys_info, - re.IGNORECASE): - return True - return False - - -class BasePlugin(object): - """Extended by vendor's plugin. - - This plugin processes request and retrieve info directly from the switch. - """ - __metaclass__ = ABCMeta - - def process_data(self, oper='SCAN', **kwargs): - """Each vendors will have some plugins to do some operations. - - Plugin will process request data and return expected result. - - :param oper: operation function name. - :param kwargs: key-value pairs of arguments - """ - raise NotImplementedError - - # At least one of these three functions below must be implemented. - def scan(self, **kwargs): - """Get multiple records at once.""" - pass - - def set(self, **kwargs): - """Set value to desired variable.""" - pass - - def get(self, **kwargs): - """Get one record from a host.""" - pass - - -class BaseSnmpMacPlugin(BasePlugin): - """Base snmp plugin.""" - - def __init__(self, host, credential, oid='BRIDGE-MIB::dot1dTpFdbPort', - vlan_oid='Q-BRIDGE-MIB::dot1qPvid'): - super(BaseSnmpMacPlugin, self).__init__() - self.host = host - self.credential = credential - self.oid = oid - self.port_oid = 'ifName' - self.vlan_oid = vlan_oid - - def process_data(self, oper='SCAN', **kwargs): - """progress data.""" - func_name = oper.lower() - return getattr(self, func_name)(**kwargs) - - def scan(self, **kwargs): - """scan.""" - results = None - try: - results = utils.snmpwalk_by_cl(self.host, self.credential, - self.oid) - except TimeoutError as error: - logging.debug("PluginMac:scan snmpwalk_by_cl failed: %s", - error.message) - return None - - mac_list = [] - for entity in results: - if_index = entity['value'] - if entity and int(if_index): - tmp = {} - mac_numbers = entity['iid'].split('.') - tmp['mac'] = self.get_mac_address(mac_numbers) - tmp['port'] = self.get_port(if_index) - tmp['vlan'] = self.get_vlan_id(if_index) - mac_list.append(tmp) - - return mac_list - - def get_vlan_id(self, port): - """Get vlan Id.""" - if not port: - return None - - oid = '.'.join((self.vlan_oid, port)) - vlan_id = None - result = None - try: - result = utils.snmpget_by_cl(self.host, self.credential, oid) - except TimeoutError as error: - logging.debug("[PluginMac:get_vlan_id snmpget_by_cl failed: %s]", - error.message) - return None - - vlan_id = result.split()[-1] - return vlan_id - - def get_port(self, if_index): - """Get port number.""" - - if_name = '.'.join((self.port_oid, if_index)) - result = None - try: - result = utils.snmpget_by_cl(self.host, self.credential, if_name) - except TimeoutError as error: - logging.debug("[PluginMac:get_port snmpget_by_cl failed: %s]", - error.message) - return None - - # A result may be like "Value: FasterEthernet1/2/34 - port = result.split()[-1].split('/')[-1] - return port - - def convert_to_hex(self, value): - """Convert the integer from decimal to hex.""" - - return "%0.2x" % int(value) - - def get_mac_address(self, mac_numbers): - """Assemble mac address from the list.""" - if len(mac_numbers) != 6: - logging.error("[PluginMac:get_mac_address] MAC address must be " - "6 digitals") - return None - - mac_in_hex = [self.convert_to_hex(num) for num in mac_numbers] - return ":".join(mac_in_hex) diff --git a/compass-tasks/hdsdiscovery/error.py b/compass-tasks/hdsdiscovery/error.py deleted file mode 100644 index 5bcf1a2..0000000 --- a/compass-tasks/hdsdiscovery/error.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""hdsdiscovery module errors.""" - - -class TimeoutError(Exception): - """Timeout error.""" - - def __init__(self, message): - super(TimeoutError, self).__init__(message) - self.message = message - - def __str__(self): - return repr(self.message) diff --git a/compass-tasks/hdsdiscovery/hdmanager.py b/compass-tasks/hdsdiscovery/hdmanager.py deleted file mode 100644 index 028d444..0000000 --- a/compass-tasks/hdsdiscovery/hdmanager.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Manage hdsdiscovery functionalities.""" -import logging -import os -import re - -from compass.hdsdiscovery.error import TimeoutError -from compass.hdsdiscovery import utils -from compass.utils import setting_wrapper as setting -from compass.utils import util - - -UNREACHABLE = 'unreachable' -NOTSUPPORTED = 'notsupported' -ERROR = 'error' -REPOLLING = 'repolling' - - -class HDManager(object): - """Process a request.""" - - def __init__(self): - base_dir = os.path.dirname(os.path.realpath(__file__)) - self.vendors_dir = os.path.join(base_dir, 'vendors') - self.vendor_plugins_dir = os.path.join(self.vendors_dir, '?/plugins') - self.snmp_sysdescr = 'sysDescr.0' - - def learn(self, host, credential, vendor, req_obj, oper="SCAN", **kwargs): - """Insert/update record of switch_info. - - Get expected results from switch according to sepcific operation. - - :param req_obj: the object of a machine - :param host: switch IP address - :param credientials: credientials to access switch - :param oper: operations of the plugin (SCAN, GETONE, SET) - :param kwargs(optional): key-value pairs - """ - plugin_dir = self.vendor_plugins_dir.replace('?', vendor) - if not os.path.exists(plugin_dir): - logging.error('No such directory: %s', plugin_dir) - return None - - plugin = utils.load_module(req_obj, plugin_dir, host, credential) - if not plugin: - # No plugin found! - # TODO(Grace): add more code to catch excpetion or unexpected state - logging.error('no plugin %s to load from %s', req_obj, plugin_dir) - return None - - return plugin.process_data(oper, **kwargs) - - def is_valid_vendor(self, host, credential, vendor): - """Check if vendor is associated with this host and credential - - :param host: switch ip - :param credential: credential to access switch - :param vendor: the vendor of switch - """ - vendor_dir = os.path.join(self.vendors_dir, vendor) - if not os.path.exists(vendor_dir): - logging.error('no such directory: %s', vendor_dir) - return False - - sys_info, err = self.get_sys_info(host, credential) - if not sys_info: - logging.debug("[hdsdiscovery][hdmanager][is_valid_vendor]" - "failded to get sys information: %s", err) - return False - - instance = utils.load_module(vendor, vendor_dir) - if not instance: - logging.debug("[hdsdiscovery][hdmanager][is_valid_vendor]" - "No such vendor found!") - return False - - if instance.is_this_vendor(sys_info): - logging.info("[hdsdiscovery][hdmanager][is_valid_vendor]" - "vendor %s is correct!", vendor) - return True - - return False - - def get_vendor(self, host, credential): - """Check and get vendor of the switch. - - :param host: switch ip: - :param credential: credential to access switch - :return a tuple (vendor, switch_state, error) - """ - - switch_lists = util.load_configs(setting.MACHINE_LIST_DIR) - switch_list = [] - for items in switch_lists: - for item in items['MACHINE_LIST']: - for k, v in item.items(): - switch_list.append(k) - if host in switch_list: - return ("appliance", "Found", "") - - # TODO(grace): Why do we need to have valid IP? - # a hostname should also work. - if not utils.valid_ip_format(host): - logging.error("host '%s' is not valid IP address!", host) - return (None, ERROR, "Invalid IP address %s!" % host) - - if not utils.is_valid_snmp_v2_credential(credential): - logging.debug("******The credential %s of host %s cannot " - "be used for either SNMP v2 or SSH*****", - credential, host) - return (None, ERROR, "Invalid credential") - - sys_info, err = self.get_sys_info(host, credential) - if not sys_info: - return (None, UNREACHABLE, err) - - # List all vendors in vendors directory -- a directory but hidden - # under ../vendors - all_vendors = [o for o in os.listdir(self.vendors_dir) - if os.path.isdir(os.path.join(self.vendors_dir, o)) - and re.match(r'^[^\.]', o)] - - logging.debug("[get_vendor][available vendors]: %s ", all_vendors) - logging.debug("[get_vendor] System Information is [%s]", sys_info) - - # TODO(grace): should not conver to lower. The vendor impl can choose - # to do case-insensitive match - # sys_info = sys_info.lower() - vendor = None - for vname in all_vendors: - vpath = os.path.join(self.vendors_dir, vname) - instance = utils.load_module(vname, vpath) - if not instance: - logging.error('no instance %s load from %s', vname, vpath) - continue - - if instance.is_this_vendor(sys_info): - logging.info("[get_vendor]****Found vendor '%s'****", vname) - vendor = vname - break - - if not vendor: - logging.debug("[get_vendor] No vendor found! <==================") - return (None, NOTSUPPORTED, "Not supported switch vendor!") - - return (vendor, REPOLLING, "") - - def get_sys_info(self, host, credential): - """get sys info.""" - sys_info = None - try: - sys_info = utils.snmpget_by_cl(host, - credential, - self.snmp_sysdescr) - except TimeoutError as error: - return (None, error.message) - - return (sys_info, "") diff --git a/compass-tasks/hdsdiscovery/utils.py b/compass-tasks/hdsdiscovery/utils.py deleted file mode 100644 index 72adb0a..0000000 --- a/compass-tasks/hdsdiscovery/utils.py +++ /dev/null @@ -1,289 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utility functions - Including functions of get/getbulk/walk/set of snmp for three versions -""" -import imp -import logging -import re -import subprocess - -from compass.hdsdiscovery.error import TimeoutError - - -def load_module(mod_name, path, host=None, credential=None): - """Load a module instance. - - :param str mod_name: module name - :param str path: directory of the module - :param str host: switch ip address - :param str credential: credential used to access switch - """ - try: - mod_file, path, descr = imp.find_module(mod_name, [path]) - if mod_file: - mod = imp.load_module(mod_name, mod_file, path, descr) - if host and credential: - instance = getattr(mod, mod.CLASS_NAME)(host, credential) - else: - instance = getattr(mod, mod.CLASS_NAME)() - - return instance - except ImportError as exc: - logging.error('No such module found: %s', mod_name) - logging.exception(exc) - return None - - -def ssh_remote_execute(host, username, password, cmd): - """SSH to execute script on remote machine - - :param host: ip of the remote machine - :param username: username to access the remote machine - :param password: password to access the remote machine - :param cmd: command to execute - """ - try: - import paramiko - if not cmd: - logging.error("[hdsdiscovery][utils][ssh_remote_execute] command" - "is None! Failed!") - return None - - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - client.connect(host, username=username, password=password, timeout=15) - stdin, stdout, stderr = client.exec_command(cmd) - result = stdout.readlines() - return result - - except ImportError as exc: - logging.error("[hdsdiscovery][utils][ssh_remote_execute] failed to" - "load module 'paramiko', donnot exist!") - logging.exception(exc) - return None - - except Exception as exc: - logging.error("[hdsdiscovery][utils][ssh_remote_execute] failed: %s", - cmd) - logging.exception(exc) - return None - - finally: - stdin.close() - stdout.close() - stderr.close() - client.close() - - -def valid_ip_format(ip_address): - """Valid the format of an Ip address.""" - - if not re.match(r'^((([0-2]?\d{0,2}\.){3}([0-2]?\d{0,2}))' - r'|(([\da-fA-F]{1,4}:){7}([\da-fA-F]{1,4})))$', - ip_address): - # check IP's format is match ipv4 or ipv6 by regex - return False - - return True - -################################################################# -# Implement snmpwalk and snmpget funtionality -# The structure of returned dictionary will by tag/iid/value/type -################################################################# -AUTH_VERSIONS = { - '1': 1, - '2c': 2, - '3': 3 -} - - -def snmp_walk(host, credential, *args, **kwargs): - """Impelmentation of snmpwalk functionality - - :param host: switch ip - :param credential: credential to access switch - :param args: OIDs - :param kwargs: key-value pairs - """ - try: - import netsnmp - - except ImportError: - logging.error("Module 'netsnmp' do not exist! Please install it first") - return None - - if 'version' not in credential or 'community' not in credential: - logging.error("[utils] missing 'version' and 'community' in %s", - credential) - return None - - version = None - if credential['version'] in AUTH_VERSIONS: - version = AUTH_VERSIONS[credential['version']] - - varbind_list = [] - for arg in args: - varbind = netsnmp.Varbind(arg) - varbind_list.append(varbind) - - var_list = netsnmp.VarList(*varbind_list) - - netsnmp.snmpwalk(var_list, - DestHost=host, - Version=version, - Community=credential['community'], - **kwargs) - - result = [] - if not var_list: - logging.error("[hsdiscovery][utils][snmp_walk] retrived no record!") - return result - - for var in var_list: - response = {} - response['elem_name'] = var.tag - response['iid'] = var.iid - response['value'] = var.val - response['type'] = var.type - result.append(response) - - return result - - -def snmp_get(host, credential, object_type, **kwargs): - """Impelmentation of snmp get functionality - - :param object_type: mib object - :param host: switch ip - :param credential: the dict of credential to access switch - """ - try: - import netsnmp - - except ImportError: - logging.error("Module 'netsnmp' do not exist! Please install it first") - return None - - if 'version' not in credential or 'community' not in credential: - logging.error('[uitls][snmp_get] missing keywords in %s for %s', - credential, host) - return None - - version = None - if credential['version'] in AUTH_VERSIONS: - version = AUTH_VERSIONS[credential['version']] - - varbind = netsnmp.Varbind(object_type) - res = netsnmp.snmpget(varbind, - DestHost=host, - Version=version, - Community=credential['community'], - **kwargs) - if res and res[0]: - return res[0] - - logging.info('no result found for %s %s', host, credential) - return None - - -SSH_CREDENTIALS = {"username": "", "password": ""} -SNMP_V2_CREDENTIALS = {"version": "", "community": ""} - - -def is_valid_snmp_v2_credential(credential): - """check if credential is valid snmp v2 credential.""" - if credential.keys() != SNMP_V2_CREDENTIALS.keys(): - return False - if credential['version'] != '2c': - logging.error("The value of version in credential is not '2c'!") - return False - return True - - -def is_valid_ssh_credential(credential): - """check if credential is valid ssh credential.""" - if credential.keys() != SSH_CREDENTIALS.keys(): - return False - return True - - -def snmpget_by_cl(host, credential, oid, timeout=8, retries=3): - """snmpget by credential.""" - if not is_valid_snmp_v2_credential(credential): - logging.error("[utils][snmpget_by_cl] Credential %s cannot be used " - "for SNMP request!", credential) - return None - - version = credential['version'] - community = credential['community'] - cmd = "snmpget -v %s -c %s -Ob -r %s -t %s %s %s" % ( - version, community, retries, timeout, host, oid) - - returncode, output, err = exec_command(cmd) - - if returncode and err: - logging.error("[snmpget_by_cl] %s", err) - raise TimeoutError(err.strip('\n')) - - return output.strip('\n') - - -def snmpwalk_by_cl(host, credential, oid, timeout=5, retries=3): - """snmpwalk by credential.""" - if not is_valid_snmp_v2_credential(credential): - logging.error("[utils][snmpwalk_by_cl] Credential %s cannot be used " - "for SNMP request!", credential) - return None - - version = credential['version'] - community = credential['community'] - cmd = "snmpwalk -v %s -c %s -Cc -r %s -t %s -Ob %s %s" % ( - version, community, retries, timeout, host, oid) - - returncode, output, err = exec_command(cmd) - - if returncode and err: - logging.debug("[snmpwalk_by_cl] %s ", err) - raise TimeoutError(err) - - result = [] - if not output: - return result - - output = output.split('\n') - for line in output: - if not line: - continue - temp = {} - arr = line.split(" ") - temp['iid'] = arr[0].split('.', 1)[-1] - temp['value'] = arr[-1] - result.append(temp) - - return result - - -def exec_command(command): - """Execute command. - - Return a tuple: returncode, output and error message(None if no error). - """ - sub_p = subprocess.Popen(command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - output, err_msg = sub_p.communicate() - return (sub_p.returncode, output, err_msg) diff --git a/compass-tasks/hdsdiscovery/vendors/__init__.py b/compass-tasks/hdsdiscovery/vendors/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/hdsdiscovery/vendors/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/hdsdiscovery/vendors/appliance/__init__.py b/compass-tasks/hdsdiscovery/vendors/appliance/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compass-tasks/hdsdiscovery/vendors/appliance/appliance.py b/compass-tasks/hdsdiscovery/vendors/appliance/appliance.py deleted file mode 100644 index 3d66f4e..0000000 --- a/compass-tasks/hdsdiscovery/vendors/appliance/appliance.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Vendor: Compass Appliance""" -from compass.hdsdiscovery import base - - -# Vendor_loader will load vendor instance by CLASS_NAME -CLASS_NAME = 'Appliance' - - -class Appliance(base.BaseSnmpVendor): - """Fake SNMP object for compass appliance.""" - - def __init__(self): - base.BaseSnmpVendor.__init__(self, ['appliance']) - self.__name = 'appliance' - - @property - def name(self): - """Get 'name' proptery.""" - return self.__name diff --git a/compass-tasks/hdsdiscovery/vendors/appliance/plugins/__init__.py b/compass-tasks/hdsdiscovery/vendors/appliance/plugins/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compass-tasks/hdsdiscovery/vendors/appliance/plugins/mac.py b/compass-tasks/hdsdiscovery/vendors/appliance/plugins/mac.py deleted file mode 100644 index cc14881..0000000 --- a/compass-tasks/hdsdiscovery/vendors/appliance/plugins/mac.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Compass Appliance Mac module.""" -from compass.hdsdiscovery import base -from compass.utils import setting_wrapper as setting -from compass.utils import util - -import logging - - -CLASS_NAME = "Mac" - - -class Mac(base.BaseSnmpMacPlugin): - """Processes MAC address.""" - - def __init__(self, host, credential): - self.host = host - # self.credential = credential - # return - - def scan(self): - """Implemnets the scan method in BasePlugin class. - - .. note:: - Dummy scan function for compass appliance. - Returns fixed mac addresses. - """ - mac_list = None - machine_lists = util.load_configs(setting.MACHINE_LIST_DIR) - for items in machine_lists: - for item in items['MACHINE_LIST']: - for k, v in item.items(): - if k == self.host: - mac_list = v - return mac_list diff --git a/compass-tasks/hdsdiscovery/vendors/arista/__init__.py b/compass-tasks/hdsdiscovery/vendors/arista/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/hdsdiscovery/vendors/arista/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/hdsdiscovery/vendors/arista/arista.py b/compass-tasks/hdsdiscovery/vendors/arista/arista.py deleted file mode 100644 index 5eacea1..0000000 --- a/compass-tasks/hdsdiscovery/vendors/arista/arista.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Vendor: Arista.""" -from compass.hdsdiscovery import base - - -# Vendor_loader will load vendor instance by CLASS_NAME -CLASS_NAME = 'Arista' - - -class Arista(base.BaseSnmpVendor): - """Arista Network switch object.""" - - def __init__(self): - base.BaseSnmpVendor.__init__(self, ['arista']) - self._name = 'arista' - - @property - def name(self): - """Get 'name' proptery.""" - return self._name diff --git a/compass-tasks/hdsdiscovery/vendors/arista/plugins/__init__.py b/compass-tasks/hdsdiscovery/vendors/arista/plugins/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/hdsdiscovery/vendors/arista/plugins/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/hdsdiscovery/vendors/arista/plugins/mac.py b/compass-tasks/hdsdiscovery/vendors/arista/plugins/mac.py deleted file mode 100644 index ed2f331..0000000 --- a/compass-tasks/hdsdiscovery/vendors/arista/plugins/mac.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Pica8 Switch Mac module.""" -from compass.hdsdiscovery.base import BaseSnmpMacPlugin - - -CLASS_NAME = 'Mac' - - -class Mac(BaseSnmpMacPlugin): - """Process MAC address by Arista switch.""" - pass diff --git a/compass-tasks/hdsdiscovery/vendors/hp/__init__.py b/compass-tasks/hdsdiscovery/vendors/hp/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/hdsdiscovery/vendors/hp/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/hdsdiscovery/vendors/hp/hp.py b/compass-tasks/hdsdiscovery/vendors/hp/hp.py deleted file mode 100644 index 7ddc5e9..0000000 --- a/compass-tasks/hdsdiscovery/vendors/hp/hp.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Vendor: HP.""" -from compass.hdsdiscovery import base - - -# Vendor_loader will load vendor instance by CLASS_NAME -CLASS_NAME = 'Hp' - - -class Hp(base.BaseSnmpVendor): - """Hp switch object.""" - - def __init__(self): - base.BaseSnmpVendor.__init__(self, ['hp', 'procurve']) - self.names = ['hp', 'procurve'] - - @property - def name(self): - """Get 'name' proptery.""" - return self.names[0] diff --git a/compass-tasks/hdsdiscovery/vendors/hp/plugins/__init__.py b/compass-tasks/hdsdiscovery/vendors/hp/plugins/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/hdsdiscovery/vendors/hp/plugins/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/hdsdiscovery/vendors/hp/plugins/mac.py b/compass-tasks/hdsdiscovery/vendors/hp/plugins/mac.py deleted file mode 100644 index 3bc81f4..0000000 --- a/compass-tasks/hdsdiscovery/vendors/hp/plugins/mac.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""HP Switch Mac module.""" -from compass.hdsdiscovery.base import BaseSnmpMacPlugin - -CLASS_NAME = 'Mac' - - -class Mac(BaseSnmpMacPlugin): - """Process MAC address by HP switch.""" - pass diff --git a/compass-tasks/hdsdiscovery/vendors/huawei/__init__.py b/compass-tasks/hdsdiscovery/vendors/huawei/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/hdsdiscovery/vendors/huawei/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/hdsdiscovery/vendors/huawei/huawei.py b/compass-tasks/hdsdiscovery/vendors/huawei/huawei.py deleted file mode 100644 index 19fd043..0000000 --- a/compass-tasks/hdsdiscovery/vendors/huawei/huawei.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Huawei Switch.""" -from compass.hdsdiscovery import base - - -# Vendor_loader will load vendor instance by CLASS_NAME -CLASS_NAME = "Huawei" - - -class Huawei(base.BaseSnmpVendor): - """Huawei switch.""" - - def __init__(self): - base.BaseSnmpVendor.__init__(self, ["huawei"]) - self.__name = "huawei" - - @property - def name(self): - """Return switch name.""" - return self.__name diff --git a/compass-tasks/hdsdiscovery/vendors/huawei/plugins/__init__.py b/compass-tasks/hdsdiscovery/vendors/huawei/plugins/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/hdsdiscovery/vendors/huawei/plugins/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/hdsdiscovery/vendors/huawei/plugins/mac.py b/compass-tasks/hdsdiscovery/vendors/huawei/plugins/mac.py deleted file mode 100644 index 49d3863..0000000 --- a/compass-tasks/hdsdiscovery/vendors/huawei/plugins/mac.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Huawei Switch Mac module.""" -import logging - -from compass.hdsdiscovery.base import BaseSnmpMacPlugin -from compass.hdsdiscovery import utils - - -CLASS_NAME = "Mac" - - -class Mac(BaseSnmpMacPlugin): - """Processes MAC address.""" - - def __init__(self, host, credential): - super(Mac, self).__init__( - host, credential, - 'HUAWEI-L2MAM-MIB::hwDynFdbPort') - - def scan(self): - """Implemnets the scan method in BasePlugin class. - - .. note:: - In this mac module, mac addesses were retrieved by - snmpwalk commandline. - """ - results = utils.snmpwalk_by_cl(self.host, self.credential, self.oid) - - if not results: - logging.info("[Huawei][mac] No results returned from SNMP walk!") - return None - - mac_list = [] - - for entity in results: - # The format of 'iid' is like '248.192.1.214.34.15.31.1.48' - # The first 6 numbers will be the MAC address - # The 7th number is its vlan ID - numbers = entity['iid'].split('.') - mac = self.get_mac_address(numbers[:6]) - vlan = numbers[6] - port = self.get_port(entity['value']) - - tmp = {} - tmp['port'] = port - tmp['mac'] = mac - tmp['vlan'] = vlan - mac_list.append(tmp) - - return mac_list diff --git a/compass-tasks/hdsdiscovery/vendors/ovswitch/__init__.py b/compass-tasks/hdsdiscovery/vendors/ovswitch/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/hdsdiscovery/vendors/ovswitch/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/hdsdiscovery/vendors/ovswitch/ovswitch.py b/compass-tasks/hdsdiscovery/vendors/ovswitch/ovswitch.py deleted file mode 100644 index 4d03328..0000000 --- a/compass-tasks/hdsdiscovery/vendors/ovswitch/ovswitch.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Open Vswitch module.""" -import logging -import re - -from compass.hdsdiscovery import base -from compass.hdsdiscovery import utils - - -# Vendor_loader will load vendor instance by CLASS_NAME -CLASS_NAME = "OVSwitch" - - -class OVSwitch(base.BaseVendor): - """Open Vswitch.""" - def __init__(self): - self.__name = "Open vSwitch" - - def is_this_vendor(self, sys_info, host=None, credential=None, **kwargs): - """Determine if the hostname is accociated witH this vendor. - - :param host: swtich's IP address - :param credential: credential to access switch - """ - result = sys_info - if host and credential: - if utils.is_valid_ssh_credential(credential): - user = credential['username'] - pwd = credential['password'] - - else: - msg = ("[OVSwitch]The format of credential %r is not for SSH " - "or incorrect Keywords! " % credential) - logging.info(msg) - return False - - cmd = "ovs-vsctl -V" - result = None - try: - result = utils.ssh_remote_execute(host, user, pwd, cmd) - logging.debug('%s result for %s is %s', cmd, host, result) - if not result: - return False - except Exception as exc: - logging.error("No vendor or connection failed to run %s", cmd) - logging.exception(exc) - return False - - if isinstance(result, str): - result = [result] - - for line in result: - if not line: - continue - if re.search(r"\b" + re.escape(self.__name) + r"\b", line): - return True - - return False - - @property - def name(self): - """Open Vswitch name.""" - return self.__name diff --git a/compass-tasks/hdsdiscovery/vendors/ovswitch/plugins/__init__.py b/compass-tasks/hdsdiscovery/vendors/ovswitch/plugins/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/hdsdiscovery/vendors/ovswitch/plugins/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/hdsdiscovery/vendors/ovswitch/plugins/mac.py b/compass-tasks/hdsdiscovery/vendors/ovswitch/plugins/mac.py deleted file mode 100644 index 5f497a0..0000000 --- a/compass-tasks/hdsdiscovery/vendors/ovswitch/plugins/mac.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Open Vswitch Mac address module.""" -import logging - -from compass.hdsdiscovery import base -from compass.hdsdiscovery import utils - - -CLASS_NAME = "Mac" - - -class Mac(base.BasePlugin): - """Open Vswitch MAC address module.""" - def __init__(self, host, credential): - self.host = host - self.credential = credential - - def process_data(self, oper="SCAN", **kwargs): - """Dynamically call the function according 'oper' - - :param oper: operation of data processing - """ - func_name = oper.lower() - return getattr(self, func_name)(**kwargs) - - def scan(self, **kwargs): - """Implemnets the scan method in BasePlugin class. - - .. note:: - In this module, mac addesses were retrieved by ssh. - """ - try: - user = self.credential['username'] - pwd = self.credential['password'] - except KeyError: - logging.error("Cannot find username and password in credential") - return None - - cmd = ("BRIDGES=$(ovs-vsctl show |grep Bridge |cut -f 2 -d '\"');" - "for br in $BRIDGES; do" - "PORTS=$(ovs-ofctl show $br |grep addr |cut -f 1 -d ':' " - "|egrep -v 'eth|wlan|LOCAL'|awk -F '(' '{print $1}');" - "for port in $PORTS; do" - "RESULT=$(ovs-appctl fdb/show $br |" - "awk '$1 == '$port' {print $1" "$2" "$3}');" - "echo '$RESULT'" - "done;" - "done;") - output = None - try: - output = utils.ssh_remote_execute(self.host, user, pwd, cmd) - except Exception as error: - logging.exception(error) - return None - - logging.debug("[scan][output] output is %s", output) - if not output: - return None - - fields_arr = ['port', 'vlan', 'mac'] - - result = [] - for line in output: - if not line or line == '\n': - continue - - values_arr = line.split() - temp = {} - for field, value in zip(fields_arr, values_arr): - temp[field] = value - - result.append(temp.copy()) - - return result diff --git a/compass-tasks/hdsdiscovery/vendors/pica8/__init__.py b/compass-tasks/hdsdiscovery/vendors/pica8/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/hdsdiscovery/vendors/pica8/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/hdsdiscovery/vendors/pica8/pica8.py b/compass-tasks/hdsdiscovery/vendors/pica8/pica8.py deleted file mode 100644 index 856817e..0000000 --- a/compass-tasks/hdsdiscovery/vendors/pica8/pica8.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Vendor: Pica8.""" -from compass.hdsdiscovery import base - - -# Vendor_loader will load vendor instance by CLASS_NAME -CLASS_NAME = 'Pica8' - - -class Pica8(base.BaseSnmpVendor): - """Pica8 switch object.""" - - def __init__(self): - base.BaseSnmpVendor.__init__(self, ['pica8']) - self._name = 'pica8' - - @property - def name(self): - """Get 'name' proptery.""" - return self._name diff --git a/compass-tasks/hdsdiscovery/vendors/pica8/plugins/__init__.py b/compass-tasks/hdsdiscovery/vendors/pica8/plugins/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/hdsdiscovery/vendors/pica8/plugins/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/hdsdiscovery/vendors/pica8/plugins/mac.py b/compass-tasks/hdsdiscovery/vendors/pica8/plugins/mac.py deleted file mode 100644 index d5ccfc0..0000000 --- a/compass-tasks/hdsdiscovery/vendors/pica8/plugins/mac.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Pica8 Switch Mac module.""" -from compass.hdsdiscovery.base import BaseSnmpMacPlugin - - -CLASS_NAME = 'Mac' - - -class Mac(BaseSnmpMacPlugin): - """Process MAC address by Pica8 switch.""" - pass diff --git a/compass-tasks/log_analyzor/__init__.py b/compass-tasks/log_analyzor/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/log_analyzor/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/log_analyzor/adapter_matcher.py b/compass-tasks/log_analyzor/adapter_matcher.py deleted file mode 100644 index 8630e01..0000000 --- a/compass-tasks/log_analyzor/adapter_matcher.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to provider installing progress calculation for the adapter. - - .. moduleauthor:: Xiaodong Wang -""" -import logging -import re - - -class AdapterItemMatcher(object): - """Progress matcher for the os installing or package installing.""" - - def __init__(self, file_matchers): - self.file_matchers_ = file_matchers - - def __repr__(self): - return '%r[file_matchers: %r]' % ( - self.__class__.__name__, self.file_matchers_ - ) - - def update_progress( - self, file_reader_factory, name, state, log_history_mapping - ): - """Update progress. - - :param name: the fullname of the installing host. - :type name: str - :param progress: Progress instance to update. - """ - for file_matcher in self.file_matchers_: - filename = file_matcher.filename_ - if filename not in log_history_mapping: - log_history_mapping[filename] = { - 'filename': filename, - 'partial_line': '', - 'position': 0, - 'line_matcher_name': 'start', - 'percentage': 0.0, - 'message': '', - 'severity': 'INFO' - } - log_history = log_history_mapping[filename] - file_matcher.update_progress( - file_reader_factory, name, state, log_history - ) - - -class OSMatcher(object): - """Progress matcher for os installer.""" - - def __init__( - self, os_installer_name, - os_pattern, item_matcher, - file_reader_factory - ): - self.name_ = re.compile(os_installer_name) - self.os_regex_ = re.compile(os_pattern) - self.matcher_ = item_matcher - self.file_reader_factory_ = file_reader_factory - - def __repr__(self): - return '%r[name:%r, os_pattern:%r, matcher:%r]' % ( - self.__class__.__name__, self.name_.pattern, - self.os_regex_.pattern, self.matcher_) - - def match(self, os_installer_name, os_name): - """Check if the os matcher is acceptable.""" - if os_name is None: - return False - else: - return all([ - self.name_.match(os_installer_name), - self.os_regex_.match(os_name) - ]) - - def update_progress(self, name, state, log_history_mapping): - """Update progress.""" - self.matcher_.update_progress( - self.file_reader_factory_, name, state, log_history_mapping) - - -class PackageMatcher(object): - """Progress matcher for package installer.""" - - def __init__( - self, package_installer_name, adapter_pattern, - item_matcher, file_reader_factory - ): - self.name_ = re.compile(package_installer_name) - self.adapter_regex_ = re.compile(adapter_pattern) - self.matcher_ = item_matcher - self.file_reader_factory_ = file_reader_factory - - def __repr__(self): - return '%s[name:%s, adapter_pattern:%s, matcher:%s]' % ( - self.__class__.__name__, self.name_.pattern, - self.adapter_regex_.pattern, self.matcher_) - - def match(self, package_installer_name, adapter_name): - """Check if the package matcher is acceptable.""" - if package_installer_name is None: - return False - else: - return all([ - self.name_.match(package_installer_name), - self.adapter_regex_.match(adapter_name) - ]) - - def update_progress(self, name, state, log_history_mapping): - """Update progress.""" - self.matcher_.update_progress( - self.file_reader_factory_, name, state, log_history_mapping - ) diff --git a/compass-tasks/log_analyzor/environment.py b/compass-tasks/log_analyzor/environment.py deleted file mode 100644 index 80ff738..0000000 --- a/compass-tasks/log_analyzor/environment.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""module to provide environment to load progress calculator configurations. - - .. moduleauthor:: Xiaodong Wang -""" -from compass.log_analyzor.adapter_matcher import AdapterItemMatcher -from compass.log_analyzor.file_matcher import FileMatcher -from compass.log_analyzor.file_matcher import FileReaderFactory -from compass.log_analyzor.line_matcher import IncrementalProgress -from compass.log_analyzor.line_matcher import LineMatcher -from compass.utils import setting_wrapper as setting -from compass.utils import util - - -ENV_GLOBALS = globals() -ENV_LOCALS = locals() diff --git a/compass-tasks/log_analyzor/file_matcher.py b/compass-tasks/log_analyzor/file_matcher.py deleted file mode 100644 index be3143b..0000000 --- a/compass-tasks/log_analyzor/file_matcher.py +++ /dev/null @@ -1,252 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to update intalling progress by processing log file. - - .. moduleauthor:: Xiaodong Wang -""" -import logging -import os.path - -from compass.utils import setting_wrapper as setting - - -class FileFilter(object): - """base class to filter log file.""" - def __repr__(self): - return self.__class__.__name__ - - def filter(self, pathname): - """Filter log file. - - :param pathname: the absolute path name to the log file. - """ - raise NotImplementedError(str(self)) - - -class CompositeFileFilter(FileFilter): - """filter log file based on the list of filters.""" - def __init__(self, filters): - self.filters_ = filters - - def __str__(self): - return 'CompositeFileFilter[%s]' % self.filters_ - - def append_filter(self, file_filter): - """append filter.""" - self.filters_.append(file_filter) - - def filter(self, pathname): - """filter log file.""" - for file_filter in self.filters_: - if not file_filter.filter(pathname): - return False - - return True - - -class FilterFileExist(FileFilter): - """filter log file if not exists.""" - def filter(self, pathname): - """filter log file.""" - file_exist = os.path.isfile(pathname) - if not file_exist: - logging.debug("%s is not exist", pathname) - - return file_exist - - -def get_file_filter(): - """get file filter""" - composite_filter = CompositeFileFilter([FilterFileExist()]) - return composite_filter - - -class FileReader(object): - """Class to read log file. - - The class provide support to read log file from the position - it has read last time. and update the position when it finish - reading the log. - """ - def __init__(self, pathname, log_history): - self.pathname_ = pathname - self.log_history_ = log_history - - def __repr__(self): - return ( - '%s[pathname:%s, log_history:%s]' % ( - self.__class__.__name__, self.pathname_, - self.log_history_ - ) - ) - - def readline(self): - """Generate each line of the log file.""" - old_position = self.log_history_['position'] - position = self.log_history_['position'] - partial_line = self.log_history_['partial_line'] - try: - with open(self.pathname_) as logfile: - logfile.seek(position) - while True: - line = logfile.readline() - partial_line += line - position = logfile.tell() - if position > self.log_history_['position']: - self.log_history_['position'] = position - - if partial_line.endswith('\n'): - self.log_history_['partial_line'] = '' - yield partial_line - partial_line = self.log_history_['partial_line'] - else: - self.log_history_['partial_line'] = partial_line - break - if partial_line: - yield partial_line - - except Exception as error: - logging.error('failed to processing file %s', self.pathname_) - raise error - - logging.debug( - 'processing file %s log %s bytes to position %s', - self.pathname_, position - old_position, position - ) - - -class FileReaderFactory(object): - """factory class to create FileReader instance.""" - - def __init__(self, logdir): - self.logdir_ = logdir - self.filefilter_ = get_file_filter() - - def __str__(self): - return '%s[logdir: %s filefilter: %s]' % ( - self.__class__.__name__, self.logdir_, self.filefilter_) - - def get_file_reader(self, hostname, filename, log_history): - """Get FileReader instance. - - :param fullname: fullname of installing host. - :param filename: the filename of the log file. - - :returns: :class:`FileReader` instance if it is not filtered. - """ - pathname = os.path.join(self.logdir_, hostname, filename) - logging.debug('get FileReader from %s', pathname) - if not self.filefilter_.filter(pathname): - logging.debug('%s is filtered', pathname) - return None - - return FileReader(pathname, log_history) - - -class FileMatcher(object): - """File matcher to get the installing progress from the log file.""" - def __init__(self, line_matchers, min_progress, max_progress, filename): - if not 0.0 <= min_progress <= max_progress <= 1.0: - raise IndexError( - '%s restriction is not mat: 0.0 <= min_progress' - '(%s) <= max_progress(%s) <= 1.0' % ( - self.__class__.__name__, - min_progress, - max_progress)) - if 'start' not in line_matchers: - raise KeyError( - 'key `start` does not in line matchers %s' % line_matchers - ) - self.line_matchers_ = line_matchers - self.min_progress_ = min_progress - self.max_progress_ = max_progress - self.progress_diff_ = max_progress - min_progress - self.filename_ = filename - - def __repr__(self): - return ( - '%r[filename: %r, progress:[%r:%r], ' - 'line_matchers: %r]' % ( - self.__class__.__name__, self.filename_, - self.min_progress_, - self.max_progress_, self.line_matchers_) - ) - - def update_progress_from_log_history(self, state, log_history): - file_percentage = log_history['percentage'] - percentage = max( - self.min_progress_, - min( - self.max_progress_, - self.min_progress_ + file_percentage * self.progress_diff_ - ) - ) - if ( - percentage > state['percentage'] or - ( - percentage == state['percentage'] and - log_history['message'] != state['message'] - ) - ): - state['percentage'] = percentage - state['message'] = log_history['message'] - state['severity'] = log_history['severity'] - else: - logging.debug( - 'ingore update state %s from log history %s ' - 'since the updated progress %s lag behind', - state, log_history, percentage - ) - - def update_progress(self, file_reader_factory, name, state, log_history): - """update progress from file. - - :param fullname: the fullname of the installing host. - :type fullname: str - :param total_progress: Progress instance to update. - - the function update installing progress by reading the log file. - It contains a list of line matcher, when one log line matches - with current line matcher, the installing progress is updated. - and the current line matcher got updated. - Notes: some line may be processed multi times. The case is the - last line of log file is processed in one run, while in the other - run, it will be reprocessed at the beginning because there is - no line end indicator for the last line of the file. - """ - file_reader = file_reader_factory.get_file_reader( - name, self.filename_, log_history) - if not file_reader: - return - - line_matcher_name = log_history['line_matcher_name'] - for line in file_reader.readline(): - if line_matcher_name not in self.line_matchers_: - logging.debug('early exit at\n%s\nbecause %s is not in %s', - line, line_matcher_name, self.line_matchers_) - break - - same_line_matcher_name = line_matcher_name - while same_line_matcher_name in self.line_matchers_: - line_matcher = self.line_matchers_[same_line_matcher_name] - same_line_matcher_name, line_matcher_name = ( - line_matcher.update_progress(line, log_history) - ) - log_history['line_matcher_name'] = line_matcher_name - logging.debug( - 'updated log history %s after processing %s', - log_history, self - ) - self.update_progress_from_log_history(state, log_history) diff --git a/compass-tasks/log_analyzor/line_matcher.py b/compass-tasks/log_analyzor/line_matcher.py deleted file mode 100644 index ada9ed6..0000000 --- a/compass-tasks/log_analyzor/line_matcher.py +++ /dev/null @@ -1,206 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to get the progress when found match with a line of the log.""" -import logging -import re - -from abc import ABCMeta - -from compass.utils import util - - -class ProgressCalculator(object): - """base class to generate progress.""" - - __metaclass__ = ABCMeta - - @classmethod - def update_progress( - cls, progress_data, message, - severity, log_history - ): - """Update progress with the given progress_data, message and severity. - - :param progress_data: installing progress. - :type progress_data: float between 0 to 1. - :param message: installing progress message. - :param severity: installing message severity. - :param progress: :class:`Progress` instance to update - """ - # the progress is only updated when the new progress - # is greater than the stored progress or the progress - # to update is the same but the message is different. - if ( - progress_data > log_history['percentage'] or ( - progress_data == log_history['percentage'] and - message != log_history['message'] - ) - ): - log_history['percentage'] = progress_data - if message: - log_history['message'] = message - if severity: - log_history['severity'] = severity - logging.debug('update progress to %s', log_history) - else: - logging.debug('ignore update progress %s to %s', - progress_data, log_history) - - def update(self, message, severity, log_history): - """vritual method to update progress by message and severity. - - :param message: installing message. - :param severity: installing severity. - """ - raise NotImplementedError(str(self)) - - def __repr__(self): - return self.__class__.__name__ - - -class IncrementalProgress(ProgressCalculator): - """Class to increment the progress.""" - - def __init__(self, min_progress, - max_progress, incremental_ratio): - super(IncrementalProgress, self).__init__() - if not 0.0 <= min_progress <= max_progress <= 1.0: - raise IndexError( - '%s restriction is not mat: 0.0 <= min_progress(%s)' - ' <= max_progress(%s) <= 1.0' % ( - self.__class__.__name__, min_progress, max_progress)) - - if not 0.0 <= incremental_ratio <= 1.0: - raise IndexError( - '%s restriction is not mat: ' - '0.0 <= incremental_ratio(%s) <= 1.0' % ( - self.__class__.__name__, incremental_ratio)) - - self.min_progress_ = min_progress - self.max_progress_ = max_progress - self.incremental_progress_ = ( - incremental_ratio * (max_progress - min_progress)) - - def __str__(self): - return '%s[%s:%s:%s]' % ( - self.__class__.__name__, - self.min_progress_, - self.max_progress_, - self.incremental_progress_ - ) - - def update(self, message, severity, log_history): - """update progress from message and severity.""" - progress_data = max( - self.min_progress_, - min( - self.max_progress_, - log_history['percentage'] + self.incremental_progress_ - ) - ) - self.update_progress(progress_data, - message, severity, log_history) - - -class RelativeProgress(ProgressCalculator): - """class to update progress to the given relative progress.""" - - def __init__(self, progress): - super(RelativeProgress, self).__init__() - if not 0.0 <= progress <= 1.0: - raise IndexError( - '%s restriction is not mat: 0.0 <= progress(%s) <= 1.0' % ( - self.__class__.__name__, progress)) - - self.progress_ = progress - - def __str__(self): - return '%s[%s]' % (self.__class__.__name__, self.progress_) - - def update(self, message, severity, log_history): - """update progress from message and severity.""" - self.update_progress( - self.progress_, message, severity, log_history) - - -class SameProgress(ProgressCalculator): - """class to update message and severity for progress.""" - - def update(self, message, severity, log_history): - """update progress from the message and severity.""" - self.update_progress(log_history['percentage'], message, - severity, log_history) - - -class LineMatcher(object): - """Progress matcher for each line.""" - - def __init__(self, pattern, progress=None, - message_template='', severity=None, - unmatch_sameline_next_matcher_name='', - unmatch_nextline_next_matcher_name='', - match_sameline_next_matcher_name='', - match_nextline_next_matcher_name=''): - self.regex_ = re.compile(pattern) - if not progress: - self.progress_ = SameProgress() - elif isinstance(progress, ProgressCalculator): - self.progress_ = progress - elif isinstance(progress, (int, long, float)): - self.progress_ = RelativeProgress(progress) - else: - raise TypeError( - 'progress unsupport type %s: %s' % ( - type(progress), progress)) - - self.message_template_ = message_template - self.severity_ = severity - self.unmatch_sameline_ = unmatch_sameline_next_matcher_name - self.unmatch_nextline_ = unmatch_nextline_next_matcher_name - self.match_sameline_ = match_sameline_next_matcher_name - self.match_nextline_ = match_nextline_next_matcher_name - - def __repr__(self): - return '%r[pattern:%r, message_template:%r, severity:%r]' % ( - self.__class__.__name__, self.regex_.pattern, - self.message_template_, self.severity_) - - def update_progress(self, line, log_history): - """Update progress by the line. - - :param line: one line in log file to indicate the installing progress. - .. note:: - The line may be partial if the latest line of the log file is - not the whole line. But the whole line may be resent - in the next run. - :param progress: the :class:`Progress` instance to update. - """ - mat = self.regex_.search(line) - if not mat: - return ( - self.unmatch_sameline_, - self.unmatch_nextline_) - - try: - message = self.message_template_ % mat.groupdict() - except Exception as error: - logging.error('failed to get message %s %% %s in line matcher %s', - self.message_template_, mat.groupdict(), self) - raise error - - self.progress_.update(message, self.severity_, log_history) - return ( - self.match_sameline_, - self.match_nextline_) diff --git a/compass-tasks/log_analyzor/progress_calculator.py b/compass-tasks/log_analyzor/progress_calculator.py deleted file mode 100644 index b0f35f2..0000000 --- a/compass-tasks/log_analyzor/progress_calculator.py +++ /dev/null @@ -1,208 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""module to provide updating installing process function. - - .. moduleauthor:: Xiaodong Wang -""" -import logging - -from compass.log_analyzor.adapter_matcher import OSMatcher -from compass.log_analyzor.adapter_matcher import PackageMatcher -from compass.log_analyzor.environment import ENV_GLOBALS -from compass.log_analyzor.environment import ENV_LOCALS -from compass.log_analyzor.file_matcher import FileReaderFactory - -from compass.utils import setting_wrapper as setting -from compass.utils import util - -OS_ADAPTER_CONFIGURATIONS = None -PACKAGE_ADAPTER_CONFIGURATIONS = None -PROGRESS_CALCULATOR_CONFIGURATIONS = None - - -def _load_calculator_configurations(force=False): - global PROGRESS_CALCULATOR_CONFIGURATIONS - if force or PROGRESS_CALCULATOR_CONFIGURATIONS is None: - env_locals = {} - env_locals.update(ENV_GLOBALS) - env_locals.update(ENV_LOCALS) - PROGRESS_CALCULATOR_CONFIGURATIONS = util.load_configs( - setting.PROGRESS_CALCULATOR_DIR, - env_locals=env_locals - ) - if not PROGRESS_CALCULATOR_CONFIGURATIONS: - logging.info('No configuration found for progress calculator.') - - global OS_ADAPTER_CONFIGURATIONS - if force or OS_ADAPTER_CONFIGURATIONS is None: - OS_ADAPTER_CONFIGURATIONS = [] - for progress_calculator_configuration in ( - PROGRESS_CALCULATOR_CONFIGURATIONS - ): - if 'OS_LOG_CONFIGURATIONS' in ( - progress_calculator_configuration - ): - os_installer_configurations = ( - progress_calculator_configuration['OS_LOG_CONFIGURATIONS'] - ) - for os_installer_configuration in os_installer_configurations: - OS_ADAPTER_CONFIGURATIONS.append(OSMatcher( - os_installer_name=( - os_installer_configuration['os_installer_name'] - ), - os_pattern=os_installer_configuration['os_pattern'], - item_matcher=( - os_installer_configuration['item_matcher'] - ), - file_reader_factory=FileReaderFactory( - os_installer_configuration['logdir'] - ) - )) - if not OS_ADAPTER_CONFIGURATIONS: - logging.info( - 'no OS_LOG_CONFIGURATIONS section found ' - 'in progress calculator.' - ) - else: - logging.debug( - 'OS_ADAPTER_CONFIGURATIONS is\n%s', - OS_ADAPTER_CONFIGURATIONS - ) - - global PACKAGE_ADAPTER_CONFIGURATIONS - if force or PACKAGE_ADAPTER_CONFIGURATIONS is None: - PACKAGE_ADAPTER_CONFIGURATIONS = [] - for progress_calculator_configuration in ( - PROGRESS_CALCULATOR_CONFIGURATIONS - ): - if 'ADAPTER_LOG_CONFIGURATIONS' in ( - progress_calculator_configuration - ): - package_installer_configurations = ( - progress_calculator_configuration[ - 'ADAPTER_LOG_CONFIGURATIONS' - ] - ) - for package_installer_configuration in ( - package_installer_configurations - ): - PACKAGE_ADAPTER_CONFIGURATIONS.append(PackageMatcher( - package_installer_name=( - package_installer_configuration[ - 'package_installer_name' - ] - ), - adapter_pattern=( - package_installer_configuration['adapter_pattern'] - ), - item_matcher=( - package_installer_configuration['item_matcher'] - ), - file_reader_factory=FileReaderFactory( - package_installer_configuration['logdir'] - ) - )) - if not PACKAGE_ADAPTER_CONFIGURATIONS: - logging.info( - 'no PACKAGE_LOG_CONFIGURATIONS section found ' - 'in progress calculator.' - ) - else: - logging.debug( - 'PACKAGE_ADAPTER_CONFIGURATIONS is\n%s', - PACKAGE_ADAPTER_CONFIGURATIONS - ) - - -def load_calculator_configurations(force_reload=False): - _load_calculator_configurations(force=force_reload) - - -def _get_os_matcher(os_installer_name, os_name): - """Get OS adapter matcher by os name and installer name.""" - _load_calculator_configurations() - for configuration in OS_ADAPTER_CONFIGURATIONS: - if configuration.match(os_installer_name, os_name): - return configuration - else: - logging.debug('configuration %s does not match %s and %s', - configuration, os_name, os_installer_name) - logging.error('No configuration found for os installer %s os %s', - os_installer_name, os_name) - return None - - -def _get_package_matcher( - package_installer_name, adapter_name -): - """Get package adapter matcher by adapter name and installer name.""" - _load_calculator_configurations() - for configuration in PACKAGE_ADAPTER_CONFIGURATIONS: - if configuration.match( - package_installer_name, - adapter_name - ): - return configuration - else: - logging.debug('configuration %s does not match %s and %s', - configuration, adapter_name, - package_installer_name) - logging.error('No configuration found for package installer %s adapter %s', - package_installer_name, adapter_name) - return None - - -def update_host_progress(host_mappping): - for host_id, (host, host_state, host_log_history_mapping) in ( - host_mappping.items() - ): - os_name = host['os_name'] - os_installer_name = host['os_installer']['name'] - os_matcher = _get_os_matcher( - os_installer_name, os_name - ) - if not os_matcher: - continue - name = host[setting.HOST_INSTALLATION_LOGDIR_NAME] - os_matcher.update_progress( - name, host_state, host_log_history_mapping - ) - - -def update_clusterhost_progress(clusterhost_mapping): - for ( - clusterhost_id, - (clusterhost, clusterhost_state, clusterhost_log_history_mapping) - ) in ( - clusterhost_mapping.items() - ): - adapter_name = clusterhost['adapter_name'] - package_installer_name = clusterhost['package_installer']['name'] - package_matcher = _get_package_matcher( - package_installer_name, - adapter_name - ) - if not package_matcher: - continue - name = clusterhost[setting.CLUSTERHOST_INATALLATION_LOGDIR_NAME] - package_matcher.update_progress( - name, clusterhost_state, - clusterhost_log_history_mapping - ) - - -def update_cluster_progress(cluster_mapping): - for cluster_id, (cluster, cluster_state) in cluster_mapping.items(): - pass diff --git a/compass-tasks/misc/Dockerfile b/compass-tasks/misc/Dockerfile deleted file mode 100644 index ed47aea..0000000 --- a/compass-tasks/misc/Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -from centos:latest - -# repos -COPY misc/compass_install.repo /etc/yum.repos.d/compass_install.repo - -RUN rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \ - sed -i 's/^mirrorlist=https/mirrorlist=http/g' /etc/yum.repos.d/epel.repo && \ - yum update -y - -# packages -RUN yum --enablerepo=compass_install --nogpgcheck install -y python python-devel git amqp python-pip libffi-devel openssl-devel gcc python-setuptools MySQL-python supervisor redis sshpass python-keyczar vim ansible-2.2.1.0 - -# code -RUN mkdir -p /root/compass-tasks -COPY . /root/compass-tasks -RUN mkdir -p /root/compass-tasks/compass && \ - touch /root/compass-tasks/compass/__init__.py -RUN mv /root/compass-tasks/actions /root/compass-tasks/compass/ && \ - mv /root/compass-tasks/apiclient /root/compass-tasks/compass/ && \ - mv /root/compass-tasks/tasks /root/compass-tasks/compass/ && \ - mv /root/compass-tasks/utils /root/compass-tasks/compass/ && \ - mv /root/compass-tasks/deployment /root/compass-tasks/compass/ && \ - mv /root/compass-tasks/db /root/compass-tasks/compass/ && \ - mv /root/compass-tasks/hdsdiscovery /root/compass-tasks/compass/ && \ - mv /root/compass-tasks/log_analyzor /root/compass-tasks/compass/ - -# pip -RUN easy_install --upgrade pip && \ - pip install --upgrade pip && \ - pip install --upgrade setuptools && \ - pip install --upgrade Flask - -# conf -RUN mkdir -p /etc/compass/ && \ - mkdir -p /etc/compass/machine_list && \ - mkdir -p /etc/compass/switch_list && \ - #cp -rf /root/compass-tasks/conf/* /etc/compass/ && \ - cd /root/compass-tasks && \ - python setup.py install - -RUN mkdir -p /root/.ssh; \ - echo "UserKnownHostsFile /dev/null" >> /root/.ssh/config; \ - echo "StrictHostKeyChecking no" >> /root/.ssh/config - -COPY supervisord.conf /etc/supervisord.conf -COPY start.sh /usr/local/bin/start.sh -RUN mkdir -p /var/log/compass -RUN mkdir -p /opt/ansible_callbacks -#RUN git clone https://github.com/openstack-ansible/openstack-ansible-modules /opt/openstack-ansible-modules -EXPOSE 6379 -VOLUME ["/var/ansible", "/etc/compass/machine_list", "/etc/compass/switch_list"] -ENTRYPOINT ["/bin/bash", "-c"] -CMD ["/usr/local/bin/start.sh"] diff --git a/compass-tasks/misc/compass_install.repo b/compass-tasks/misc/compass_install.repo deleted file mode 100644 index 6b97ed0..0000000 --- a/compass-tasks/misc/compass_install.repo +++ /dev/null @@ -1,5 +0,0 @@ -[compass_install] -name=compass_repo -baseurl=http://192.168.104.2:9999/download/compass_install/centos7/ -gpgcheck=0 -enabled=1 diff --git a/compass-tasks/requirements.txt b/compass-tasks/requirements.txt deleted file mode 100644 index 7e7ba40..0000000 --- a/compass-tasks/requirements.txt +++ /dev/null @@ -1,23 +0,0 @@ -amqplib -argparse -celery -Markdown<2.5 -Cheetah<=2.4.1 -Flask -Flask-Login<=0.3.2 -Flask-RESTful -Flask-Script -Flask-SQLAlchemy -Flask-WTF -itsdangerous -importlib -lazypy -lockfile -netaddr -# MySQL-python -paramiko -SQLAlchemy>=0.9.0 -simplejson -requests -redis -ansible==2.2.0 diff --git a/compass-tasks/run.sh b/compass-tasks/run.sh new file mode 100644 index 0000000..dcc96e4 --- /dev/null +++ b/compass-tasks/run.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +yum install https://rdoproject.org/repos/openstack-pike/rdo-release-pike.rpm -y +yum install git ntp wget ntpdate openssh-server python-devel sudo '@Development Tools' -y + +systemctl stop firewalld +systemctl mask firewalld + +pip install ansible==2.3.2.0 +rm -rf /opt/kargo_k8s +git clone https://github.com/kubernetes-incubator/kubespray.git /opt/kargo_k8s +cd /opt/kargo_k8s +git checkout v2.2.1 + +mkdir -p /opt/git/ +cd /opt/git/ +wget artifacts.opnfv.org/compass4nfv/package/openstack_pike.tar.gz +tar -zxvf openstack_pike.tar.gz +rm -rf openstack_pike.tar.gz +#cd openstack +#git clone https://github.com/openstack/tacker.git -b stable/pike +#cd tacker +#git checkout a0f1e680d81c7db66ae7a2a08c3d069901d0765a + + +git clone https://git.openstack.org/openstack/openstack-ansible /opt/openstack-ansible + +cd /opt/openstack-ansible + +#git checkout b962eed003580ee4c3bd69da911f20b3905a9176 +#git checkout da37351ca0a96ed38de72f3e00a7549a024cb810 +#git checkout 71110d6bc0f459b668948aca185139c1d79f0452 +git checkout 16c69046bfd90d1b984de43bc6267fece6b75f1c + +git checkout -b stable/pike + +#/bin/cp -rf /opt/tacker_conf/ansible-role-requirements.yml /opt/openstack-ansible/ +#/bin/cp -rf /opt/tacker_conf/openstack_services.yml /opt/openstack-ansible/playbooks/defaults/repo_packages/ +#/bin/cp -rf /opt/tacker_conf/os-tacker-install.yml /opt/openstack-ansible/playbooks/ +#/bin/cp -rf /opt/tacker_conf/setup-openstack.yml /opt/openstack-ansible/playbooks/ +#/bin/cp -rf /opt/tacker_conf/tacker.yml /opt/openstack-ansible/playbooks/inventory/env.d/ +#/bin/cp -rf /opt/tacker_conf/tacker_all.yml /opt/openstack-ansible/playbooks/inventory/group_vars/ +#/bin/cp -rf /opt/tacker_conf/user_secrets.yml /opt/openstack-ansible/etc/openstack_deploy/ +#/bin/cp -rf /opt/tacker_conf/haproxy_config.yml /opt/openstack-ansible/playbooks/vars/configs/ + +/bin/cp -rf /opt/openstack-ansible/etc/openstack_deploy /etc/openstack_deploy + +cd /opt/openstack-ansible + +scripts/bootstrap-ansible.sh + +rm -f /usr/local/bin/ansible-playbook + +cd /opt/openstack-ansible/scripts/ +python pw-token-gen.py --file /etc/openstack_deploy/user_secrets.yml + +cd /opt/openstack-ansible/group_vars +sed -i 's/#repo_build_git_cache/repo_build_git_cache/g' repo_all.yml + +cp /opt/setup-complete.yml /opt/openstack-ansible/playbooks/ +echo "- include: setup-complete.yml" >> /opt/openstack-ansible/playbooks/setup-infrastructure.yml + +mv /usr/local/bin/ansible /tmp diff --git a/compass-tasks/setup-complete.yml b/compass-tasks/setup-complete.yml new file mode 100644 index 0000000..f8a7b54 --- /dev/null +++ b/compass-tasks/setup-complete.yml @@ -0,0 +1,7 @@ +--- + +- hosts: localhost + user: root + tasks: + - name: Mark setup-infrastructure completed + shell: echo "Setup infrastructure completed!" diff --git a/compass-tasks/setup.py b/compass-tasks/setup.py deleted file mode 100644 index 5ee2e12..0000000 --- a/compass-tasks/setup.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""setup script.""" -try: - from setuptools import find_packages -except ImportError: - from ez_setup import use_setuptools - use_setuptools() - - -from setuptools.command.test import test as TestCommand -from setuptools import setup - - -import os -import sys - - -# This helps python setup.py test command to utilize tox -# See the instruction at https://testrun.org/tox/latest/example/basic.html\ -# #integration-with-setuptools-distribute-test-commands - -class Tox(TestCommand): - """Tox to do the setup.""" - - def finalize_options(self): - TestCommand.finalize_options(self) - self.test_args = [] - self.test_suite = True - - def run_tests(self): - import tox - errno = tox.cmdline(self.test_args) - sys.exit(errno) - - -INSTALL_REQUIRES_FILE = os.path.join( - os.path.dirname(__file__), 'requirements.txt') -with open(INSTALL_REQUIRES_FILE, 'r') as requires_file: - REQUIREMENTS = [line.strip() for line in requires_file if line != '\n'] - -DATA_FILES_DIR = os.path.join( - os.path.dirname(__file__), 'conf') -DATA_FILES = [] -for parent_dir, sub_dirs, files in os.walk(DATA_FILES_DIR): - if files == []: - pass - for file in files: - DATA_FILES.append((parent_dir, [os.path.join(parent_dir, file)])) - -setup( - name='compass-tasks', - version='0.1.0', - - # general info - description="""compass-tasks: Compass tasks module""", - author='Compass Development Group', - author_email='dev@syscompass.org', - url='https://github.com/openstack/compass-core', - download_url='', - - # dependency - install_requires=REQUIREMENTS, - packages=find_packages(exclude=['compass.tests']), - include_package_data=True, - classifiers=[ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - ], - # data - # data_files=DATA_FILES, - # test, - tests_require=['tox'], - cmdclass={'test': Tox}, -) diff --git a/compass-tasks/start.sh b/compass-tasks/start.sh deleted file mode 100755 index 4d3956d..0000000 --- a/compass-tasks/start.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -/sbin/init -/usr/bin/supervisord -tail -f /dev/null diff --git a/compass-tasks/supervisord.conf b/compass-tasks/supervisord.conf deleted file mode 100644 index 9d993d3..0000000 --- a/compass-tasks/supervisord.conf +++ /dev/null @@ -1,135 +0,0 @@ -; Sample supervisor config file. - -[unix_http_server] -file=/var/run/supervisor/supervisor.sock ; (the path to the socket file) -;chmod=0700 ; sockef file mode (default 0700) -;chown=nobody:nogroup ; socket file uid:gid owner -;username=user ; (default is no username (open server)) -;password=123 ; (default is no password (open server)) - -;[inet_http_server] ; inet (TCP) server disabled by default -;port=127.0.0.1:9001 ; (ip_address:port specifier, *:port for all iface) -;username=user ; (default is no username (open server)) -;password=123 ; (default is no password (open server)) - -[supervisord] -logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log) -logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) -logfile_backups=10 ; (num of main logfile rotation backups;default 10) -loglevel=info ; (log level;default info; others: debug,warn,trace) -pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) -nodaemon=false ; (start in foreground if true;default false) -minfds=1024 ; (min. avail startup file descriptors;default 1024) -minprocs=200 ; (min. avail process descriptors;default 200) -;umask=022 ; (process file creation umask;default 022) -;user=chrism ; (default is current user, required if root) -;identifier=supervisor ; (supervisord identifier, default is 'supervisor') -;directory=/tmp ; (default is not to cd during start) -;nocleanup=true ; (don't clean up tempfiles at start;default false) -;childlogdir=/tmp ; ('AUTO' child log dir, default $TEMP) -;environment=KEY=value ; (key value pairs to add to environment) -;strip_ansi=false ; (strip ansi escape codes in logs; def. false) - -; the below section must remain in the config file for RPC -; (supervisorctl/web interface) to work, additional interfaces may be -; added by defining them in separate rpcinterface: sections -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisorctl] -serverurl=unix:///var/run/supervisor/supervisor.sock ; use a unix:// URL for a unix socket -;serverurl=http://127.0.0.1:9001 ; use an http:// url to specify an inet socket -;username=chris ; should be same as http_username if set -;password=123 ; should be same as http_password if set -;prompt=mysupervisor ; cmd line prompt (default "supervisor") -;history_file=~/.sc_history ; use readline history if available - -; The below sample program section shows all possible program subsection values, -; create one or more 'real' program: sections to be able to control them under -; supervisor. - -;[program:theprogramname] -;command=/bin/cat ; the program (relative uses PATH, can take args) -;process_name=%(program_name)s ; process_name expr (default %(program_name)s) -;numprocs=1 ; number of processes copies to start (def 1) -;directory=/tmp ; directory to cwd to before exec (def no cwd) -;umask=022 ; umask for process (default None) -;priority=999 ; the relative start priority (default 999) -;autostart=true ; start at supervisord start (default: true) -;autorestart=true ; retstart at unexpected quit (default: true) -;startsecs=10 ; number of secs prog must stay running (def. 1) -;startretries=3 ; max # of serial start failures (default 3) -;exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) -;stopsignal=QUIT ; signal used to kill process (default TERM) -;stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) -;user=chrism ; setuid to this UNIX account to run the program -;redirect_stderr=true ; redirect proc stderr to stdout (default false) -;stdout_logfile=/a/path ; stdout log path, NONE for none; default AUTO -;stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) -;stdout_logfile_backups=10 ; # of stdout logfile backups (default 10) -;stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) -;stdout_events_enabled=false ; emit events on stdout writes (default false) -;stderr_logfile=/a/path ; stderr log path, NONE for none; default AUTO -;stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) -;stderr_logfile_backups=10 ; # of stderr logfile backups (default 10) -;stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) -;stderr_events_enabled=false ; emit events on stderr writes (default false) -;environment=A=1,B=2 ; process environment additions (def no adds) -;serverurl=AUTO ; override serverurl computation (childutils) - -; The below sample eventlistener section shows all possible -; eventlistener subsection values, create one or more 'real' -; eventlistener: sections to be able to handle event notifications -; sent by supervisor. - -;[eventlistener:theeventlistenername] -;command=/bin/eventlistener ; the program (relative uses PATH, can take args) -;process_name=%(program_name)s ; process_name expr (default %(program_name)s) -;numprocs=1 ; number of processes copies to start (def 1) -;events=EVENT ; event notif. types to subscribe to (req'd) -;buffer_size=10 ; event buffer queue size (default 10) -;directory=/tmp ; directory to cwd to before exec (def no cwd) -;umask=022 ; umask for process (default None) -;priority=-1 ; the relative start priority (default -1) -;autostart=true ; start at supervisord start (default: true) -;autorestart=unexpected ; restart at unexpected quit (default: unexpected) -;startsecs=10 ; number of secs prog must stay running (def. 1) -;startretries=3 ; max # of serial start failures (default 3) -;exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) -;stopsignal=QUIT ; signal used to kill process (default TERM) -;stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) -;user=chrism ; setuid to this UNIX account to run the program -;redirect_stderr=true ; redirect proc stderr to stdout (default false) -;stdout_logfile=/a/path ; stdout log path, NONE for none; default AUTO -;stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) -;stdout_logfile_backups=10 ; # of stdout logfile backups (default 10) -;stdout_events_enabled=false ; emit events on stdout writes (default false) -;stderr_logfile=/a/path ; stderr log path, NONE for none; default AUTO -;stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) -;stderr_logfile_backups ; # of stderr logfile backups (default 10) -;stderr_events_enabled=false ; emit events on stderr writes (default false) -;environment=A=1,B=2 ; process environment additions -;serverurl=AUTO ; override serverurl computation (childutils) - -; The below sample group section shows all possible group values, -; create one or more 'real' group: sections to create "heterogeneous" -; process groups. - -;[group:thegroupname] -;programs=progname1,progname2 ; each refers to 'x' in [program:x] definitions -;priority=999 ; the relative start priority (default 999) - -; The [include] section can just contain the "files" setting. This -; setting can list multiple files (separated by whitespace or -; newlines). It can also contain wildcards. The filenames are -; interpreted as relative to this file. Included files *cannot* -; include files themselves. - -[program:celeryd] -command=nohup bash -c "C_FORCE_ROOT=1 CELERY_CONFIG_MODULE=compass.utils.celeryconfig_wrapper /usr/bin/celery worker &> /tmp/celery-worker.log" & - -[program:orca-heartbeatd] -command=nohup bash -c "/usr/bin/redis-server" & -[include] -files = supervisord.d/*.ini - diff --git a/compass-tasks/tacker_conf/ansible-role-requirements.yml b/compass-tasks/tacker_conf/ansible-role-requirements.yml new file mode 100644 index 0000000..40e0f91 --- /dev/null +++ b/compass-tasks/tacker_conf/ansible-role-requirements.yml @@ -0,0 +1,192 @@ +- name: apt_package_pinning + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-apt_package_pinning + version: d5c0d7e8d21fc1539cc043097e23851296e2dd95 +- name: pip_install + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-pip_install + version: e142f5642646449d98cd65d8fd0690907c403801 +- name: galera_client + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-galera_client + version: 1d3cdcd33c75a668ac3be046ac53fe1842780058 +- name: galera_server + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-galera_server + version: e47877adc99bf64b900c83056716dfc09f2bcc0b +- name: ceph_client + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-ceph_client + version: d1b30ae7b660acbb4f9d6e4f4ab702581c6748d9 +- name: haproxy_server + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-haproxy_server + version: ffba50ecf0c99f41d1d5a1195268a2c23c38f89f +- name: keepalived + scm: git + src: https://github.com/evrardjp/ansible-keepalived + version: 2.6.0 +- name: lxc_container_create + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-lxc_container_create + version: 540bd82f8ded168ccab25a4a762a5a88e5a4ecbd +- name: lxc_hosts + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-lxc_hosts + version: 73a62894aa26e4b63bb69336411209fc0df6abb2 +- name: memcached_server + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-memcached_server + version: ddd1dc402b7c825265a9e59483272f2dfcd65a93 +- name: openstack-ansible-security + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-security + version: c58c839f9cc9fcb75d32fc8de00b4f27add36661 +- name: openstack_hosts + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-openstack_hosts + version: 1ba80641ed5c4d55a41906e9d37df10b9fd720ee +- name: os_keystone + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_keystone + version: 600e0e58102ccbdd082b5c90237cb7d27c63e5fa +- name: openstack_openrc + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-openstack_openrc + version: dfdf65b3db7c6240fa173a3a62af4e5028023626 +- name: os_aodh + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_aodh + version: a1ec96ee54d7b8c7fd24f1148545d3556a0a8b5b +- name: os_barbican + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_barbican + version: 2d087176610c712f51dec8722b6129e648b149e2 +- name: os_ceilometer + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_ceilometer + version: 1d709bd6eace68bfd7fb7d1e52df8d5b6f28810b +- name: os_cinder + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_cinder + version: 66542db4c7045dc559a1b3e6bd293ca28e09c34e +- name: os_glance + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_glance + version: 79d6b78e894715ba3a17b50cc18917b251025069 +- name: os_gnocchi + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_gnocchi + version: 5932746fe1717ef2798b46c374c268945f34814e +- name: os_heat + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_heat + version: cbe917fefeb90c506e1c73c31b9a1d00fbf9beff +- name: os_horizon + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_horizon + version: 25435372547d217116dd4ba10b5d0e8c39307b52 +- name: os_ironic + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_ironic + version: 4110fca2b9e1c6a1698b9be9797253db4a1cad2e +- name: os_magnum + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_magnum + version: f452cc54d1b3240f81a82fd110a467cd996a3331 +- name: os_trove + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_trove + version: 51b38916ad99f3ffbc1723a7d97f9034b4c02dc4 +- name: os_neutron + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_neutron + version: 4b43221adae78627d8035cd82e791662821a461c +- name: os_nova + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_nova + version: 8aacdd01ad98d82b3b817895d39af44b3a862847 +- name: os_rally + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_rally + version: 1410d71f60aa354259fa11d9dddfe4ed743177ab +- name: os_sahara + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_sahara + version: 5e29921df9cc7d3ba2a89b0d6cdddcf02553c423 +- name: os_swift + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_swift + version: 70c3795215b5b3ea90090ffd99381c97bc45be80 +- name: os_tacker + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_tacker + version: c6a563731b4482a28b7a01982252d79f52fe586b +- name: os_tempest + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_tempest + version: 5934f81385870cc836f7d1cd84b8dbc6a97cedad +- name: plugins + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-plugins + version: 9ce61bdc6cb537c2377e95da1c3bfc9e837c9784 +- name: rabbitmq_server + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-rabbitmq_server + version: 3d1de52f75c08b3265d2b35c56c9217d60c2c840 +- name: repo_build + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-repo_build + version: 6d00b7d6f431fc8706be81e1089ee351d2172e48 +- name: repo_server + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-repo_server + version: 860ff80b348f1c9e93745da55b1df69d05283afb +- name: rsyslog_client + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-rsyslog_client + version: 5f5275720d13bd22945348751131c39877b731ab +- name: rsyslog_server + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-rsyslog_server + version: a7d4b0df2e9ed3c2a1414613e96c7c9db0f2db60 +- name: sshd + scm: git + src: https://github.com/willshersystems/ansible-sshd + version: 0.4.5 +- name: bird + scm: git + src: https://github.com/logan2211/ansible-bird + version: '1.2' +- name: etcd + scm: git + src: https://github.com/logan2211/ansible-etcd + version: '1.2' +- name: unbound + scm: git + src: https://github.com/logan2211/ansible-unbound + version: '1.3' +- name: resolvconf + scm: git + src: https://github.com/logan2211/ansible-resolvconf + version: '1.2' +- name: os_designate + scm: git + src: https://git.openstack.org/openstack/openstack-ansible-os_designate + version: fc4f0fb40197954c96f486f4ebbd679162b27a6d +- name: ceph.ceph-common + scm: git + src: https://github.com/ceph/ansible-ceph-common + version: v2.2.4 +- name: ceph.ceph-docker-common + scm: git + src: https://github.com/ceph/ansible-ceph-docker-common + version: ca86fd0ef6d24aa2c750a625acdcb8012c374aa0 +- name: ceph-mon + scm: git + src: https://github.com/ceph/ansible-ceph-mon + version: v2.2.4 +- name: ceph-osd + scm: git + src: https://github.com/ceph/ansible-ceph-osd + version: v2.2.4 diff --git a/compass-tasks/tacker_conf/haproxy_config.yml b/compass-tasks/tacker_conf/haproxy_config.yml new file mode 100644 index 0000000..bbac6d7 --- /dev/null +++ b/compass-tasks/tacker_conf/haproxy_config.yml @@ -0,0 +1,285 @@ +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +haproxy_default_services: + - service: + haproxy_service_name: galera + haproxy_backend_nodes: "{{ [groups['galera_all'][0]] | default([]) }}" # list expected + haproxy_backup_nodes: "{{ groups['galera_all'][1:] | default([]) }}" + haproxy_bind: "{{ [internal_lb_vip_address] }}" + haproxy_port: 3306 + haproxy_balance_type: tcp + haproxy_timeout_client: 5000s + haproxy_timeout_server: 5000s + haproxy_backend_options: + - "mysql-check user {{ galera_monitoring_user }}" + haproxy_whitelist_networks: "{{ haproxy_galera_whitelist_networks }}" + - service: + haproxy_service_name: repo_git + haproxy_backend_nodes: "{{ groups['repo_all'] | default([]) }}" + haproxy_bind: "{{ [internal_lb_vip_address] }}" + haproxy_port: 9418 + haproxy_balance_type: tcp + haproxy_backend_options: + - tcp-check + haproxy_whitelist_networks: "{{ haproxy_repo_git_whitelist_networks }}" + - service: + haproxy_service_name: repo_all + haproxy_backend_nodes: "{{ groups['repo_all'] | default([]) }}" + haproxy_bind: "{{ [internal_lb_vip_address] }}" + haproxy_port: 8181 + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk HEAD /" + - service: + haproxy_service_name: repo_cache + haproxy_backend_nodes: "{{ [groups['repo_all'][0]] | default([]) }}" # list expected + haproxy_backup_nodes: "{{ groups['repo_all'][1:] | default([]) }}" + haproxy_bind: "{{ [internal_lb_vip_address] }}" + haproxy_port: "{{ repo_pkg_cache_port }}" + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk HEAD /acng-report.html" + haproxy_whitelist_networks: "{{ haproxy_repo_cache_whitelist_networks }}" + - service: + haproxy_service_name: glance_api + haproxy_backend_nodes: "{{ groups['glance_api'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: 9292 + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk /healthcheck" + - service: + haproxy_service_name: glance_registry + haproxy_backend_nodes: "{{ groups['glance_registry'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: 9191 + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk /healthcheck" + haproxy_whitelist_networks: "{{ haproxy_glance_registry_whitelist_networks }}" + - service: + haproxy_service_name: gnocchi + haproxy_backend_nodes: "{{ groups['gnocchi_all'] | default([]) }}" + haproxy_port: 8041 + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk /healthcheck" + - service: + haproxy_service_name: heat_api_cfn + haproxy_backend_nodes: "{{ groups['heat_api_cfn'] | default([]) }}" + haproxy_port: 8000 + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk HEAD /" + - service: + haproxy_service_name: heat_api_cloudwatch + haproxy_backend_nodes: "{{ groups['heat_api_cloudwatch'] | default([]) }}" + haproxy_port: 8003 + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk HEAD /" + - service: + haproxy_service_name: heat_api + haproxy_backend_nodes: "{{ groups['heat_api'] | default([]) }}" + haproxy_port: 8004 + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk HEAD /" + - service: + haproxy_service_name: keystone_service + haproxy_backend_nodes: "{{ groups['keystone_all'] | default([]) }}" + haproxy_port: 5000 + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_balance_type: "http" + haproxy_backend_options: + - "httpchk HEAD /" + - service: + haproxy_service_name: keystone_admin + haproxy_backend_nodes: "{{ groups['keystone_all'] | default([]) }}" + haproxy_port: 35357 + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_balance_type: "http" + haproxy_backend_options: + - "httpchk HEAD /" + haproxy_whitelist_networks: "{{ haproxy_keystone_admin_whitelist_networks }}" + - service: + haproxy_service_name: neutron_server + haproxy_backend_nodes: "{{ groups['neutron_server'] | default([]) }}" + haproxy_port: 9696 + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk HEAD /" + - service: + haproxy_service_name: nova_api_metadata + haproxy_backend_nodes: "{{ groups['nova_api_metadata'] | default([]) }}" + haproxy_port: 8775 + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk HEAD /" + haproxy_whitelist_networks: "{{ haproxy_nova_metadata_whitelist_networks }}" + - service: + haproxy_service_name: nova_api_os_compute + haproxy_backend_nodes: "{{ groups['nova_api_os_compute'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: 8774 + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk HEAD /" + - service: + haproxy_service_name: nova_api_placement + haproxy_backend_nodes: "{{ groups['nova_api_placement'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: 8780 + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk HEAD /" + - service: + haproxy_service_name: nova_console + haproxy_backend_nodes: "{{ groups['nova_console'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: "{{ nova_console_port }}" + haproxy_balance_type: http + haproxy_timeout_client: 60m + haproxy_timeout_server: 60m + haproxy_balance_alg: source + haproxy_backend_options: + - "httpchk HEAD /" + haproxy_backend_httpcheck_options: + - "expect status 404" + - service: + haproxy_service_name: cinder_api + haproxy_backend_nodes: "{{ groups['cinder_api'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: 8776 + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk HEAD /" + - service: + haproxy_service_name: horizon + haproxy_backend_nodes: "{{ groups['horizon_all'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_ssl_all_vips: true + haproxy_port: "{{ haproxy_ssl | ternary(443,80) }}" + haproxy_backend_port: 80 + haproxy_redirect_http_port: 80 + haproxy_balance_type: http + haproxy_balance_alg: source + haproxy_backend_options: + - "httpchk HEAD /" + - service: + haproxy_service_name: sahara_api + haproxy_backend_nodes: "{{ groups['sahara_api'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_balance_alg: source + haproxy_port: 8386 + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk /healthcheck" + - service: + haproxy_service_name: swift_proxy + haproxy_backend_nodes: "{{ groups['swift_proxy'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_balance_alg: source + haproxy_port: 8080 + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk /healthcheck" + - service: + haproxy_service_name: ceilometer_api + haproxy_backend_nodes: "{{ groups['ceilometer_api_container'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: 8777 + haproxy_balance_type: tcp + haproxy_backend_options: + - tcp-check + - service: + haproxy_service_name: aodh_api + haproxy_backend_nodes: "{{ groups['aodh_api'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: 8042 + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk HEAD /" + haproxy_backend_httpcheck_options: + - "expect status 401" + - service: + haproxy_service_name: ironic_api + haproxy_backend_nodes: "{{ groups['ironic_api'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: 6385 + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk GET /" + - service: + haproxy_service_name: rabbitmq_mgmt + haproxy_backend_nodes: "{{ groups['rabbitmq'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: 15672 + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk HEAD /" + haproxy_whitelist_networks: "{{ haproxy_rabbitmq_management_whitelist_networks }}" + - service: + haproxy_service_name: magnum + haproxy_backend_nodes: "{{ groups['magnum_all'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: 9511 + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk GET /" + - service: + haproxy_service_name: trove + haproxy_backend_nodes: "{{ groups['trove_api'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: 8779 + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk HEAD /" + - service: + haproxy_service_name: barbican + haproxy_backend_nodes: "{{ groups['barbican_api'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: 9311 + haproxy_balance_type: http + haproxy_backend_options: + - "httpchk HEAD /" + haproxy_backend_httpcheck_options: + - "expect status 401" + - service: + haproxy_service_name: designate_api + haproxy_backend_nodes: "{{ groups['designate_api'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: 9001 + haproxy_balance_type: http + haproxy_backend_options: + - "forwardfor" + - "httpchk /versions" + - "httplog" + - service: + haproxy_service_name: tacker + haproxy_backend_nodes: "{{ groups['tacker_all'] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: 9890 + haproxy_balance_type: http + haproxy_backend_options: + - "forwardfor" + - "httpchk" + - "httplog" diff --git a/compass-tasks/tacker_conf/openstack_services.yml b/compass-tasks/tacker_conf/openstack_services.yml new file mode 100644 index 0000000..5a772e5 --- /dev/null +++ b/compass-tasks/tacker_conf/openstack_services.yml @@ -0,0 +1,228 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +## NOTICE on items in this file: +## * If you use anything in the *._git_install_branch field that is not a TAG +## make sure to leave an in-line comment as to "why". + +## For the sake of anyone else editing this file: +## * If you add services to this file please do so in alphabetical order. +## * Every entry should be name spaced with the name of the client followed by an "_" +## * All items with this file should be separated by `name_` note that the name of the +## package should be one long name with no additional `_` separating it. + + +### Before this is shipped all of these services should have a tag set as the branch, +### or have a comment / reason attached to them as to why a tag can not work. + + +## Global Requirements +requirements_git_repo: https://git.openstack.org/openstack/requirements +requirements_git_install_branch: 90094c5d578ecfc4ab1e9f38a86bca5b615a3527 # HEAD of "stable/ocata" as of 15.05.2017 +requirements_git_dest: "/opt/requirements_{{ requirements_git_install_branch | replace('/', '_') }}" + + +## Aodh service +aodh_git_repo: https://git.openstack.org/openstack/aodh +aodh_git_install_branch: 5c6b46ecc85743798c04b2d5fd094a0e9ef661a7 # HEAD of "stable/ocata" as of 15.05.2017 +aodh_git_dest: "/opt/aodh_{{ aodh_git_install_branch | replace('/', '_') }}" +aodh_git_project_group: aodh_all + + +## Barbican service +barbican_git_repo: https://git.openstack.org/openstack/barbican +barbican_git_install_branch: b04aa152843d9cda3b11e8736e740c71fd798d0f # HEAD of "stable/ocata" as of 15.05.2017 +barbican_git_dest: "/opt/barbican_{{ barbican_git_install_branch | replace('/', '_') }}" +barbican_git_project_group: barbican_all + + +## Ceilometer service +ceilometer_git_repo: https://git.openstack.org/openstack/ceilometer +ceilometer_git_install_branch: c25e934b8c48b00889f8fa6bb3e5bcb9591df618 # HEAD of "stable/ocata" as of 15.05.2017 +ceilometer_git_dest: "/opt/ceilometer_{{ceilometer_git_install_branch | replace('/', '_') }}" +ceilometer_git_project_group: ceilometer_all + + +## Cinder service +cinder_git_repo: https://git.openstack.org/openstack/cinder +cinder_git_install_branch: 501485b25087ff398986aedbbdb893741f8aabae # HEAD of "stable/ocata" as of 15.05.2017 +cinder_git_dest: "/opt/cinder_{{ cinder_git_install_branch | replace('/', '_') }}" +cinder_git_project_group: cinder_all + + +## Designate service +designate_git_repo: https://git.openstack.org/openstack/designate +designate_git_install_branch: 021aa52325c2b58be6317a014cb3fd44be02f29f # HEAD of "stable/ocata" as of 15.05.2017 +designate_git_dest: "/opt/designate_{{ designate_git_install_branch | replace('/', '_') }}" +designate_git_project_group: designate_all + + +## Horizon Designate dashboard plugin +designate_dashboard_git_repo: https://git.openstack.org/openstack/designate-dashboard +designate_dashboard_git_install_branch: fb8108354937ef6d3a1bb389481af79681f7c5b3 # HEAD of "stable/ocata" as of 15.05.2017 +designate_dashboard_git_dest: "/opt/designate_dashboard_{{ designate_dashboard_git_install_branch | replace('/', '_') }}" +designate_dashboard_git_project_group: horizon_all + + +## Dragonflow service +dragonflow_git_repo: https://git.openstack.org/openstack/dragonflow +dragonflow_git_install_branch: b72ffa06721faf95f42c9702f60a9acc3a7d8b61 # HEAD of "stable/ocata" as of 15.05.2017 +dragonflow_git_dest: "/opt/dragonflow_{{ dragonflow_git_install_branch | replace('/', '_') }}" +dragonflow_git_project_group: neutron_all + + +## Glance service +glance_git_repo: https://git.openstack.org/openstack/glance +glance_git_install_branch: 57c4d7d78f37e840660719b944ebabe91cbf231b # HEAD of "stable/ocata" as of 15.05.2017 +glance_git_dest: "/opt/glance_{{ glance_git_install_branch | replace('/', '_') }}" +glance_git_project_group: glance_all + + +## Heat service +heat_git_repo: https://git.openstack.org/openstack/heat +heat_git_install_branch: fe6ae385bc7bba77864b32061600ac40b24f2a10 # HEAD of "stable/ocata" as of 15.05.2017 +heat_git_dest: "/opt/heat_{{ heat_git_install_branch | replace('/', '_') }}" +heat_git_project_group: heat_all + + +## Horizon service +horizon_git_repo: https://git.openstack.org/openstack/horizon +horizon_git_install_branch: 2684552f09ee5d597a507833cf232f0f9d48d9b1 # HEAD of "stable/ocata" as of 15.05.2017 +horizon_git_dest: "/opt/horizon_{{ horizon_git_install_branch | replace('/', '_') }}" +horizon_git_project_group: horizon_all + +## Horizon Ironic dashboard plugin +ironic_dashboard_git_repo: https://git.openstack.org/openstack/ironic-ui +ironic_dashboard_git_install_branch: 6a0cdbff587df20ae913ca7f0d434ad8381b761c # HEAD of "stable/ocata" as of 15.05.2017 +ironic_dashboard_git_dest: "/opt/ironic_dashboard_{{ ironic_dashboard_git_install_branch | replace('/', '_') }}" +ironic_dashboard_git_project_group: horizon_all + +## Horizon Magnum dashboard plugin +magnum_dashboard_git_repo: https://git.openstack.org/openstack/magnum-ui +magnum_dashboard_git_install_branch: db3bfff643ceb9353c7c0cafdd4e15e0fcf01fb1 # HEAD of "stable/ocata" as of 15.05.2017 +magnum_dashboard_git_dest: "/opt/magnum_dashboard_{{ magnum_dashboard_git_install_branch | replace('/', '_') }}" +magnum_dashboard_git_project_group: horizon_all + +## Horizon LBaaS dashboard plugin +neutron_lbaas_dashboard_git_repo: https://git.openstack.org/openstack/neutron-lbaas-dashboard +neutron_lbaas_dashboard_git_install_branch: 3653432bb8287fa0ec1248d866e4717b90ab824b # HEAD of "stable/ocata" as of 15.05.2017 +neutron_lbaas_dashboard_git_dest: "/opt/neutron_lbaas_dashboard_{{ neutron_lbaas_dashboard_git_install_branch | replace('/', '_') }}" +neutron_lbaas_dashboard_git_project_group: horizon_all + +## Horizon Sahara dashboard plugin +sahara_dashboard_git_repo: https://git.openstack.org/openstack/sahara-dashboard +sahara_dashboard_git_install_branch: 5270e94b168b3c20e5c787a3f43d44b7e101c448 # HEAD of "stable/ocata" as of 15.05.2017 +sahara_dashboard_git_dest: "/opt/sahara_dashboard_{{ sahara_dashboard_git_install_branch | replace('/', '_') }}" +sahara_dashboard_git_project_group: horizon_all + + +## Keystone service +keystone_git_repo: https://git.openstack.org/openstack/keystone +keystone_git_install_branch: 955fd6ca3758e217d9d98480852e0014dc11e988 # HEAD of "stable/ocata" as of 15.05.2017 +keystone_git_dest: "/opt/keystone_{{ keystone_git_install_branch | replace('/', '_') }}" +keystone_git_project_group: keystone_all + + +## Neutron service +neutron_git_repo: https://git.openstack.org/openstack/neutron +neutron_git_install_branch: 515a2ff4ce3239f6a077d8b07ff80544023f0631 # HEAD of "stable/ocata" as of 15.05.2017 +neutron_git_dest: "/opt/neutron_{{ neutron_git_install_branch | replace('/', '_') }}" +neutron_git_project_group: neutron_all + +neutron_lbaas_git_repo: https://git.openstack.org/openstack/neutron-lbaas +neutron_lbaas_git_install_branch: 674e3604a6285655f2dc7f2949629cc4de06bbf4 # HEAD of "stable/ocata" as of 15.05.2017 +neutron_lbaas_git_dest: "/opt/neutron_lbaas_{{ neutron_lbaas_git_install_branch | replace('/', '_') }}" +neutron_lbaas_git_project_group: neutron_all + +neutron_vpnaas_git_repo: https://git.openstack.org/openstack/neutron-vpnaas +neutron_vpnaas_git_install_branch: 4a639ac09ccfc7573452d0ac9d189d934c169d34 # HEAD of "stable/ocata" as of 15.05.2017 +neutron_vpnaas_git_dest: "/opt/neutron_vpnaas_{{ neutron_vpnaas_git_install_branch | replace('/', '_') }}" +neutron_vpnaas_git_project_group: neutron_all + +neutron_fwaas_git_repo: https://git.openstack.org/openstack/neutron-fwaas +neutron_fwaas_git_install_branch: 3301972f46d877d0f68d9cf7f9246e0df897a91e # HEAD of "stable/ocata" as of 15.05.2017 +neutron_fwaas_git_dest: "/opt/neutron_fwaas_{{ neutron_fwaas_git_install_branch | replace('/', '_') }}" +neutron_fwaas_git_project_group: neutron_all + +neutron_dynamic_routing_git_repo: https://git.openstack.org/openstack/neutron-dynamic-routing +neutron_dynamic_routing_git_install_branch: da877412200e460fca34edaf56f84286557bd486 # HEAD of "stable/ocata" as of 15.05.2017 +neutron_dynamic_routing_git_dest: "/opt/neutron_dynamic_routing_{{ neutron_dynamic_routing_git_install_branch | replace('/', '_') }}" +neutron_dynamic_routing_git_project_group: neutron_all + + +## Nova service +nova_git_repo: https://git.openstack.org/openstack/nova +nova_git_install_branch: 506465a027dd1ba1c90949dc58297edae32da7e4 # HEAD of "stable/ocata" as of 15.05.2017 +nova_git_dest: "/opt/nova_{{ nova_git_install_branch | replace('/', '_') }}" +nova_git_project_group: nova_all + + +## PowerVM Virt Driver +nova_powervm_git_repo: https://git.openstack.org/openstack/nova-powervm +nova_powervm_git_install_branch: 53a5c75a0db38b606f51a53fd159b7ffab7c3a33 # HEAD of "stable/ocata" as of 15.05.2017 +nova_powervm_git_dest: "/opt/nova_powervm_{{ nova_powervm_git_install_branch | replace('/', '_') }}" +nova_powervm_git_project_group: nova_compute + + +## LXD Virt Driver +nova_lxd_git_repo: https://git.openstack.org/openstack/nova-lxd +nova_lxd_git_install_branch: ed05fa417c4a78970dd5bdcdd3e1922f3c07f0ac # HEAD of "stable/ocata" as of 15.05.2017 +nova_lxd_git_dest: "/opt/nova_lxd_{{ nova_lxd_git_install_branch | replace('/', '_') }}" +nova_lxd_git_project_group: nova_compute + + +## Sahara service +sahara_git_repo: https://git.openstack.org/openstack/sahara +sahara_git_install_branch: 5241340d3e668bcb3b0e9f09c38afdc21e6c136d # HEAD of "stable/ocata" as of 15.05.2017 +sahara_git_dest: "/opt/sahara_{{ sahara_git_install_branch | replace('/', '_') }}" +sahara_git_project_group: sahara_all + + +## Swift service +swift_git_repo: https://git.openstack.org/openstack/swift +swift_git_install_branch: 177fca2b3a6a7a6e17e20d161c23ac8a10500939 # HEAD of "stable/ocata" as of 15.05.2017 +swift_git_dest: "/opt/swift_{{ swift_git_install_branch | replace('/', '_') }}" +swift_git_project_group: swift_all + + +## Ironic service +ironic_git_repo: https://git.openstack.org/openstack/ironic +ironic_git_install_branch: ed2f6cea892ccb2a1343dd935ad279de8fd3a471 # HEAD of "stable/ocata" as of 15.05.2017 +ironic_git_dest: "/opt/ironic_{{ ironic_git_install_branch | replace('/', '_') }}" +ironic_git_project_group: ironic_all + +## Magnum service +magnum_git_repo: https://git.openstack.org/openstack/magnum +magnum_git_install_branch: 1a685113d8df479c56ad85aa001930f6b8f1e2dd # HEAD of "stable/ocata" as of 15.05.2017 +magnum_git_dest: "/opt/magnum_{{ magnum_git_install_branch | replace('/', '_') }}" +magnum_git_project_group: magnum_all + +## Trove service +trove_git_repo: https://git.openstack.org/openstack/trove +trove_git_install_branch: ca0a1128f91aa5f54b9f57fedf2f5db4b6f82e20 # HEAD of "stable/ocata" as of 15.05.2017 +trove_git_dest: "/opt/trove_{{ trove_git_install_branch | replace('/', '_') }}" +trove_git_project_group: trove_all + +## Horizon Trove dashboard plugin +trove_dashboard_git_repo: https://git.openstack.org/openstack/trove-dashboard +trove_dashboard_git_install_branch: 5fd256e58219074b55cc9474a74516105787ac03 # HEAD of "stable/ocata" as of 15.05.2017 +trove_dashboard_git_dest: "/opt/trove_dashboard_{{ trove_dashboard_git_install_branch | replace('/', '_') }}" +trove_dashboard_git_project_group: horizon_all + +## Tacker service +tacker_git_repo: https://git.openstack.org/openstack/tacker +tacker_git_install_branch: a0f1e680d81c7db66ae7a2a08c3d069901d0765a # HEAD of "ocata" as of 04.09.2017 +tacker_git_project_group: tacker_all diff --git a/compass-tasks/tacker_conf/os-tacker-install.yml b/compass-tasks/tacker_conf/os-tacker-install.yml new file mode 100644 index 0000000..dd96595 --- /dev/null +++ b/compass-tasks/tacker_conf/os-tacker-install.yml @@ -0,0 +1,63 @@ +--- +# Copyright 2017, SUSE LINUX GmbH. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install the tacker components + hosts: tacker_all + gather_facts: "{{ gather_facts | default(True) }}" + max_fail_percentage: 20 + user: root + pre_tasks: + - include: common-tasks/os-lxc-container-setup.yml + - include: common-tasks/rabbitmq-vhost-user.yml + static: no + vars: + user: "{{ tacker_rabbitmq_userid }}" + password: "{{ tacker_rabbitmq_password }}" + vhost: "{{ tacker_rabbitmq_vhost }}" + _rabbitmq_host_group: "{{ tacker_rabbitmq_host_group }}" + when: + - inventory_hostname == groups['tacker_all'][0] + - groups[tacker_rabbitmq_host_group] | length > 0 + - include: common-tasks/os-log-dir-setup.yml + vars: + log_dirs: + - src: "/openstack/log/{{ inventory_hostname }}-tacker" + dest: "/var/log/tacker" + - include: common-tasks/mysql-db-user.yml + static: no + vars: + user_name: "{{ tacker_galera_user }}" + password: "{{ tacker_container_mysql_password }}" + login_host: "{{ tacker_galera_address }}" + db_name: "{{ tacker_galera_database }}" + when: inventory_hostname == groups['tacker_all'][0] + - include: common-tasks/package-cache-proxy.yml + roles: + - role: "os_tacker" + - role: "openstack_openrc" + tags: + - openrc + - role: "rsyslog_client" + rsyslog_client_log_rotate_file: tacker_log_rotate + rsyslog_client_log_dir: "/var/log/tacker" + rsyslog_client_config_name: "99-tacker-rsyslog-client.conf" + tags: + - rsyslog + vars: + is_metal: "{{ properties.is_metal|default(false) }}" + tacker_galera_address: "{{ internal_lb_vip_address }}" + environment: "{{ deployment_environment_variables | default({}) }}" + tags: + - tacker diff --git a/compass-tasks/tacker_conf/setup-openstack.yml b/compass-tasks/tacker_conf/setup-openstack.yml new file mode 100644 index 0000000..d32eb15 --- /dev/null +++ b/compass-tasks/tacker_conf/setup-openstack.yml @@ -0,0 +1,43 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- include: os-keystone-install.yml +- include: os-barbican-install.yml +- include: os-glance-install.yml +- include: os-cinder-install.yml +- include: os-nova-install.yml +- include: os-neutron-install.yml +- include: os-heat-install.yml +- include: os-horizon-install.yml +- include: os-ceilometer-install.yml +- include: os-aodh-install.yml +- include: os-designate-install.yml +#NOTE(stevelle) Ensure Gnocchi identities exist before Swift +- include: os-gnocchi-install.yml + when: + - gnocchi_storage_driver is defined + - gnocchi_storage_driver == 'swift' + vars: + gnocchi_identity_only: True +- include: os-swift-install.yml +- include: os-gnocchi-install.yml +- include: os-ironic-install.yml +- include: os-magnum-install.yml +- include: os-trove-install.yml +- include: os-sahara-install.yml +- include: os-tacker-install.yml +- include: os-tempest-install.yml + when: (tempest_install | default(False)) | bool or (tempest_run | default(False)) | bool + diff --git a/compass-tasks/tacker_conf/tacker.yml b/compass-tasks/tacker_conf/tacker.yml new file mode 100644 index 0000000..9ceabbc --- /dev/null +++ b/compass-tasks/tacker_conf/tacker.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2017, SUSE Linux GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +component_skel: + tacker_server: + belongs_to: + - tacker_all + + +container_skel: + tacker_container: + belongs_to: + - mano_containers + contains: + - tacker_server + + +physical_skel: + mano_containers: + belongs_to: + - all_containers + mano_hosts: + belongs_to: + - hosts diff --git a/compass-tasks/tacker_conf/tacker_all.yml b/compass-tasks/tacker_conf/tacker_all.yml new file mode 100644 index 0000000..95d5b83 --- /dev/null +++ b/compass-tasks/tacker_conf/tacker_all.yml @@ -0,0 +1,46 @@ +--- +# Copyright 2017, SUSE LINUX GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tacker_service_user_name: tacker +tacker_service_tenant_name: service + +tacker_rabbitmq_userid: tacker +tacker_rabbitmq_vhost: /tacker +tacker_rabbitmq_port: "{{ rabbitmq_port }}" +tacker_rabbitmq_use_ssl: "{{ rabbitmq_use_ssl }}" +tacker_rabbitmq_servers: "{{ rabbitmq_servers }}" +tacker_rabbitmq_host_group: "{{ rabbitmq_host_group }}" + +tacker_service_publicuri: "{{ openstack_service_publicuri_proto|default(tacker_service_proto) }}://{{ external_lb_vip_address }}:{{ tacker_service_port }}" +tacker_service_adminurl: "{{ tacker_service_adminuri }}/" +tacker_service_region: "{{ service_region }}" +tacker_service_in_ldap: "{{ service_ldap_backend_enabled }}" + +tacker_aodh_enabled: "{{ groups['aodh_all'] is defined and groups['aodh_all'] | length > 0 }}" +tacker_gnocchi_enabled: "{{ groups['gnocchi_all'] is defined and groups['gnocchi_all'] | length > 0 }}" + +# NOTE: these and their swift_all.yml counterpart should be moved back to all.yml once swift with tacker gets proper SSL support +# swift_rabbitmq_telemetry_port: "{{ rabbitmq_port }}" +# swift_rabbitmq_telemetry_use_ssl: "{{ rabbitmq_use_ssl }}" + +# Ensure that the package state matches the global setting +tacker_package_state: "{{ package_state }}" + +# venv fetch configuration +tacker_venv_tag: "{{ venv_tag }}" +tacker_venv_download_url: "{{ venv_base_download_url }}/tacker-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz" + +# locations for fetching the default files from the git source +tacker_git_config_lookup_location: "{{ openstack_repo_url }}/openstackgit/tacker" diff --git a/compass-tasks/tacker_conf/user_secrets.yml b/compass-tasks/tacker_conf/user_secrets.yml new file mode 100644 index 0000000..b5a5796 --- /dev/null +++ b/compass-tasks/tacker_conf/user_secrets.yml @@ -0,0 +1,155 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################# WARNING ######################################## +# The playbooks do not currently manage changing passwords in an existing +# environment. Changing passwords and re-running the playbooks will fail +# and may break your OpenStack environment. +############################# WARNING ######################################## + + +## Rabbitmq Options +rabbitmq_cookie_token: +rabbitmq_monitoring_password: + +## Tokens +memcached_encryption_key: + +## Galera Options +galera_root_password: + +## Keystone Options +keystone_container_mysql_password: +keystone_auth_admin_password: +keystone_service_password: +keystone_rabbitmq_password: + +## Ceilometer Options: +ceilometer_container_db_password: +ceilometer_service_password: +ceilometer_telemetry_secret: +ceilometer_rabbitmq_password: + +## Aodh Options: +aodh_container_db_password: +aodh_service_password: +aodh_rabbitmq_password: + +## Cinder Options +cinder_container_mysql_password: +cinder_service_password: +cinder_profiler_hmac_key: +cinder_rabbitmq_password: + +## Ceph/rbd: a UUID to be used by libvirt to refer to the client.cinder user +cinder_ceph_client_uuid: + +## Glance Options +glance_container_mysql_password: +glance_service_password: +glance_profiler_hmac_key: +glance_rabbitmq_password: + +## Gnocchi Options: +gnocchi_container_mysql_password: +gnocchi_service_password: + +## Heat Options +heat_stack_domain_admin_password: +heat_container_mysql_password: +### THE HEAT AUTH KEY NEEDS TO BE 32 CHARACTERS LONG ## +heat_auth_encryption_key: +### THE HEAT AUTH KEY NEEDS TO BE 32 CHARACTERS LONG ## +heat_service_password: +heat_rabbitmq_password: + +## Ironic options +ironic_rabbitmq_password: +ironic_container_mysql_password: +ironic_service_password: +ironic_swift_temp_url_secret_key: + +## Horizon Options +horizon_container_mysql_password: +horizon_secret_key: + +## Neutron Options +neutron_container_mysql_password: +neutron_service_password: +neutron_rabbitmq_password: +neutron_ha_vrrp_auth_password: + +## Nova Options +nova_container_mysql_password: +nova_api_container_mysql_password: +nova_metadata_proxy_secret: +nova_service_password: +nova_rabbitmq_password: +nova_placement_service_password: +nova_placement_container_mysql_password: + +# LXD Options for nova compute +lxd_trust_password: + +## Sahara Options +sahara_container_mysql_password: +sahara_rabbitmq_password: +sahara_service_password: + +## Swift Options: +swift_service_password: +swift_dispersion_password: +### Once the swift cluster has been setup DO NOT change these hash values! +swift_hash_path_suffix: +swift_hash_path_prefix: +# Swift needs a telemetry password when using ceilometer +swift_rabbitmq_telemetry_password: + +## haproxy stats password +haproxy_stats_password: +haproxy_keepalived_authentication_password: + +## Magnum Options +magnum_service_password: +magnum_galera_password: +magnum_rabbitmq_password: +magnum_trustee_password: + +## Rally Options: +rally_galera_password: + +## Trove Options +trove_galera_password: +trove_rabbitmq_password: +trove_service_password: +trove_admin_user_password: +trove_regular_user_password: +trove_taskmanager_rpc_encr_key: +trove_inst_rpc_key_encr_key: + +## Barbican Options +barbican_galera_password: +barbican_rabbitmq_password: +barbican_service_password: + +## Designate Options +designate_galera_password: +designate_rabbitmq_password: +designate_service_password: + +# Tacker options +tacker_rabbitmq_password: +tacker_service_password: +tacker_container_mysql_password: diff --git a/compass-tasks/tasks/__init__.py b/compass-tasks/tasks/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/tasks/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/tasks/client.py b/compass-tasks/tasks/client.py deleted file mode 100644 index ca7ad14..0000000 --- a/compass-tasks/tasks/client.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to setup celery client. - - .. moduleauthor:: Xiaodong Wang - - .. note:: - If CELERY_CONFIG_MODULE is set in environment, load celery config from - the filename declared in CELERY_CONFIG_MODULE. -""" -import os - -from celery import Celery - - -celery = Celery(__name__) -if 'CELERY_CONFIG_MODULE' in os.environ: - celery.config_from_envvar('CELERY_CONFIG_MODULE') -else: - from compass.utils import celeryconfig_wrapper as celeryconfig - celery.config_from_object(celeryconfig) diff --git a/compass-tasks/tasks/tasks.py b/compass-tasks/tasks/tasks.py deleted file mode 100644 index f649afd..0000000 --- a/compass-tasks/tasks/tasks.py +++ /dev/null @@ -1,326 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to define celery tasks. - - .. moduleauthor:: Xiaodong Wang -""" -import logging - -from celery.signals import celeryd_init -from celery.signals import setup_logging - -from compass.actions import clean -from compass.actions import delete -from compass.actions import deploy -from compass.actions import install_callback -from compass.actions import patch -from compass.actions import poll_switch -from compass.actions import update_progress -from compass.db.api import adapter_holder as adapter_api -from compass.db.api import database -from compass.db.api import metadata_holder as metadata_api -from compass.log_analyzor import progress_calculator - -from compass.tasks.client import celery -from compass.utils import flags -from compass.utils import logsetting -from compass.utils import setting_wrapper as setting - - -@celeryd_init.connect() -def global_celery_init(**_): - """Initialization code.""" - flags.init() - flags.OPTIONS.logfile = setting.CELERY_LOGFILE - logsetting.init() - database.init() - adapter_api.load_adapters() - metadata_api.load_metadatas() - adapter_api.load_flavors() - progress_calculator.load_calculator_configurations() - - -@setup_logging.connect() -def tasks_setup_logging(**_): - """Setup logging options from compass setting.""" - flags.init() - flags.OPTIONS.logfile = setting.CELERY_LOGFILE - logsetting.init() - - -@celery.task(name='compass.tasks.pollswitch') -def pollswitch( - poller_email, ip_addr, credentials, - req_obj='mac', oper='SCAN' -): - """Query switch and return expected result. - - :param ip_addr: switch ip address. - :type ip_addr: str - :param credentials: switch credentials - :type credentials: dict - :param reqObj: the object requested to query from switch. - :type reqObj: str - :param oper: the operation to query the switch (SCAN, GET, SET). - :type oper: str - """ - try: - poll_switch.poll_switch( - poller_email, ip_addr, credentials, - req_obj=req_obj, oper=oper - ) - except Exception as error: - logging.exception(error) - - -@celery.task(name='compass.tasks.cluster_health') -def health_check(cluster_id, send_report_url, useremail): - """Verify the deployed cluster functionally works. - - :param cluster_id: ID of the cluster - :param send_report_url: The URL which reports should send back - """ - try: - deploy.health_check(cluster_id, send_report_url, useremail) - except Exception as error: - logging.exception(error) - - -@celery.task(name='compass.tasks.deploy_cluster') -def deploy_cluster(deployer_email, cluster_id, clusterhost_ids): - """Deploy the given cluster. - - :param cluster_id: id of the cluster - :type cluster_id: int - :param clusterhost_ids: the id of the hosts in the cluster - :type clusterhost_ids: list of int - """ - try: - deploy.deploy(cluster_id, clusterhost_ids, deployer_email) - except Exception as error: - logging.exception(error) - - -@celery.task(name='compass.tasks.redeploy_cluster') -def redeploy_cluster(deployer_email, cluster_id): - """Redeploy the given cluster. - - :param cluster_id: id of the cluster - :type cluster_id: int - """ - try: - deploy.redeploy(cluster_id, deployer_email) - except Exception as error: - logging.exception(error) - - -@celery.task(name='compass.tasks.patch_cluster') -def patch_cluster(patcher_email, cluster_id): - """Patch the existing cluster. - - :param cluster_id: id of the cluster - :type cluster_id: int - """ - try: - patch.patch(cluster_id, patcher_email) - except Exception as error: - logging.exception(error) - - -@celery.task(name='compass.tasks.reinstall_cluster') -def reinstall_cluster(installer_email, cluster_id, clusterhost_ids): - """reinstall the given cluster. - - :param cluster_id: id of the cluster - :type cluster_id: int - :param clusterhost_ids: the id of the hosts in the cluster - :type clusterhost_ids: list of int - """ - try: - deploy.redeploy(cluster_id, clusterhost_ids, installer_email) - except Exception as error: - logging.exception(error) - - -@celery.task(name='compass.tasks.delete_cluster') -def delete_cluster( - deleter_email, cluster_id, clusterhost_ids, - delete_underlying_host=False -): - """Delete the given cluster. - - :param cluster_id: id of the cluster - :type cluster_id: int - :param clusterhost_ids: the id of the hosts in the cluster - :type clusterhost_ids: list of int - """ - try: - delete.delete_cluster( - cluster_id, clusterhost_ids, deleter_email, - delete_underlying_host=delete_underlying_host - ) - except Exception as error: - logging.exception(error) - - -@celery.task(name='compass.tasks.delete_cluster_host') -def delete_cluster_host( - deleter_email, cluster_id, host_id, - delete_underlying_host=False -): - """Delte the given cluster host. - - :param cluster_id: id of the cluster - :type cluster_id: int - :param host_id: id of the host - :type host_id: int - """ - try: - delete.delete_cluster_host( - cluster_id, host_id, deleter_email, - delete_underlying_host=delete_underlying_host - ) - except Exception as error: - logging.exception(error) - - -@celery.task(name='compass.tasks.delete_host') -def delete_host(deleter_email, host_id, cluster_ids): - """Delete the given host. - - :param host_id: id of the host - :type host_id: int - :param cluster_ids: list of cluster id - :type cluster_ids: list of int - """ - try: - delete.delete_host( - host_id, cluster_ids, deleter_email - ) - except Exception as error: - logging.exception(error) - - -@celery.task(name='compass.tasks.clean_os_installer') -def clean_os_installer( - os_installer_name, os_installer_settings -): - """Clean os installer.""" - try: - clean.clean_os_installer( - os_installer_name, os_installer_settings - ) - except Exception as error: - logging.excception(error) - - -@celery.task(name='compass.tasks.clean_package_installer') -def clean_package_installer( - package_installer_name, package_installer_settings -): - """Clean package installer.""" - try: - clean.clean_package_installer( - package_installer_name, package_installer_settings - ) - except Exception as error: - logging.excception(error) - - -@celery.task(name='compass.tasks.poweron_host') -def poweron_host(host_id): - """Deploy the given cluster.""" - pass - - -@celery.task(name='compass.tasks.poweroff_host') -def poweroff_host(host_id): - """Deploy the given cluster.""" - pass - - -@celery.task(name='compass.tasks.reset_host') -def reset_host(host_id): - """Deploy the given cluster.""" - pass - - -@celery.task(name='compass.tasks.poweron_machine') -def poweron_machine(machine_id): - """Deploy the given cluster.""" - pass - - -@celery.task(name='compass.tasks.poweroff_machine') -def poweroff_machine(machine_id): - """Deploy the given cluster.""" - pass - - -@celery.task(name='compass.tasks.reset_machine') -def reset_machine(machine_id): - """Deploy the given cluster.""" - pass - - -@celery.task(name='compass.tasks.os_installed') -def os_installed( - host_id, clusterhosts_ready, - clusters_os_ready -): - """callback when os is installed.""" - try: - install_callback.os_installed( - host_id, clusterhosts_ready, - clusters_os_ready - ) - except Exception as error: - logging.exception(error) - - -@celery.task(name='compass.tasks.package_installed') -def package_installed( - cluster_id, host_id, cluster_ready, host_ready -): - """callback when package is installed.""" - try: - install_callback.package_installed( - cluster_id, host_id, cluster_ready, host_ready - ) - except Exception as error: - logging.exception(error) - - -@celery.task(name='compass.tasks.cluster_installed') -def cluster_installed( - cluster_id, clusterhosts_ready -): - """callback when package is installed.""" - try: - install_callback.cluster_installed( - cluster_id, clusterhosts_ready - ) - except Exception as error: - logging.exception(error) - - -@celery.task(name='compass.tasks.update_progress') -def update_clusters_progress(): - """Calculate the installing progress of the given cluster.""" - logging.info('update_clusters_progress') - try: - update_progress.update_progress() - except Exception as error: - logging.exception(error) diff --git a/compass-tasks/utils/__init__.py b/compass-tasks/utils/__init__.py deleted file mode 100644 index 4ee55a4..0000000 --- a/compass-tasks/utils/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/utils/celeryconfig_wrapper.py b/compass-tasks/utils/celeryconfig_wrapper.py deleted file mode 100644 index b6644ba..0000000 --- a/compass-tasks/utils/celeryconfig_wrapper.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""celeryconfig wrapper. - - .. moduleauthor:: Xiaodong Wang -""" -import logging -import os.path -import urllib - -from compass.utils import setting_wrapper as setting - - -# CELERY_RESULT_BACKEND = 'amqp://' - -# BROKER_URL = 'amqp://guest:guest@localhost:5672//' - - -CELERY_IMPORTS = ('compass.tasks.tasks',) - - -if setting.CELERYCONFIG_FILE: - CELERY_CONFIG = os.path.join( - str(setting.CELERYCONFIG_DIR), - str(setting.CELERYCONFIG_FILE)) - - try: - logging.info('load celery config from %s', CELERY_CONFIG) - execfile(CELERY_CONFIG, globals(), locals()) - except Exception as error: - logging.exception(error) - raise error diff --git a/compass-tasks/utils/flags.py b/compass-tasks/utils/flags.py deleted file mode 100644 index a3169f5..0000000 --- a/compass-tasks/utils/flags.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to load flags. - - .. moduleauthor:: Xiaodong Wang -""" -import sys - -from optparse import OptionParser - - -class Flags(object): - """Class to store flags.""" - - PARSER = OptionParser() - PARSED_OPTIONS = None - - @classmethod - def parse_args(cls): - """parse args.""" - (options, argv) = Flags.PARSER.parse_args() - sys.argv = [sys.argv[0]] + argv - Flags.PARSED_OPTIONS = options - - def __getattr__(self, name): - if Flags.PARSED_OPTIONS and hasattr(Flags.PARSED_OPTIONS, name): - return getattr(Flags.PARSED_OPTIONS, name) - - for option in Flags.PARSER.option_list: - if option.dest == name: - return option.default - - raise AttributeError('Option instance has no attribute %s' % name) - - def __setattr__(self, name, value): - if Flags.PARSED_OPTIONS and hasattr(Flags.PARSED_OPTIONS, name): - setattr(Flags.PARSED_OPTIONS, name, value) - return - - for option in Flags.PARSER.option_list: - if option.dest == name: - option.default = value - return - - object.__setattr__(self, name, value) - - -OPTIONS = Flags() - - -def init(): - """Init flag parsing.""" - OPTIONS.parse_args() - - -def add(flagname, **kwargs): - """Add a flag name and its setting. - - :param flagname: flag name declared in cmd as --=... - :type flagname: str - """ - Flags.PARSER.add_option('--%s' % flagname, - dest=flagname, **kwargs) - - -def add_bool(flagname, default=True, **kwargs): - """Add a bool flag name and its setting. - - :param flagname: flag name declared in cmd as --[no]. - :type flagname: str - :param default: default value - :type default: bool - """ - Flags.PARSER.add_option('--%s' % flagname, - dest=flagname, default=default, - action="store_true", **kwargs) - Flags.PARSER.add_option('--no%s' % flagname, - dest=flagname, - action="store_false", **kwargs) diff --git a/compass-tasks/utils/logsetting.py b/compass-tasks/utils/logsetting.py deleted file mode 100644 index 836ebcb..0000000 --- a/compass-tasks/utils/logsetting.py +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to setup logging configuration. - - .. moduleauthor:: Xiaodong Wang -""" - -import logging -import logging.handlers -import os -import os.path -import sys - -from compass.utils import flags -from compass.utils import setting_wrapper as setting - - -flags.add('loglevel', - help='logging level', default=setting.DEFAULT_LOGLEVEL) -flags.add('logdir', - help='logging directory', default=setting.DEFAULT_LOGDIR) -flags.add('logfile', - help='logging filename', default=None) -flags.add('log_interval', type='int', - help='log interval', default=setting.DEFAULT_LOGINTERVAL) -flags.add('log_interval_unit', - help='log interval unit', default=setting.DEFAULT_LOGINTERVAL_UNIT) -flags.add('log_format', - help='log format', default=setting.DEFAULT_LOGFORMAT) -flags.add('log_backup_count', type='int', - help='log backup count', default=setting.DEFAULT_LOGBACKUPCOUNT) - - -# mapping str setting in flag --loglevel to logging level. -LOGLEVEL_MAPPING = { - 'finest': logging.DEBUG - 2, # more detailed log. - 'fine': logging.DEBUG - 1, # detailed log. - 'debug': logging.DEBUG, - 'info': logging.INFO, - 'warning': logging.WARNING, - 'error': logging.ERROR, - 'critical': logging.CRITICAL, -} - - -logging.addLevelName(LOGLEVEL_MAPPING['fine'], 'fine') -logging.addLevelName(LOGLEVEL_MAPPING['finest'], 'finest') - - -# disable logging when logsetting.init not called -logging.getLogger().setLevel(logging.CRITICAL) - - -def getLevelByName(level_name): - """Get log level by level name.""" - return LOGLEVEL_MAPPING[level_name] - - -def init(): - """Init loggsetting. It should be called after flags.init.""" - loglevel = flags.OPTIONS.loglevel.lower() - logdir = flags.OPTIONS.logdir - logfile = flags.OPTIONS.logfile - logger = logging.getLogger() - if logger.handlers: - for handler in logger.handlers: - logger.removeHandler(handler) - - if logdir: - if not logfile: - logfile = '%s.log' % os.path.basename(sys.argv[0]) - - handler = logging.handlers.TimedRotatingFileHandler( - os.path.join(logdir, logfile), - when=flags.OPTIONS.log_interval_unit, - interval=flags.OPTIONS.log_interval, - backupCount=flags.OPTIONS.log_backup_count) - else: - if not logfile: - handler = logging.StreamHandler(sys.stderr) - else: - handler = logging.handlers.TimedRotatingFileHandler( - logfile, - when=flags.OPTIONS.log_interval_unit, - interval=flags.OPTIONS.log_interval, - backupCount=flags.OPTIONS.log_backup_count) - - if loglevel in LOGLEVEL_MAPPING: - logger.setLevel(LOGLEVEL_MAPPING[loglevel]) - handler.setLevel(LOGLEVEL_MAPPING[loglevel]) - - formatter = logging.Formatter( - flags.OPTIONS.log_format) - - handler.setFormatter(formatter) - logger.addHandler(handler) diff --git a/compass-tasks/utils/setting_wrapper.py b/compass-tasks/utils/setting_wrapper.py deleted file mode 100644 index 0b3e9f7..0000000 --- a/compass-tasks/utils/setting_wrapper.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""comapss setting wrapper. - - .. moduleauthor:: Xiaodong Wang ,xiaodongwang@huawei.com> -""" -import datetime -import lazypy -import logging -import os -import os.path - - -# default setting -CONFIG_DIR = os.environ.get('COMPASS_CONFIG_DIR', '/etc/compass') -SQLALCHEMY_DATABASE_URI = 'sqlite://' -SQLALCHEMY_DATABASE_POOL_TYPE = 'static' -COBBLER_INSTALLATION_LOGDIR = '/var/log/cobbler/anamon' -CHEF_INSTALLATION_LOGDIR = '/var/log/chef' -INSTALLATION_LOGDIR = { - 'CobblerInstaller': COBBLER_INSTALLATION_LOGDIR, - 'ChefInstaller': CHEF_INSTALLATION_LOGDIR -} -CLUSTERHOST_INATALLATION_LOGDIR_NAME = 'name' -HOST_INSTALLATION_LOGDIR_NAME = 'name' -DEFAULT_LOGLEVEL = 'debug' -DEFAULT_LOGDIR = '/tmp' -DEFAULT_LOGINTERVAL = 1 -DEFAULT_LOGINTERVAL_UNIT = 'h' -DEFAULT_LOGFORMAT = ( - '%(asctime)s - %(filename)s - %(lineno)d - %(levelname)s - %(message)s') -DEFAULT_LOGBACKUPCOUNT = 5 -WEB_LOGFILE = '' -CELERY_LOGFILE = '' -CELERYCONFIG_DIR = lazypy.delay(lambda: CONFIG_DIR) -CELERYCONFIG_FILE = '' -PROGRESS_UPDATE_INTERVAL = 30 -POLLSWITCH_INTERVAL = 60 -SWITCHES = [ -] - -USER_AUTH_HEADER_NAME = 'X-Auth-Token' -USER_TOKEN_DURATION = '2h' -COMPASS_ADMIN_EMAIL = 'admin@huawei.com' -COMPASS_ADMIN_PASSWORD = 'admin' -COMPASS_DEFAULT_PERMISSIONS = [ - 'list_permissions', -] -SWITCHES_DEFAULT_FILTERS = [] -DEFAULT_SWITCH_IP = '0.0.0.0' -DEFAULT_SWITCH_PORT = 0 - -COMPASS_SUPPORTED_PROXY = 'http://127.0.0.1:3128' -COMPASS_SUPPORTED_DEFAULT_NOPROXY = ['127.0.0.1'] -COMPASS_SUPPORTED_NTP_SERVER = '127.0.0.1' -COMPASS_SUPPORTED_DNS_SERVERS = ['127.0.0.1'] -COMPASS_SUPPORTED_DOMAINS = [] -COMPASS_SUPPORTED_DEFAULT_GATEWAY = '127.0.0.1' -COMPASS_SUPPORTED_LOCAL_REPO = 'http://127.0.0.1' - -PROGRESS_UPDATE_PID_FILE = '/var/run/progress_update.pid' - -PROXY_URL_PREFIX = 'http://10.145.81.205:5000' - -OS_INSTALLER_DIR = '' -PACKAGE_INSTALLER_DIR = '' -OS_DIR = '' -ADAPTER_DIR = '' -OS_METADATA_DIR = '' -PACKAGE_METADATA_DIR = '' -FLAVOR_METADATA_DIR = '' -OS_FIELD_DIR = '' -PACKAGE_FIELD_DIR = '' -FLAVOR_FIELD_DIR = '' -ADAPTER_ROLE_DIR = '' -ADAPTER_FLAVOR_DIR = '' -VALIDATOR_DIR = '' -CALLBACK_DIR = '' -TMPL_DIR = '' -MACHINE_LIST_DIR = '' -PROGRESS_CALCULATOR_DIR = '' -OS_MAPPING_DIR = '' -FLAVOR_MAPPING_DIR = '' -PLUGINS_DIR = '' - -if ( - 'COMPASS_IGNORE_SETTING' in os.environ and - os.environ['COMPASS_IGNORE_SETTING'] -): - pass -else: - if 'COMPASS_SETTING' in os.environ: - SETTING = os.environ['COMPASS_SETTING'] - else: - SETTING = '/etc/compass/setting' - - try: - logging.info('load setting from %s', SETTING) - execfile(SETTING, globals(), locals()) - except Exception as error: - logging.exception(error) - raise error - -if not OS_INSTALLER_DIR: - OS_INSTALLER_DIR = os.path.join(CONFIG_DIR, 'os_installer') - -if not PACKAGE_INSTALLER_DIR: - PACKAGE_INSTALLER_DIR = os.path.join(CONFIG_DIR, 'package_installer') - -if not OS_DIR: - OS_DIR = os.path.join(CONFIG_DIR, 'os') - -if not ADAPTER_DIR: - ADAPTER_DIR = os.path.join(CONFIG_DIR, 'adapter') - -if not OS_METADATA_DIR: - OS_METADATA_DIR = os.path.join(CONFIG_DIR, 'os_metadata') - -if not PACKAGE_METADATA_DIR: - PACKAGE_METADATA_DIR = os.path.join(CONFIG_DIR, 'package_metadata') - -if not FLAVOR_METADATA_DIR: - FLAVOR_METADATA_DIR = os.path.join(CONFIG_DIR, 'flavor_metadata') - -if not OS_FIELD_DIR: - OS_FIELD_DIR = os.path.join(CONFIG_DIR, 'os_field') - -if not PACKAGE_FIELD_DIR: - PACKAGE_FIELD_DIR = os.path.join(CONFIG_DIR, 'package_field') - -if not FLAVOR_FIELD_DIR: - FLAVOR_FIELD_DIR = os.path.join(CONFIG_DIR, 'flavor_field') - -if not ADAPTER_ROLE_DIR: - ADAPTER_ROLE_DIR = os.path.join(CONFIG_DIR, 'role') - -if not ADAPTER_FLAVOR_DIR: - ADAPTER_FLAVOR_DIR = os.path.join(CONFIG_DIR, 'flavor') - -if not VALIDATOR_DIR: - VALIDATOR_DIR = os.path.join(CONFIG_DIR, 'validator') - -if not CALLBACK_DIR: - CALLBACK_DIR = os.path.join(CONFIG_DIR, 'callback') - -if not TMPL_DIR: - TMPL_DIR = os.path.join(CONFIG_DIR, 'templates') - -if not MACHINE_LIST_DIR: - MACHINE_LIST_DIR = os.path.join(CONFIG_DIR, 'machine_list') - -if not PROGRESS_CALCULATOR_DIR: - PROGRESS_CALCULATOR_DIR = os.path.join(CONFIG_DIR, 'progress_calculator') - -if not OS_MAPPING_DIR: - OS_MAPPING_DIR = os.path.join(CONFIG_DIR, 'os_mapping') - -if not FLAVOR_MAPPING_DIR: - FLAVOR_MAPPING_DIR = os.path.join(CONFIG_DIR, 'flavor_mapping') - -if not PLUGINS_DIR: - PLUGINS_DIR = os.environ.get('COMPASS_PLUGINS_DIR', - os.path.join(CONFIG_DIR, 'plugins')) diff --git a/compass-tasks/utils/util.py b/compass-tasks/utils/util.py deleted file mode 100644 index 39978ca..0000000 --- a/compass-tasks/utils/util.py +++ /dev/null @@ -1,395 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module to provider util functions in all compass code - - .. moduleauthor:: Xiaodong Wang -""" - -import crypt -import datetime -import logging -import os -import os.path -import re -import setting_wrapper as setting -import sys -import warnings - - -def deprecated(func): - """This is a decorator which can be used to mark functions as deprecated. - - It will result in a warning being emitted when the function is used. - """ - def new_func(*args, **kwargs): - warnings.warn( - "Call to deprecated function %s." % func.__name__, - category=DeprecationWarning - ) - return func(*args, **kwargs) - - new_func.__name__ = func.__name__ - new_func.__doc__ = func.__doc__ - new_func.__dict__.update(func.__dict__) - return new_func - - -def parse_datetime(date_time, exception_class=Exception): - """Parse datetime str to get datetime object. - - The date time format is %Y-%m-%d %H:%M:%S - """ - try: - return datetime.datetime.strptime( - date_time, '%Y-%m-%d %H:%M:%S' - ) - except Exception as error: - logging.exception(error) - raise exception_class( - 'date time %s format is invalid' % date_time - ) - - -def parse_datetime_range(date_time_range, exception_class=Exception): - """parse datetime range str to pair of datetime objects. - - The date time range format is %Y-%m-%d %H:%M:%S,%Y-%m-%d %H:%M:%S - """ - try: - start, end = date_time_range.split(',') - except Exception as error: - logging.exception(error) - raise exception_class( - 'there is no `,` in date time range %s' % date_time_range - ) - if start: - start_datetime = parse_datetime(start, exception_class) - else: - start_datetime = None - if end: - end_datetime = parse_datetime(end, exception_class) - else: - end_datetime = None - return start_datetime, end_datetime - - -def parse_request_arg_dict(arg, exception_class=Exception): - """parse string to dict. - - The str is formatted like a=b;c=d and parsed to - {'a': 'b', 'c': 'd'} - """ - arg_dict = {} - arg_pairs = arg.split(';') - for arg_pair in arg_pairs: - try: - arg_name, arg_value = arg_pair.split('=', 1) - except Exception as error: - logging.exception(error) - raise exception_class( - 'there is no `=` in %s' % arg_pair - ) - arg_dict[arg_name] = arg_value - return arg_dict - - -def format_datetime(date_time): - """Generate string from datetime object.""" - return date_time.strftime("%Y-%m-%d %H:%M:%S") - - -def merge_dict(lhs, rhs, override=True): - """Merge nested right dict into left nested dict recursively. - - :param lhs: dict to be merged into. - :type lhs: dict - :param rhs: dict to merge from. - :type rhs: dict - :param override: the value in rhs overide the value in left if True. - :type override: boolean - """ - if not isinstance(lhs, dict) or not isinstance(rhs, dict): - if override: - return rhs - else: - return lhs - - for key, value in rhs.items(): - if key not in lhs: - lhs[key] = rhs[key] - else: - lhs[key] = merge_dict(lhs[key], value, override) - - return lhs - - -def recursive_merge_dict(name, all_dicts, parents): - """Recursively merge parent dict into base dict.""" - parent_name = parents.get(name, None) - base_dict = all_dicts.get(name, {}) - if not parent_name: - return base_dict - merged = recursive_merge_dict(parent_name, all_dicts, parents) - return merge_dict(base_dict, merged, override=False) - - -def encrypt(value, crypt_method=None): - """Get encrypted value.""" - if not crypt_method: - if hasattr(crypt, 'METHOD_MD5'): - crypt_method = crypt.METHOD_MD5 - else: - # for python2.7, copy python2.6 METHOD_MD5 logic here. - from random import choice - import string - - _saltchars = string.ascii_letters + string.digits + './' - - def _mksalt(): - """generate salt.""" - salt = '$1$' - salt += ''.join(choice(_saltchars) for _ in range(8)) - return salt - - crypt_method = _mksalt() - - return crypt.crypt(value, crypt_method) - - -def parse_time_interval(time_interval_str): - """parse string of time interval to time interval. - - supported time interval unit: ['d', 'w', 'h', 'm', 's'] - Examples: - time_interval_str: '3d 2h' time interval to 3 days and 2 hours. - """ - if not time_interval_str: - return 0 - - time_interval_tuple = [ - time_interval_element - for time_interval_element in time_interval_str.split(' ') - if time_interval_element - ] - time_interval_dict = {} - time_interval_unit_mapping = { - 'd': 'days', - 'w': 'weeks', - 'h': 'hours', - 'm': 'minutes', - 's': 'seconds' - } - for time_interval_element in time_interval_tuple: - mat = re.match(r'^([+-]?\d+)(w|d|h|m|s).*', time_interval_element) - if not mat: - continue - - time_interval_value = int(mat.group(1)) - time_interval_unit = time_interval_unit_mapping[mat.group(2)] - time_interval_dict[time_interval_unit] = ( - time_interval_dict.get(time_interval_unit, 0) + time_interval_value - ) - - time_interval = datetime.timedelta(**time_interval_dict) - if sys.version_info[0:2] > (2, 6): - return time_interval.total_seconds() - else: - return ( - time_interval.microseconds + ( - time_interval.seconds + time_interval.days * 24 * 3600 - ) * 1e6 - ) / 1e6 - - -def get_plugins_config_files(name, suffix=".conf"): - """walk through each of plugin to find all the config files in the""" - """name directory""" - - plugins_path = setting.PLUGINS_DIR - files = [] - if os.path.exists(plugins_path): - for plugin in os.listdir(plugins_path): - plugin_path = os.path.join(plugins_path, plugin) - plugin_config = os.path.join(plugin_path, name) - if os.path.exists(plugin_config): - for component in os.listdir(plugin_config): - if not component.endswith(suffix): - continue - files.append(os.path.join(plugin_config, component)) - return files - - -def load_configs( - config_dir, config_name_suffix='.conf', - env_globals={}, env_locals={} -): - """Load configurations from config dir.""" - """The config file could be in the config_dir or in plugins config_dir""" - """The plugins config_dir is formed as, for example /etc/compass/adapter""" - """Then the plugins config_dir is /etc/compass/plugins/xxx/adapter""" - - # TODO(Carl) instead of using config_dir, it should use a name such as - # adapter etc, however, doing it requires a lot client sites changes, - # will do it later. - - configs = [] - config_files = [] - config_dir = str(config_dir) - - """search for config_dir""" - if os.path.exists(config_dir): - for component in os.listdir(config_dir): - if not component.endswith(config_name_suffix): - continue - config_files.append(os.path.join(config_dir, component)) - - """search for plugins config_dir""" - index = config_dir.rfind("/") - - config_files.extend(get_plugins_config_files(config_dir[index + 1:], - config_name_suffix)) - - if not config_files: - logging.error('path %s and plugins does not exist', config_dir) - for path in config_files: - logging.debug('load config from %s', path) - config_globals = {} - config_globals.update(env_globals) - config_locals = {} - config_locals.update(env_locals) - try: - execfile(path, config_globals, config_locals) - except Exception as error: - logging.exception(error) - raise error - configs.append(config_locals) - return configs - - -def pretty_print(*contents): - """pretty print contents.""" - if len(contents) == 0: - print "" - else: - print "\n".join(content for content in contents) - - -def get_switch_machines_from_file(filename): - """get switch machines from file.""" - switches = [] - switch_machines = {} - with open(filename) as switch_file: - for line in switch_file: - line = line.strip() - if not line: - # ignore empty line - continue - - if line.startswith('#'): - # ignore comments - continue - - columns = [column for column in line.split(',')] - if not columns: - # ignore empty line - continue - - if columns[0] == 'switch': - (switch_ip, switch_vendor, switch_version, - switch_community, switch_state) = columns[1:] - switches.append({ - 'ip': switch_ip, - 'vendor': switch_vendor, - 'credentials': { - 'version': switch_version, - 'community': switch_community, - }, - 'state': switch_state, - }) - elif columns[0] == 'machine': - switch_ip, switch_port, mac = columns[1:] - switch_machines.setdefault(switch_ip, []).append({ - 'mac': mac, - 'port': switch_port, - }) - - return (switches, switch_machines) - - -def execute_cli_by_ssh(cmd, host, username, password=None, - keyfile='/root/.ssh/id_rsa', nowait=False): - """SSH to execute script on remote machine - - :param host: ip of the remote machine - :param username: username to access the remote machine - :param password: password to access the remote machine - :param cmd: command to execute - - """ - if not cmd: - logging.error("No command found!") - raise Exception('No command found!') - - if nowait: - cmd = "nohup %s >/dev/null 2>&1 &" % cmd - - stdin = None - stdout = None - stderr = None - try: - import paramiko - from paramiko import ssh_exception - - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - if password: - client.connect(host, username=username, password=password) - else: - client.load_system_host_keys() - client.connect( - host, username=username, - key_filename=keyfile, look_for_keys=True - ) - stdin, stdout, stderr = client.exec_command(cmd) - result = stdout.readlines() - logging.info("result of command '%s' is '%s'!" % (cmd, result)) - return result - - except ImportError: - err_msg = "Cannot find Paramiko package!" - logging.error(err_msg) - raise ImportError(err_msg) - - except (ssh_exception.BadHostKeyException, - ssh_exception.AuthenticationException, - ssh_exception.SSHException): - - err_msg = 'SSH connection error or command execution failed!' - logging.error(err_msg) - raise Exception(err_msg) - - except Exception as exc: - logging.error( - 'Failed to execute command "%s", exception is %s' % (cmd, exc) - ) - raise Exception(exc) - - finally: - for resource in [stdin, stdout, stderr]: - if resource: - resource.close() - - client.close() -- cgit 1.2.3-korg