From c8440f4158805fed1c49d1f06cd71f324eac26a2 Mon Sep 17 00:00:00 2001 From: Koren Lev Date: Mon, 18 Dec 2017 19:16:16 +0200 Subject: release 1.2 + new tagging Change-Id: I1e876451ec4a330f458dd57adadb15e39969b225 Signed-off-by: Koren Lev (cherry picked from commit 98c3ac7c859e34fe60d061b9ca591aba429e4118) --- app/api/responders/resource/clique_types.py | 140 +- app/api/responders/resource/environment_configs.py | 76 +- app/api/responders/responder_base.py | 6 +- app/api/validation/data_validate.py | 4 +- app/config/link_finders.json | 3 +- app/config/scanners.json | 25 +- app/connection_test/connection_test.py | 283 ----- app/discover/clique_finder.py | 138 +- app/discover/event_manager.py | 4 +- app/discover/fetchers/api/api_access.py | 61 +- .../fetchers/api/api_fetch_host_instances.py | 2 +- .../fetchers/api/api_fetch_project_hosts.py | 44 +- app/discover/fetchers/api/api_fetch_regions.py | 2 +- app/discover/fetchers/db/db_access.py | 29 +- app/discover/fetchers/kube/__init__.py | 9 + app/discover/fetchers/kube/kube_access.py | 28 + .../fetchers/kube/kube_fetch_namespaces.py | 32 + app/discover/link_finders/find_implicit_links.py | 128 ++ app/discover/link_finders/find_links.py | 3 + .../link_finders/find_links_for_instance_vnics.py | 2 + app/discover/scan_manager.py | 125 +- app/discover/scan_metadata_parser.py | 28 +- app/discover/scanner.py | 21 +- app/install/calipso-installer.py | 21 +- app/install/db/clique_types.json | 26 + app/install/db/constants.json | 33 + app/install/db/environments_config.json | 94 +- app/install/db/monitoring_config_templates.json | 51 + app/install/db/supported_environments.json | 15 + .../checks/check_instance_communictions.py | 85 ++ app/monitoring/checks/check_vconnector.py | 50 + app/monitoring/handlers/handle_vconnector.py | 28 + app/monitoring/handlers/monitor.py | 83 +- .../handlers/monitoring_check_handler.py | 4 +- app/monitoring/setup/monitoring_check_handler.py | 15 +- app/monitoring/setup/monitoring_host.py | 11 +- app/monitoring/setup/monitoring_instance.py | 67 + app/monitoring/setup/monitoring_setup_manager.py | 4 + app/monitoring/setup/monitoring_vconnector.py | 24 + app/monitoring/setup/sensu_client_installer.py | 158 +++ .../responders_test/resource/test_clique_types.py | 108 +- .../resource/test_environment_configs.py | 311 ++--- app/test/api/responders_test/test_data/base.py | 71 +- .../api/responders_test/test_data/clique_types.py | 201 ++- .../test_data/environment_configs.py | 6 +- app/test/api/test_base.py | 28 +- app/test/fetch/api_fetch/test_api_access.py | 73 +- .../api_fetch/test_api_fetch_project_hosts.py | 43 +- .../test_data/api_fetch_host_project_hosts.py | 21 + app/test/fetch/link_finders/__init__.py | 9 + app/test/fetch/link_finders/test_data/__init__.py | 9 + .../test_data/test_find_implicit_links.py | 303 +++++ .../fetch/link_finders/test_find_implicit_links.py | 107 ++ app/test/scan/test_data/configurations.py | 1 + app/test/scan/test_data/scanner.py | 13 + app/test/scan/test_scan_metadata_parser.py | 2 + app/test/scan/test_scanner.py | 8 +- app/test/verify.sh | 4 + app/utils/api_access_base.py | 51 + app/utils/inventory_mgr.py | 26 +- app/utils/mongo_access.py | 6 +- app/utils/ssh_connection.py | 6 +- app/utils/util.py | 9 +- docs/release/Calipso-usage-stories.rst | 446 +++++++ docs/release/apex-scenario-guide.rst | 282 +++++ docs/release/developer-guide.pdf | Bin 0 -> 252310 bytes docs/release/developer-guide.rst | 1338 ++++++++++++++++++++ docs/release/media/image101.png | Bin 0 -> 119090 bytes docs/release/media/image102.png | Bin 0 -> 104849 bytes docs/release/media/image103.png | Bin 0 -> 10664 bytes docs/release/media/image104.png | Bin 0 -> 37854 bytes docs/release/media/image105.png | Bin 0 -> 23555 bytes docs/release/media/image106.png | Bin 0 -> 58686 bytes docs/release/media/image107.png | Bin 0 -> 89583 bytes 74 files changed, 4555 insertions(+), 889 deletions(-) delete mode 100644 app/connection_test/connection_test.py create mode 100644 app/discover/fetchers/kube/__init__.py create mode 100644 app/discover/fetchers/kube/kube_access.py create mode 100644 app/discover/fetchers/kube/kube_fetch_namespaces.py create mode 100644 app/discover/link_finders/find_implicit_links.py create mode 100644 app/monitoring/checks/check_instance_communictions.py create mode 100644 app/monitoring/checks/check_vconnector.py create mode 100644 app/monitoring/handlers/handle_vconnector.py create mode 100644 app/monitoring/setup/monitoring_instance.py create mode 100644 app/monitoring/setup/monitoring_vconnector.py create mode 100644 app/monitoring/setup/sensu_client_installer.py create mode 100644 app/test/fetch/link_finders/__init__.py create mode 100644 app/test/fetch/link_finders/test_data/__init__.py create mode 100644 app/test/fetch/link_finders/test_data/test_find_implicit_links.py create mode 100644 app/test/fetch/link_finders/test_find_implicit_links.py create mode 100644 app/utils/api_access_base.py create mode 100644 docs/release/Calipso-usage-stories.rst create mode 100644 docs/release/apex-scenario-guide.rst create mode 100644 docs/release/developer-guide.pdf create mode 100644 docs/release/developer-guide.rst create mode 100644 docs/release/media/image101.png create mode 100644 docs/release/media/image102.png create mode 100644 docs/release/media/image103.png create mode 100644 docs/release/media/image104.png create mode 100644 docs/release/media/image105.png create mode 100644 docs/release/media/image106.png create mode 100644 docs/release/media/image107.png diff --git a/app/api/responders/resource/clique_types.py b/app/api/responders/resource/clique_types.py index ff42f8c..e2e9e71 100644 --- a/app/api/responders/resource/clique_types.py +++ b/app/api/responders/resource/clique_types.py @@ -21,31 +21,53 @@ class CliqueTypes(ResponderBase): "focal_point_type": True, "link_types": True, "environment": True, - "name": True + "name": True, + "distribution": True, + "distribution_version": True, + "mechanism_drivers": True, + "type_drivers": True, + "use_implicit_links": True } RESERVED_NAMES = ["ANY"] + def __init__(self): + super().__init__() + self.focal_point_types = self.get_constants_by_name("object_types") + self.link_types = self.get_constants_by_name("link_types") + self.mechanism_drivers = self.get_constants_by_name("mechanism_drivers") + self.type_drivers = self.get_constants_by_name("type_drivers") + def on_get(self, req, resp): self.log.debug("Getting clique types") filters = self.parse_query_params(req) - focal_point_types = self.get_constants_by_name("object_types") - link_types = self.get_constants_by_name("link_types") filters_requirements = { - 'env_name': self.require(str, mandatory=True), + 'env_name': self.require(str), 'id': self.require(ObjectId, convert_to_type=True), + 'distribution': self.require(str), + 'distribution_version': self.require(str), + 'mechanism_drivers': self.require(str, + validate=DataValidate.LIST, + requirement=self.mechanism_drivers), + 'type_drivers': self.require(str, + validate=DataValidate.LIST, + requirement=self.type_drivers), 'focal_point_type': self.require(str, validate=DataValidate.LIST, - requirement=focal_point_types), + requirement=self.focal_point_types), 'link_type': self.require([list, str], validate=DataValidate.LIST, - requirement=link_types), + requirement=self.link_types), 'name': self.require(str), 'page': self.require(int, convert_to_type=True), 'page_size': self.require(int, convert_to_type=True) } self.validate_query_data(filters, filters_requirements) + if 'distribution_version' in filters and 'distribution' not in filters: + self.bad_request("Distribution version without distribution " + "is not allowed") + page, page_size = self.get_pagination(filters) query = self.build_query(filters) if self.ID in query: @@ -64,40 +86,46 @@ class CliqueTypes(ResponderBase): error, clique_type = self.get_content_from_request(req) if error: self.bad_request(error) - focal_point_types = self.get_constants_by_name("object_types") - link_types = self.get_constants_by_name("link_types") + clique_type_requirements = { - 'environment': self.require(str, mandatory=True), + 'environment': self.require(str), 'focal_point_type': self.require(str, mandatory=True, validate=DataValidate.LIST, - requirement=focal_point_types), + requirement=self.focal_point_types), 'link_types': self.require(list, mandatory=True, validate=DataValidate.LIST, - requirement=link_types), - 'name': self.require(str, mandatory=True) + requirement=self.link_types), + 'name': self.require(str, mandatory=True), + 'distribution': self.require(str), + 'distribution_version': self.require(str), + 'mechanism_drivers': self.require(str, + validate=DataValidate.LIST, + requirement=self.mechanism_drivers), + 'type_drivers': self.require(str, + validate=DataValidate.LIST, + requirement=self.type_drivers), + 'use_implicit_links': self.require(bool) } self.validate_query_data(clique_type, clique_type_requirements) - - env_name = clique_type['environment'] - if not self.check_environment_name(env_name): - self.bad_request("Unknown environment: {}".format(env_name)) - elif env_name.upper() in self.RESERVED_NAMES: - self.bad_request("Environment name '{}' is reserved".format(env_name)) + self.validate_required_fields(clique_type) + self.validate_focal_point_type(clique_type) + self.validate_duplicate_configuration(clique_type) self.write(clique_type, self.COLLECTION) self.set_successful_response(resp, - {"message": "created a new clique_type " - "for environment {0}" - .format(env_name)}, + {"message": "created a new clique_type"}, "201") def build_query(self, filters): query = {} - filters_keys = ['name', 'focal_point_type'] + filters_keys = ['name', 'focal_point_type', + 'distribution', 'distribution_version', + 'mechanism_drivers', 'type_drivers'] self.update_query_with_filters(filters, filters_keys, query) + link_types = filters.get('link_type') if link_types: if type(link_types) != list: @@ -107,5 +135,71 @@ class CliqueTypes(ResponderBase): if _id: query[self.ID] = _id - query['environment'] = filters['env_name'] + env_name = filters.get('env_name') + if env_name: + query['environment'] = filters['env_name'] return query + + def validate_required_fields(self, clique_type): + env_name = clique_type.get('environment') + distribution = clique_type.get('distribution') + distribution_version = clique_type.get('distribution_version') + if distribution_version and not distribution: + self.bad_request("Distribution version without distribution " + "is not allowed") + + configuration_specified = ((distribution and distribution_version) + or clique_type.get('mechanism_drivers') + or clique_type.get('type_drivers')) + if env_name: + if configuration_specified: + self.bad_request("Either environment or configuration " + "should be specified (not both).") + + if not self.check_environment_name(env_name): + self.bad_request("Unknown environment: {}".format(env_name)) + elif env_name.upper() in self.RESERVED_NAMES: + self.bad_request( + "Environment name '{}' is reserved".format(env_name)) + elif not configuration_specified: + self.bad_request("Either environment or configuration " + "should be specified.") + + def validate_focal_point_type(self, clique_type): + focal_point_type = clique_type['focal_point_type'] + environment = clique_type.get('environment') + if environment: + env_match = self.read( + matches={"environment": environment, + "focal_point_type": focal_point_type}, + collection="clique_types" + ) + if env_match: + self.bad_request("Clique type with focal point {} " + "is already registered for environment {}" + .format(focal_point_type, environment)) + else: + pass + + def validate_duplicate_configuration(self, clique_type): + if clique_type.get('environment'): + return + + search = {'focal_point_type': clique_type['focal_point_type']} + for field in ['distribution', 'mechanism_drivers', 'type_drivers']: + value = clique_type.get(field) + if value: + search[field] = value + if field == 'distribution': + dv = clique_type.get('distribution_version') + if dv: + search['distribution_version'] = dv + # Got a match with higher score, no need to look further + break + + env_match = self.read(matches=search, + collection="clique_types") + if env_match: + self.bad_request("Clique type with configuration '{}' " + "is already registered" + .format(search)) diff --git a/app/api/responders/resource/environment_configs.py b/app/api/responders/resource/environment_configs.py index c24aec8..76cc8a9 100644 --- a/app/api/responders/resource/environment_configs.py +++ b/app/api/responders/resource/environment_configs.py @@ -13,7 +13,6 @@ from api.responders.responder_base import ResponderBase from bson.objectid import ObjectId from datetime import datetime from utils.constants import EnvironmentFeatures -from utils.inventory_mgr import InventoryMgr class EnvironmentConfigs(ResponderBase): @@ -27,9 +26,13 @@ class EnvironmentConfigs(ResponderBase): "distribution": True } CONFIGURATIONS_NAMES = ["mysql", "OpenStack", "CLI", "AMQP", - "Monitoring", "NFV_provider", "ACI"] - OPTIONAL_CONFIGURATIONS_NAMES = ["AMQP", "Monitoring", - "NFV_provider", "ACI"] + "Monitoring", "NFV_provider", "ACI", + "Kubernetes", "VMware", "Bare-metal"] + REQUIRED_CONFIGURATIONS_NAMES = { + "OpenStack": ["OpenStack", "mysql", "CLI"], + "Kubernetes": ["Kubernetes", "CLI"], + } + DEFAULT_ENV_TYPE = "OpenStack" def __init__(self): super().__init__() @@ -49,6 +52,8 @@ class EnvironmentConfigs(ResponderBase): get_constants_by_name("environment_operational_status") self.type_drivers = self.\ get_constants_by_name("type_drivers") + self.environment_types = self.\ + get_constants_by_name("environment_types") self.CONFIGURATIONS_REQUIREMENTS = { "mysql": { @@ -108,6 +113,7 @@ class EnvironmentConfigs(ResponderBase): }, "Monitoring": { "name": self.require(str, mandatory=True), + "install_monitoring_client": self.require(bool), "config_folder": self.require(str, mandatory=True, validate=DataValidate.REGEX, @@ -169,6 +175,20 @@ class EnvironmentConfigs(ResponderBase): requirement=[regex.IP, regex.HOSTNAME]), "user": self.require(str, mandatory=True), "pwd": self.require(str, mandatory=True) + }, + "Kubernetes": { + "name": self.require(str, mandatory=True), + "host": self.require(str, + mandatory=True, + validate=DataValidate.REGEX, + requirement=[regex.IP, regex.HOSTNAME]), + "port": self.require(int, + mandatory=True, + convert_to_type=True, + validate=DataValidate.REGEX, + requirement=regex.PORT), + "user": self.require(str, mandatory=True), + "token": self.require(str, mandatory=True) } } self.AUTH_REQUIREMENTS = { @@ -201,6 +221,9 @@ class EnvironmentConfigs(ResponderBase): "operational": self.require(str, validate=DataValidate.LIST, requirement=self.operational_values), + "environment_type": self.require(str, + validate=DataValidate.LIST, + requirement=self.environment_types), "page": self.require(int, convert_to_type=True), "page_size": self.require(int, convert_to_type=True) } @@ -223,7 +246,8 @@ class EnvironmentConfigs(ResponderBase): query = {} filters_keys = ["name", "distribution", "distribution_version", "type_drivers", "user", "listen", - "monitoring_setup_done", "scanned", "operational"] + "monitoring_setup_done", "scanned", "operational", + "environment_type"] self.update_query_with_filters(filters, filters_keys, query) mechanism_drivers = filters.get("mechanism_drivers") if mechanism_drivers: @@ -272,16 +296,26 @@ class EnvironmentConfigs(ResponderBase): "enable_monitoring": self.require(bool, convert_to_type=True), "monitoring_setup_done": self.require(bool, convert_to_type=True), "auth": self.require(dict), - "aci_enabled": self.require(bool, convert_to_type=True) + "aci_enabled": self.require(bool, convert_to_type=True), + "environment_type": self.require(str, + validate=DataValidate.LIST, + requirement=self.environment_types), } self.validate_query_data(env_config, environment_config_requirement, - can_be_empty_keys=["last_scanned"] - ) + can_be_empty_keys=["last_scanned", + "environment_type"]) self.check_and_convert_datetime("last_scanned", env_config) + # validate the configurations + environment_type = env_config.get("environment_type") + if not environment_type: + environment_type = self.DEFAULT_ENV_TYPE configurations = env_config['configuration'] - config_validation = self.validate_environment_config(configurations) + config_validation = ( + self.validate_environment_config(configurations=configurations, + environment_type=environment_type) + ) if not config_validation['passed']: self.bad_request(config_validation['error_message']) @@ -310,12 +344,11 @@ class EnvironmentConfigs(ResponderBase): .format(env_config["name"])}, "201") - def validate_environment_config(self, configurations, + def validate_environment_config(self, configurations, environment_type=None, require_mandatory=True): configurations_of_names = {} validation = {"passed": True} - if [config for config in configurations - if 'name' not in config]: + if any('name' not in config for config in configurations): validation['passed'] = False validation['error_message'] = "configuration must have name" return validation @@ -338,12 +371,19 @@ class EnvironmentConfigs(ResponderBase): "configuration for {0}".format(name) return validation configurations_of_names[name] = configs[0] - elif require_mandatory: - if name not in self.OPTIONAL_CONFIGURATIONS_NAMES: - validation["passed"] = False - validation['error_message'] = "configuration for {0} " \ - "is mandatory".format(name) - return validation + + if require_mandatory: + required_list = ( + self.REQUIRED_CONFIGURATIONS_NAMES.get(environment_type, []) + ) + if any(required_conf not in configurations_of_names + for required_conf + in required_list): + validation["passed"] = False + validation['error_message'] = ("configurations for ({})" + "are mandatory for " + "this environment type" + .format(", ".join(required_list))) for name, config in configurations_of_names.items(): error_message = self.validate_configuration(name, config) diff --git a/app/api/responders/responder_base.py b/app/api/responders/responder_base.py index e59f4cf..0ac08d6 100644 --- a/app/api/responders/responder_base.py +++ b/app/api/responders/responder_base.py @@ -71,7 +71,7 @@ class ResponderBase(DataValidate, DictNamingConverter): def validate_query_data(self, data, data_requirements, additional_key_reg=None, - can_be_empty_keys=[]): + can_be_empty_keys=None): error_message = self.validate_data(data, data_requirements, additional_key_reg, can_be_empty_keys) @@ -197,7 +197,9 @@ class ResponderBase(DataValidate, DictNamingConverter): ': no "value" key for data: ' + str(d)) return consts - def read(self, collection, matches={}, projection=None, skip=0, limit=1000): + def read(self, collection, matches=None, projection=None, skip=0, limit=1000): + if matches is None: + matches = {} collection = self.get_collection_by_name(collection) skip *= limit query = collection.find(matches, projection).skip(skip).limit(limit) diff --git a/app/api/validation/data_validate.py b/app/api/validation/data_validate.py index 6928c4b..4dfb214 100644 --- a/app/api/validation/data_validate.py +++ b/app/api/validation/data_validate.py @@ -75,7 +75,9 @@ class DataValidate: def validate_data(self, data, requirements, additional_key_re=None, - can_be_empty_keys=[]): + can_be_empty_keys=None): + if can_be_empty_keys is None: + can_be_empty_keys = [] illegal_keys = [key for key in data.keys() if key not in requirements.keys()] diff --git a/app/config/link_finders.json b/app/config/link_finders.json index 55c31f6..b421ee9 100644 --- a/app/config/link_finders.json +++ b/app/config/link_finders.json @@ -7,6 +7,7 @@ "FindLinksForVconnectors", "FindLinksForVedges", "FindLinksForVserviceVnics", - "FindLinksForPnics" + "FindLinksForPnics", + "FindImplicitLinks" ] } \ No newline at end of file diff --git a/app/config/scanners.json b/app/config/scanners.json index c5efb06..a96029a 100644 --- a/app/config/scanners.json +++ b/app/config/scanners.json @@ -36,7 +36,8 @@ "types_name": "regions", "parent_type": "environment" }, - "children_scanner": "ScanRegionsRoot" + "children_scanner": "ScanRegionsRoot", + "environment_condition": {"environment_type": "OpenStack"} }, { "type": "projects_folder", @@ -45,7 +46,20 @@ "types_name": "projects", "parent_type": "environment" }, - "children_scanner": "ScanProjectsRoot" + "children_scanner": "ScanProjectsRoot", + "environment_condition": {"environment_type": "OpenStack"} + }, + { + "type": "namespaces_folder", + "fetcher": { + "folder": true, + "types_name": "namespaces", + "parent_type": "environment" + }, + "children_scanner": "ScanNamespacesRoot", + "environment_condition": { + "environment_type": "Kubernetes" + } } ], "ScanHostNetworkAgentsRoot": [ @@ -377,6 +391,13 @@ "type": "vservice", "fetcher": "CliFetchHostVservices" } + ], + "ScanNamespacesRoot": [ + { + "type": "namespace", + "fetcher": "KubeFetchNamespaces", + "environment_condition": {"environment_type": "Kubernetes"} + } ] } } diff --git a/app/connection_test/connection_test.py b/app/connection_test/connection_test.py deleted file mode 100644 index d9d6af7..0000000 --- a/app/connection_test/connection_test.py +++ /dev/null @@ -1,283 +0,0 @@ -############################################################################### -# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # -# and others # -# # -# All rights reserved. This program and the accompanying materials # -# are made available under the terms of the Apache License, Version 2.0 # -# which accompanies this distribution, and is available at # -# http://www.apache.org/licenses/LICENSE-2.0 # -############################################################################### -import argparse -import datetime -from kombu import Connection - -import time - -import pymongo -from functools import partial - -from discover.fetchers.api.api_access import ApiAccess -from discover.fetchers.db.db_access import DbAccess -from discover.manager import Manager -from utils.constants import ConnectionTestStatus, ConnectionTestType -from utils.logging.file_logger import FileLogger -from utils.mongo_access import MongoAccess -from utils.ssh_connection import * - - -def test_openstack(config, test_request): - try: - api = ApiAccess(config) - ConnectionTest.report_success(test_request, - ConnectionTestType.OPENSTACK.value) - if api: - pass - except ValueError: - pass - - -def test_mysql(config, test_request): - db_access = DbAccess(config) - ConnectionTest.report_success(test_request, ConnectionTestType.MYSQL.value) - if db_access: - pass - - -def test_ssh_connect(config) -> bool: - ssh = SshConnection(config.get('host', ''), - config.get('user', ''), - _pwd=config.get('pwd'), - _key=config.get('key'), - _port=int(config.get('port', - SshConnection.DEFAULT_PORT))) - ret = ssh.connect() - return ret - - -def test_cli(config, test_request): - ret = test_ssh_connect(config) - ConnectionTest.set_test_result(test_request, - ConnectionTestType.CLI.value, - ret) - - -def test_amqp_connect(config): - connect_url = 'amqp://{user}:{pwd}@{host}:{port}//' \ - .format(user=config.get("user", ''), - pwd=config.get('pwd', ''), - host=config.get('host', ''), - port=int(config.get('port', 5671))) - conn = Connection(connect_url) - conn.connect() - - -def test_amqp(config, test_request): - test_amqp_connect(config) - ConnectionTest.report_success(test_request, ConnectionTestType.AMQP.value) - - -def test_monitoring(config, test_request): - # for monitoring configuration test, need to test: - # 1. SSH access - # 2. RabbitMQ access - ssh_config = { - 'host': config.get('server_ip'), - 'user': config.get('ssh_user'), - 'pwd': config.get('ssh_password'), - 'port': int(config.get('ssh_port', 0)) - } - if not test_ssh_connect(ssh_config): - return - amqp_connect_config = { - 'user': config.get('rabbitmq_user', ''), - 'pwd': config.get('rabbitmq_pass', ''), - 'host': config.get('server_ip'), - 'port': int(config.get('rabbitmq_port', 5672)), - } - test_amqp_connect(amqp_connect_config) - ConnectionTest.report_success(test_request, ConnectionTestType.AMQP.value) - - -def test_aci(config, test_request): - pass - - -TEST_HANDLERS = { - ConnectionTestType.OPENSTACK.value: test_openstack, - ConnectionTestType.MYSQL.value: test_mysql, - ConnectionTestType.CLI.value: test_cli, - ConnectionTestType.AMQP.value: test_amqp, - ConnectionTestType.ACI.value: test_aci, - ConnectionTestType.MONITORING.value: test_monitoring -} - - -class ConnectionTest(Manager): - - DEFAULTS = { - 'mongo_config': '', - 'connection_tests': 'connection_tests', - 'environments': 'environments_config', - 'interval': 1, - 'loglevel': 'WARNING' - } - - def __init__(self): - self.args = self.get_args() - super().__init__(log_directory=self.args.log_directory, - mongo_config_file=self.args.mongo_config) - self.db_client = None - self.connection_tests_collection = None - self.environments_collection = None - - @staticmethod - def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-m', '--mongo_config', nargs='?', type=str, - default=ConnectionTest.DEFAULTS['mongo_config'], - help='Name of config file ' + - 'with MongoDB server access details') - parser.add_argument('-c', '--connection_tests_collection', nargs='?', - type=str, - default=ConnectionTest.DEFAULTS['connection_tests'], - help='connection_tests collection to read from') - parser.add_argument('-e', '--environments_collection', nargs='?', - type=str, - default=ConnectionTest.DEFAULTS['environments'], - help='Environments collection to update ' - 'after tests') - parser.add_argument('-i', '--interval', nargs='?', type=float, - default=ConnectionTest.DEFAULTS['interval'], - help='Interval between collection polls' - '(must be more than {} seconds)' - .format(ConnectionTest.MIN_INTERVAL)) - parser.add_argument('-l', '--loglevel', nargs='?', type=str, - default=ConnectionTest.DEFAULTS['loglevel'], - help='Logging level \n(default: {})' - .format(ConnectionTest.DEFAULTS['loglevel'])) - parser.add_argument('-d', '--log_directory', nargs='?', type=str, - default=FileLogger.LOG_DIRECTORY, - help='File logger directory \n(default: {})' - .format(FileLogger.LOG_DIRECTORY)) - args = parser.parse_args() - return args - - def configure(self): - self.db_client = MongoAccess() - self.connection_tests_collection = \ - self.db_client.db[self.args.connection_tests_collection] - self.environments_collection = \ - self.db_client.db[self.args.environments_collection] - self._update_document = \ - partial(MongoAccess.update_document, - self.connection_tests_collection) - self.interval = max(self.MIN_INTERVAL, self.args.interval) - self.log.set_loglevel(self.args.loglevel) - - self.log.info('Started ConnectionTest with following configuration:\n' - 'Mongo config file path: {0.args.mongo_config}\n' - 'connection_tests collection: ' - '{0.connection_tests_collection.name}\n' - 'Polling interval: {0.interval} second(s)' - .format(self)) - - def _build_test_args(self, test_request: dict): - args = { - 'mongo_config': self.args.mongo_config - } - - def set_arg(name_from: str, name_to: str = None): - if name_to is None: - name_to = name_from - val = test_request.get(name_from) - if val: - args[name_to] = val - - set_arg('object_id', 'id') - set_arg('log_level', 'loglevel') - set_arg('environment', 'env') - set_arg('scan_only_inventory', 'inventory_only') - set_arg('scan_only_links', 'links_only') - set_arg('scan_only_cliques', 'cliques_only') - set_arg('inventory') - set_arg('clear') - set_arg('clear_all') - - return args - - def _finalize_test(self, test_request: dict): - # update the status and timestamps. - self.log.info('Request {} has been tested.' - .format(test_request['_id'])) - start_time = test_request['submit_timestamp'] - end_time = datetime.datetime.utcnow() - test_request['response_timestamp'] = end_time - test_request['response_time'] = \ - str(end_time - start_time.replace(tzinfo=None)) - test_request['status'] = ConnectionTestStatus.RESPONSE.value - self._update_document(test_request) - - @staticmethod - def set_test_result(test_request, target, result): - test_request.get('test_results', {})[target] = result - - @staticmethod - def report_success(test_request, target): - ConnectionTest.set_test_result(test_request, target, True) - - @staticmethod - def handle_test_target(target, test_request): - targets_config = test_request.get('targets_configuration', []) - try: - config = next(t for t in targets_config if t['name'] == target) - except StopIteration: - raise ValueError('failed to find {} in targets_configuration' - .format(target)) - handler = TEST_HANDLERS.get(target) - if not handler: - raise ValueError('unknown test target: {}'.format(target)) - handler(config, test_request) - - def do_test(self, test_request): - targets = [t for t in test_request.get('test_targets', [])] - test_request['test_results'] = {t: False for t in targets} - for test_target in test_request.get('test_targets', []): - self.log.info('testing connection to: {}'.format(test_target)) - try: - self.handle_test_target(test_target, test_request) - except Exception as e: - self.log.exception(e) - if 'errors' not in test_request: - test_request['errors'] = {} - test_request['errors'][test_target] = str(e) - self.log.error('Test of target {} failed (id: {}):\n{}' - .format(test_target, - test_request['_id'], - str(e))) - self._finalize_test(test_request) - self._set_env_operational(test_request['environment']) - - # if environment_config document for this specific environment exists, - # update the value of the 'operational' field to 'running' - def _set_env_operational(self, env): - self.environments_collection. \ - update_one({'name': env}, {'$set': {'operational': 'running'}}) - - def do_action(self): - while True: - # Find a pending request that is waiting the longest time - results = self.connection_tests_collection \ - .find({'status': ConnectionTestStatus.REQUEST.value, - 'submit_timestamp': {'$ne': None}}) \ - .sort('submit_timestamp', pymongo.ASCENDING) \ - .limit(1) - - # If no connection tests are pending, sleep for some time - if results.count() == 0: - time.sleep(self.interval) - else: - self.do_test(results[0]) - - -if __name__ == '__main__': - ConnectionTest().run() diff --git a/app/discover/clique_finder.py b/app/discover/clique_finder.py index 57b2e3b..4d68eb4 100644 --- a/app/discover/clique_finder.py +++ b/app/discover/clique_finder.py @@ -48,61 +48,53 @@ class CliqueFinder(Fetcher): self.find_cliques_for_type(clique_type) self.log.info("finished scanning for cliques") - # Calculate priority score + # Calculate priority score for clique type per environment and configuration def _get_priority_score(self, clique_type): + # environment-specific clique type takes precedence if self.env == clique_type['environment']: + return 16 + if (self.env_config['distribution'] == clique_type.get('distribution') + and + self.env_config['distribution_version'] == + clique_type.get('distribution_version')): + return 8 + if clique_type.get('mechanism_drivers') \ + in self.env_config['mechanism_drivers']: return 4 - if (self.env_config['distribution'] == clique_type.get('distribution') and - self.env_config['distribution_version'] == clique_type.get('distribution_version')): - return 3 - if clique_type.get('mechanism_drivers') in self.env_config['mechanism_drivers']: - return 2 if self.env_config['type_drivers'] == clique_type.get('type_drivers'): + return 2 + if clique_type.get('environment', '') == 'ANY': + # environment=ANY serves as fallback option, but it's not mandatory return 1 else: return 0 # Get clique type with max priority - # for given environment configuration and focal point type - def _get_clique_type(self, focal_point, clique_types): - # If there's no configuration match for the specified environment, - # we use the default clique type definition with environment='ANY' - fallback_type = next( - filter(lambda t: t['environment'] == 'ANY', clique_types), - None - ) - if not fallback_type: - raise ValueError("No fallback clique type (ANY) " - "defined for focal point type '{}'" - .format(focal_point)) - - clique_types.remove(fallback_type) - - priority_scores = [self._get_priority_score(clique_type) - for clique_type - in clique_types] - max_score = max(priority_scores) if priority_scores else 0 - - return (fallback_type - if max_score == 0 - else clique_types[priority_scores.index(max_score)]) + # for given focal point type + def _get_clique_type(self, clique_types): + scored_clique_types = [{'score': self._get_priority_score(clique_type), + 'clique_type': clique_type} + for clique_type in clique_types] + max_score = max(scored_clique_types, key=lambda t: t['score']) + if max_score['score'] == 0: + self.log.warn('No matching clique types for focal point type: {}' + .format(clique_types[0].get('focal_point_type'))) + return None + return max_score.get('clique_type') def get_clique_types(self): if not self.clique_types_by_type: - clique_types_by_focal_point = self.clique_types.aggregate([{ - "$group": { - "_id": "$focal_point_type", - "types": {"$push": "$$ROOT"} - } - }]) - - self.clique_types_by_type = { - cliques['_id']: self._get_clique_type(cliques['_id'], - cliques['types']) - for cliques in - clique_types_by_focal_point - } - + clique_types_candidates = {} + for clique in self.clique_types.find({}): + fp_type = clique.get('focal_point_type', '') + if not clique_types_candidates.get(fp_type): + clique_types_candidates[fp_type] = [] + clique_types_candidates[fp_type].append(clique) + for t in clique_types_candidates.keys(): + selected = self._get_clique_type(clique_types_candidates[t]) + if not selected: + continue + self.clique_types_by_type[t] = selected return self.clique_types_by_type def find_cliques_for_type(self, clique_type): @@ -125,11 +117,14 @@ class CliqueFinder(Fetcher): .find_one({"focal_point_type": o['type']}) constraints = [] if not constraint else constraint["constraints"] clique_types = self.get_clique_types() - clique_type = clique_types[o['type']] - new_clique = self.construct_clique_for_focal_point(o, clique_type, - constraints) - if not new_clique: + clique_type = clique_types.get(o['type']) + if not clique_type: self.cliques.delete({'_id': clique['_id']}) + else: + new_clique = self.construct_clique_for_focal_point(o, clique_type, + constraints) + if not new_clique: + self.cliques.delete({'_id': clique['_id']}) def construct_clique_for_focal_point(self, o, clique_type, constraints): # keep a hash of nodes in clique that were visited for each type @@ -146,12 +141,15 @@ class CliqueFinder(Fetcher): for c in constraints: val = o[c] if c in o else None clique["constraints"][c] = val + allow_implicit = clique_type.get('use_implicit_links', False) for link_type in clique_type["link_types"]: - self.check_link_type(clique, link_type, nodes_of_type) + self.check_link_type(clique, link_type, nodes_of_type, + allow_implicit=allow_implicit) # after adding the links to the clique, create/update the clique if not clique["links"]: return None + clique["clique_type"] = clique_type["_id"] focal_point_obj = self.inventory.find({"_id": clique["focal_point"]}) if not focal_point_obj: return None @@ -198,25 +196,32 @@ class CliqueFinder(Fetcher): '-'.join(link_type_parts) return CliqueFinder.link_type_reversed.get(link_type) - def check_link_type(self, clique, link_type, nodes_of_type): + def check_link_type(self, clique, link_type, nodes_of_type, + allow_implicit=False): # check if it's backwards link_type_reversed = self.get_link_type_reversed(link_type) # handle case of links like T<-->T self_linked = link_type == link_type_reversed use_reversed = False if not self_linked: - matches = self.links.find_one({ + link_search_condition = { "environment": self.env, "link_type": link_type_reversed - }) + } + if not allow_implicit: + link_search_condition['implicit'] = False + matches = self.links.find_one(link_search_condition) use_reversed = True if matches else False if self_linked or not use_reversed: - self.check_link_type_forward(clique, link_type, nodes_of_type) + self.check_link_type_forward(clique, link_type, nodes_of_type, + allow_implicit=allow_implicit) if self_linked or use_reversed: - self.check_link_type_back(clique, link_type, nodes_of_type) + self.check_link_type_back(clique, link_type, nodes_of_type, + allow_implicit=allow_implicit) def check_link_type_for_direction(self, clique, link_type, nodes_of_type, - is_reversed=False): + is_reversed=False, + allow_implicit=False): if is_reversed: link_type = self.get_link_type_reversed(link_type) from_type = link_type[:link_type.index("-")] @@ -233,7 +238,8 @@ class CliqueFinder(Fetcher): clique, link_type, side_to_match, - other_side) + other_side, + allow_implicit=allow_implicit) nodes_to_add = nodes_to_add | matches if other_side_type not in nodes_of_type: nodes_of_type[other_side_type] = set() @@ -241,13 +247,17 @@ class CliqueFinder(Fetcher): nodes_of_type[other_side_type] | nodes_to_add def find_matches_for_point(self, match_point, clique, link_type, - side_to_match, other_side) -> set: + side_to_match, other_side, + allow_implicit=False) -> set: nodes_to_add = set() - matches = self.links.find({ + link_search_condition = { "environment": self.env, "link_type": link_type, side_to_match: ObjectId(match_point) - }) + } + if not allow_implicit: + link_search_condition['implicit'] = False + matches = self.links.find(link_search_condition) for link in matches: link_id = link["_id"] if link_id in clique["links"]: @@ -260,10 +270,14 @@ class CliqueFinder(Fetcher): nodes_to_add.add(other_side_point) return nodes_to_add - def check_link_type_forward(self, clique, link_type, nodes_of_type): + def check_link_type_forward(self, clique, link_type, nodes_of_type, + allow_implicit=False): self.check_link_type_for_direction(clique, link_type, nodes_of_type, - is_reversed=False) + is_reversed=False, + allow_implicit=allow_implicit) - def check_link_type_back(self, clique, link_type, nodes_of_type): + def check_link_type_back(self, clique, link_type, nodes_of_type, + allow_implicit=False): self.check_link_type_for_direction(clique, link_type, nodes_of_type, - is_reversed=True) + is_reversed=True, + allow_implicit=allow_implicit) diff --git a/app/discover/event_manager.py b/app/discover/event_manager.py index 4855acc..c01916c 100644 --- a/app/discover/event_manager.py +++ b/app/discover/event_manager.py @@ -113,8 +113,8 @@ class EventManager(Manager): def get_listener(self, env: str): env_config = self.inv.get_env_config(env) return (self.LISTENERS.get(env_config.get('distribution'), {}) - .get(env_config.get('distribution_version', - DefaultListener))) + .get(env_config.get('distribution_version'), + DefaultListener)) def listen_to_events(self, listener: ListenerBase, env_name: str, process_vars: dict): listener.listen({ diff --git a/app/discover/fetchers/api/api_access.py b/app/discover/fetchers/api/api_access.py index f685faf..1fca202 100644 --- a/app/discover/fetchers/api/api_access.py +++ b/app/discover/fetchers/api/api_access.py @@ -12,21 +12,18 @@ import re import requests import time -from discover.configuration import Configuration -from discover.fetcher import Fetcher +from utils.api_access_base import ApiAccessBase from utils.string_utils import jsonify -class ApiAccess(Fetcher): +class ApiAccess(ApiAccessBase): + + ADMIN_PORT = "35357" + subject_token = None initialized = False regions = {} - config = None - api_config = None - host = "" - base_url = "" - admin_token = "" tokens = {} admin_endpoint = "" admin_project = None @@ -38,28 +35,19 @@ class ApiAccess(Fetcher): # identity API v2 version with admin token def __init__(self, config=None): - super(ApiAccess, self).__init__() - if ApiAccess.initialized: + super().__init__('OpenStack', config) + self.base_url = "http://" + self.host + ":" + self.port + if self.initialized: return - ApiAccess.config = {'OpenStack': config} if config else Configuration() - ApiAccess.api_config = ApiAccess.config.get("OpenStack") - host = ApiAccess.api_config.get("host", "") - ApiAccess.host = host - port = ApiAccess.api_config.get("port", "") - if not (host and port): - raise ValueError('Missing definition of host or port ' + - 'for OpenStack API access') - ApiAccess.base_url = "http://" + host + ":" + port - ApiAccess.admin_token = ApiAccess.api_config.get("admin_token", "") - ApiAccess.admin_project = ApiAccess.api_config.get("admin_project", - "admin") - ApiAccess.admin_endpoint = "http://" + host + ":" + "35357" + ApiAccess.admin_project = self.api_config.get("admin_project", "admin") + ApiAccess.admin_endpoint = "http://" + self.host + ":" + self.ADMIN_PORT token = self.v2_auth_pwd(ApiAccess.admin_project) if not token: raise ValueError("Authentication failed. Failed to obtain token") else: self.subject_token = token + self.initialized = True @staticmethod def parse_time(time_str): @@ -95,9 +83,9 @@ class ApiAccess(Fetcher): subject_token = self.get_existing_token(project_id) if subject_token: return subject_token - req_url = ApiAccess.base_url + "/v2.0/tokens" + req_url = self.base_url + "/v2.0/tokens" response = requests.post(req_url, json=post_body, headers=headers, - timeout=5) + timeout=self.CONNECT_TIMEOUT) response = response.json() ApiAccess.auth_response[project_id] = response if 'error' in response: @@ -120,8 +108,8 @@ class ApiAccess(Fetcher): return token_details def v2_auth_pwd(self, project): - user = ApiAccess.api_config["user"] - pwd = ApiAccess.api_config["pwd"] + user = self.api_config["user"] + pwd = self.api_config["pwd"] post_body = { "auth": { "passwordCredentials": { @@ -148,23 +136,6 @@ class ApiAccess(Fetcher): auth_response = ApiAccess.auth_response.get('admin', {}) return auth_response - def get_rel_url(self, relative_url, headers): - req_url = ApiAccess.base_url + relative_url - return self.get_url(req_url, headers) - - def get_url(self, req_url, headers): - response = requests.get(req_url, headers=headers) - if response.status_code != requests.codes.ok: - # some error happened - if "reason" in response: - msg = ", reason: {}".format(response.reason) - else: - msg = ", response: {}".format(response.text) - self.log.error("req_url: {} {}".format(req_url, msg)) - return None - ret = response.json() - return ret - def get_region_url(self, region_name, service): if region_name not in self.regions: return None @@ -174,7 +145,7 @@ class ApiAccess(Fetcher): return None orig_url = s["adminURL"] # replace host name with the host found in config - url = re.sub(r"^([^/]+)//[^:]+", r"\1//" + ApiAccess.host, orig_url) + url = re.sub(r"^([^/]+)//[^:]+", r"\1//" + self.host, orig_url) return url # like get_region_url(), but remove everything starting from the "/v2" diff --git a/app/discover/fetchers/api/api_fetch_host_instances.py b/app/discover/fetchers/api/api_fetch_host_instances.py index 56cffda..bf8513a 100644 --- a/app/discover/fetchers/api/api_fetch_host_instances.py +++ b/app/discover/fetchers/api/api_fetch_host_instances.py @@ -18,7 +18,7 @@ class ApiFetchHostInstances(ApiAccess, DbAccess, metaclass=Singleton): def __init__(self): super(ApiFetchHostInstances, self).__init__() self.inv = InventoryMgr() - self.endpoint = ApiAccess.base_url.replace(":5000", ":8774") + self.endpoint = self.base_url.replace(":5000", ":8774") self.projects = None self.db_fetcher = DbFetchInstances() diff --git a/app/discover/fetchers/api/api_fetch_project_hosts.py b/app/discover/fetchers/api/api_fetch_project_hosts.py index 5b911f5..2aeb24f 100644 --- a/app/discover/fetchers/api/api_fetch_project_hosts.py +++ b/app/discover/fetchers/api/api_fetch_project_hosts.py @@ -11,9 +11,11 @@ import json from discover.fetchers.api.api_access import ApiAccess from discover.fetchers.db.db_access import DbAccess +from discover.fetchers.cli.cli_access import CliAccess +from utils.ssh_connection import SshError -class ApiFetchProjectHosts(ApiAccess, DbAccess): +class ApiFetchProjectHosts(ApiAccess, DbAccess, CliAccess): def __init__(self): super(ApiFetchProjectHosts, self).__init__() @@ -107,6 +109,7 @@ class ApiFetchProjectHosts(ApiAccess, DbAccess): s = services["nova-compute"] if s["available"] and s["active"]: self.add_host_type(doc, "Compute", az['zoneName']) + self.fetch_host_os_details(doc) return doc # fetch more details of network nodes from neutron DB agents table @@ -121,7 +124,12 @@ class ApiFetchProjectHosts(ApiAccess, DbAccess): """.format(self.neutron_db) results = self.get_objects_list(query, "") for r in results: - host = hosts[r["host"]] + host = r["host"] + if host not in hosts: + self.log.error("host from agents table not in hosts list: {}" + .format(host)) + continue + host = hosts[host] host["config"] = json.loads(r["configurations"]) self.add_host_type(host, "Network", '') @@ -136,9 +144,33 @@ class ApiFetchProjectHosts(ApiAccess, DbAccess): for db_row in results: doc.update(db_row) - def add_host_type(self, doc, type, zone): - if not type in doc["host_type"]: - doc["host_type"].append(type) - if type == 'Compute': + @staticmethod + def add_host_type(doc, host_type, zone): + if host_type not in doc["host_type"]: + doc["host_type"].append(host_type) + if host_type == 'Compute': doc['zone'] = zone doc['parent_id'] = zone + + def fetch_host_os_details(self, doc): + cmd = 'cat /etc/os-release && echo "ARCHITECURE=`arch`"' + try: + lines = self.run_fetch_lines(cmd, ssh_to_host=doc['host']) + except SshError as e: + self.log.error('{}: {}', cmd, str(e)) + os_attributes = {} + attributes_to_fetch = { + 'NAME': 'name', + 'VERSION': 'version', + 'ID': 'ID', + 'ID_LIKE': 'ID_LIKE', + 'ARCHITECURE': 'architecure' + } + for attr in attributes_to_fetch: + matches = [l for l in lines if l.startswith(attr + '=')] + if matches: + line = matches[0] + attr_name = attributes_to_fetch[attr] + os_attributes[attr_name] = line[line.index('=')+1:].strip('"') + if os_attributes: + doc['OS'] = os_attributes diff --git a/app/discover/fetchers/api/api_fetch_regions.py b/app/discover/fetchers/api/api_fetch_regions.py index 23a3736..4e83b01 100644 --- a/app/discover/fetchers/api/api_fetch_regions.py +++ b/app/discover/fetchers/api/api_fetch_regions.py @@ -13,7 +13,7 @@ from discover.fetchers.api.api_access import ApiAccess class ApiFetchRegions(ApiAccess): def __init__(self): super(ApiFetchRegions, self).__init__() - self.endpoint = ApiAccess.base_url + self.endpoint = self.base_url def get(self, regions_folder_id): token = self.v2_auth_pwd(self.admin_project) diff --git a/app/discover/fetchers/db/db_access.py b/app/discover/fetchers/db/db_access.py index 090ab84..5ff49d5 100644 --- a/app/discover/fetchers/db/db_access.py +++ b/app/discover/fetchers/db/db_access.py @@ -38,8 +38,7 @@ class DbAccess(Fetcher): conn = None query_count_per_con = 0 - # connection timeout set to 30 seconds, - # due to problems over long connections + # connection timeout set to 5 seconds TIMEOUT = 5 def __init__(self, mysql_config=None): @@ -47,6 +46,9 @@ class DbAccess(Fetcher): self.config = {'mysql': mysql_config} if mysql_config \ else Configuration() self.conf = self.config.get("mysql") + self.connect_timeout = int(self.conf['connect_timeout']) \ + if 'connect_timeout' in self.conf \ + else self.TIMEOUT self.connect_to_db() self.neutron_db = self.get_neutron_db_name() @@ -55,16 +57,18 @@ class DbAccess(Fetcher): return try: connector = mysql.connector - DbAccess.conn = connector.connect(host=_host, port=_port, - connection_timeout=self.TIMEOUT, - user=_user, - password=_pwd, - database=_database, - raise_on_warnings=True) + conn = connector.connect(host=_host, port=_port, + connection_timeout=self.connect_timeout, + user=_user, + password=_pwd, + database=_database, + raise_on_warnings=True) + DbAccess.conn = conn DbAccess.conn.ping(True) # auto-reconnect if necessary except Exception as e: - self.log.critical("failed to connect to MySQL DB: {}" - .format(str(e))) + msg = "failed to connect to MySQL DB: {}".format(str(e)) + self.log.critical(msg) + raise ScanError(msg) return DbAccess.query_count_per_con = 0 @@ -93,8 +97,11 @@ class DbAccess(Fetcher): DbAccess.conn = None self.conf = self.config.get("mysql") cnf = self.conf + pwd = cnf.get('pwd', '') + if not pwd: + raise ScanError('db_access: attribute pwd is missing') self.db_connect(cnf.get('host', ''), cnf.get('port', ''), - cnf.get('user', ''), cnf.get('pwd', ''), + cnf.get('user', ''), pwd, cnf.get('schema', 'nova')) @with_cursor diff --git a/app/discover/fetchers/kube/__init__.py b/app/discover/fetchers/kube/__init__.py new file mode 100644 index 0000000..b0637e9 --- /dev/null +++ b/app/discover/fetchers/kube/__init__.py @@ -0,0 +1,9 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### diff --git a/app/discover/fetchers/kube/kube_access.py b/app/discover/fetchers/kube/kube_access.py new file mode 100644 index 0000000..38bb978 --- /dev/null +++ b/app/discover/fetchers/kube/kube_access.py @@ -0,0 +1,28 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from kubernetes.client import Configuration as KubConf, CoreV1Api + +from utils.api_access_base import ApiAccessBase + + +class KubeAccess(ApiAccessBase): + + def __init__(self, config=None): + super().__init__('Kubernetes', config) + self.base_url = 'https://{}:{}'.format(self.host, self.port) + self.bearer_token = self.api_config.get('token', '') + conf = KubConf() + conf.host = self.base_url + conf.user = self.api_config.get('user') + conf.api_key_prefix['authorization'] = 'Bearer' + conf.api_key['authorization'] = self.bearer_token + conf.verify_ssl = False + self.api = CoreV1Api() + diff --git a/app/discover/fetchers/kube/kube_fetch_namespaces.py b/app/discover/fetchers/kube/kube_fetch_namespaces.py new file mode 100644 index 0000000..951ddb8 --- /dev/null +++ b/app/discover/fetchers/kube/kube_fetch_namespaces.py @@ -0,0 +1,32 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.kube.kube_access import KubeAccess + + +class KubeFetchNamespaces(KubeAccess): + + def __init__(self, config=None): + super().__init__(config) + + def get(self, object_id): + namespaces = self.api.list_namespace() + return [self.get_namespace(i) for i in namespaces.items] + + @staticmethod + def get_namespace(namespace): + attrs = ['creation_timestamp', 'self_link', 'uid'] + namespace_details = { + 'name': namespace.metadata.name, + 'status': namespace.status.phase + } + namespace_details.update({x: getattr(namespace.metadata, x, '') + for x in attrs}) + namespace_details['id'] = namespace_details['uid'] + return namespace_details diff --git a/app/discover/link_finders/find_implicit_links.py b/app/discover/link_finders/find_implicit_links.py new file mode 100644 index 0000000..01eaa7b --- /dev/null +++ b/app/discover/link_finders/find_implicit_links.py @@ -0,0 +1,128 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.link_finders.find_links import FindLinks + + +class FindImplicitLinks(FindLinks): + + def __init__(self): + super().__init__() + self.links = [] + self.constraint_attributes = self.get_constraint_attributes() + + def add_links(self): + self.log.info('adding implicit links') + self.get_existing_links() + self.get_transitive_closure() + + def get_constraint_attributes(self) -> list: + attributes = set() + for c in self.inv.find({'environment': self.get_env()}, + collection='clique_constraints'): + for a in c['constraints']: + attributes.add(a) + return list(attributes) + + def get_existing_links(self): + self.log.info('fetching existing links') + existing_links = self.inv.find({'environment': self.get_env()}, + collection='links') + for l in existing_links: + self.links.append({'pass': 0, 'link': l}) + + def constraints_match(self, link1, link2): + if 'attributes' not in link1 or 'attributes' not in link2: + return True + attr1 = link1['attributes'] + attr2 = link2['attributes'] + for a in self.constraint_attributes: + if a in attr1 and a in attr2 and attr1[a] != attr2[a]: + return False + return True + + def links_match(self, start, dest): + if start['link_type'] == dest['link_type']: + return False # obviously we cannot make an implicit link of this + if start['source_id'] == dest['target_id']: + return False # avoid cyclic links + if not self.constraints_match(start, dest): + return False + return start['target_id'] == dest['source_id'] + + def add_matching_links(self, link, pass_no): + self.log.debug('looking for matches for link: {};{}' + .format(link['source_id'], link['target_id'])) + matches = [l for l in self.links + if l['pass'] == 0 # take only original links + and self.links_match(link, l['link'])] + for l in matches: + implicit = self.add_implicit_link(link, l['link']) + self.links.append({'pass': pass_no, 'link': implicit}) + return len(matches) + + def get_link_constraint_attributes(self, link1, link2) -> dict: + attributes = {} + for a in self.constraint_attributes: + # constraints_match() verified the attribute values don't conflict + if a in link1.get('attributes', {}): + attributes[a] = link1['attributes'][a] + elif a in link2.get('attributes', {}): + attributes[a] = link2['attributes'][a] + return attributes + + @staticmethod + def get_attr(attr, link1, link2): + if attr not in link1 and attr not in link2: + return None + if attr not in link1: + return link2[attr] + if attr not in link2 or link1[attr] == link2[attr]: + return link1[attr] + return None + + def add_implicit_link(self, link1, link2): + link_type_from = link1['link_type'].split('-')[0] + link_type_to = link2['link_type'].split('-')[1] + link_type = '{}-{}'.format(link_type_from, link_type_to) + link_name = '' + state = 'down' \ + if link1['state'] == 'down' or link2['state'] == 'down' \ + else 'up' + link_weight = 0 # TBD + host = self.get_attr('host', link1, link2) + switch = self.get_attr('switch', link1, link2) + extra_attributes = self.get_link_constraint_attributes(link1, link2) + self.log.debug('adding implicit link: link type: {}, from: {}, to: {}' + .format(link_type, + link1['source_id'], + link2['target_id'])) + implicit = self.create_link(self.get_env(), + link1['source'], link1['source_id'], + link2['target'], link2['target_id'], + link_type, link_name, state, link_weight, + host=host, switch=switch, + implicit=True, + extra_attributes=extra_attributes) + return implicit + + def get_transitive_closure(self): + pass_no = 1 + while True: + match_count = 0 + last_pass_links = [l for l in self.links if l['pass'] == pass_no-1] + for l in last_pass_links: + match_count += self.add_matching_links(l['link'], pass_no) + self.log.info('Transitive closure pass #{}: ' + 'found {} implicit links' + .format(pass_no, match_count)) + if match_count == 0: + break + pass_no += 1 + self.log.info('done adding implicit links') diff --git a/app/discover/link_finders/find_links.py b/app/discover/link_finders/find_links.py index d234479..31d39e5 100644 --- a/app/discover/link_finders/find_links.py +++ b/app/discover/link_finders/find_links.py @@ -19,6 +19,7 @@ class FindLinks(Fetcher): def create_link(self, env, source, source_id, target, target_id, link_type, link_name, state, link_weight, host=None, switch=None, + implicit=False, extra_attributes=None): if extra_attributes is None: extra_attributes = {} @@ -27,9 +28,11 @@ class FindLinks(Fetcher): link = self.inv.create_link(env, source, source_id, target, target_id, link_type, link_name, state, link_weight, + implicit=implicit, source_label=source_label, target_label=target_label, host=host, switch=switch, extra_attributes=extra_attributes) if self.inv.monitoring_setup_manager: self.inv.monitoring_setup_manager.create_setup(link) + return link diff --git a/app/discover/link_finders/find_links_for_instance_vnics.py b/app/discover/link_finders/find_links_for_instance_vnics.py index 975ab1a..1dfb818 100644 --- a/app/discover/link_finders/find_links_for_instance_vnics.py +++ b/app/discover/link_finders/find_links_for_instance_vnics.py @@ -49,6 +49,8 @@ class FindLinksForInstanceVnics(FindLinks): network_id = net['network']['id'] v['network'] = network_id self.inv.set(v) + if self.inv.monitoring_setup_manager: + self.inv.monitoring_setup_manager.create_setup(instance) break state = "up" # TBD link_weight = 0 # TBD diff --git a/app/discover/scan_manager.py b/app/discover/scan_manager.py index 6c46d47..91dd06c 100644 --- a/app/discover/scan_manager.py +++ b/app/discover/scan_manager.py @@ -219,71 +219,74 @@ class ScanManager(Manager): for interval in self.INTERVALS.keys(): self._prepare_scheduled_requests_for_interval(interval) + def handle_scans(self): + self._prepare_scheduled_requests() + + # Find a pending request that is waiting the longest time + results = self.scans_collection \ + .find({'status': ScanStatus.PENDING.value, + 'submit_timestamp': {'$ne': None}}) \ + .sort("submit_timestamp", pymongo.ASCENDING) \ + .limit(1) + + # If no scans are pending, sleep for some time + if results.count() == 0: + time.sleep(self.interval) + else: + scan_request = results[0] + env = scan_request.get('environment') + scan_feature = EnvironmentFeatures.SCANNING + if not self.inv.is_feature_supported(env, scan_feature): + self.log.error("Scanning is not supported for env '{}'" + .format(scan_request.get('environment'))) + self._fail_scan(scan_request) + return + + scan_request['start_timestamp'] = datetime.datetime.utcnow() + scan_request['status'] = ScanStatus.RUNNING.value + self._update_document(scan_request) + + # Prepare scan arguments and run the scan with them + try: + scan_args = self._build_scan_args(scan_request) + + self.log.info("Starting scan for '{}' environment" + .format(scan_args.get('env'))) + self.log.debug("Scan arguments: {}".format(scan_args)) + result, message = ScanController().run(scan_args) + except ScanArgumentsError as e: + self.log.error("Scan request '{id}' " + "has invalid arguments. " + "Errors:\n{errors}" + .format(id=scan_request['_id'], + errors=e)) + self._fail_scan(scan_request) + except Exception as e: + self.log.exception(e) + self.log.error("Scan request '{}' has failed." + .format(scan_request['_id'])) + self._fail_scan(scan_request) + else: + # Check is scan returned success + if not result: + self.log.error(message) + self.log.error("Scan request '{}' has failed." + .format(scan_request['_id'])) + self._fail_scan(scan_request) + return + + # update the status and timestamps. + self.log.info("Request '{}' has been scanned. ({})" + .format(scan_request['_id'], message)) + end_time = datetime.datetime.utcnow() + scan_request['end_timestamp'] = end_time + self._complete_scan(scan_request, message) + def do_action(self): self._clean_up() try: while True: - self._prepare_scheduled_requests() - - # Find a pending request that is waiting the longest time - results = self.scans_collection \ - .find({'status': ScanStatus.PENDING.value, - 'submit_timestamp': {'$ne': None}}) \ - .sort("submit_timestamp", pymongo.ASCENDING) \ - .limit(1) - - # If no scans are pending, sleep for some time - if results.count() == 0: - time.sleep(self.interval) - else: - scan_request = results[0] - env = scan_request.get('environment') - scan_feature = EnvironmentFeatures.SCANNING - if not self.inv.is_feature_supported(env, scan_feature): - self.log.error("Scanning is not supported for env '{}'" - .format(scan_request.get('environment'))) - self._fail_scan(scan_request) - continue - - scan_request['start_timestamp'] = datetime.datetime.utcnow() - scan_request['status'] = ScanStatus.RUNNING.value - self._update_document(scan_request) - - # Prepare scan arguments and run the scan with them - try: - scan_args = self._build_scan_args(scan_request) - - self.log.info("Starting scan for '{}' environment" - .format(scan_args.get('env'))) - self.log.debug("Scan arguments: {}".format(scan_args)) - result, message = ScanController().run(scan_args) - except ScanArgumentsError as e: - self.log.error("Scan request '{id}' " - "has invalid arguments. " - "Errors:\n{errors}" - .format(id=scan_request['_id'], - errors=e)) - self._fail_scan(scan_request) - except Exception as e: - self.log.exception(e) - self.log.error("Scan request '{}' has failed." - .format(scan_request['_id'])) - self._fail_scan(scan_request) - else: - # Check is scan returned success - if not result: - self.log.error(message) - self.log.error("Scan request '{}' has failed." - .format(scan_request['_id'])) - self._fail_scan(scan_request) - continue - - # update the status and timestamps. - self.log.info("Request '{}' has been scanned. ({})" - .format(scan_request['_id'], message)) - end_time = datetime.datetime.utcnow() - scan_request['end_timestamp'] = end_time - self._complete_scan(scan_request, message) + self.handle_scans() finally: self._clean_up() diff --git a/app/discover/scan_metadata_parser.py b/app/discover/scan_metadata_parser.py index df27e18..8757f79 100644 --- a/app/discover/scan_metadata_parser.py +++ b/app/discover/scan_metadata_parser.py @@ -49,21 +49,28 @@ class ScanMetadataParser(MetadataParser): self.add_error('missing or empty fetcher in scanner {} type #{}' .format(scanner_name, str(type_index))) elif isinstance(fetcher, str): + error_str = None try: - module_name = ClassResolver.get_module_file_by_class_name(fetcher) + get_module = ClassResolver.get_module_file_by_class_name + module_name = get_module(fetcher) fetcher_package = module_name.split("_")[0] if package: fetcher_package = ".".join((package, fetcher_package)) - instance = ClassResolver.get_instance_of_class(package_name=fetcher_package, - module_name=module_name, - class_name=fetcher) - except ValueError: - instance = None - if not instance: + # get the fetcher qualified class but not a class instance + # instances will be created just-in-time (before fetching): + # this avoids init of access classes not needed in some envs + get_class = ClassResolver.get_fully_qualified_class + class_qualified = get_class(fetcher, fetcher_package, + module_name) + except ValueError as e: + class_qualified = None + error_str = str(e) + if not class_qualified: self.add_error('failed to find fetcher class {} in scanner {}' - ' type #{}' - .format(fetcher, scanner_name, type_index)) - scan_type[self.FETCHER] = instance + ' type #{} ({})' + .format(fetcher, scanner_name, type_index, + error_str)) + scan_type[self.FETCHER] = class_qualified elif isinstance(fetcher, dict): is_folder = fetcher.get('folder', False) if not is_folder: @@ -81,7 +88,6 @@ class ScanMetadataParser(MetadataParser): def validate_children_scanner(self, scanner_name: str, type_index: int, scanners: dict, scan_type: dict): - scanner = scanners[scanner_name] if 'children_scanner' in scan_type: children_scanner = scan_type.get('children_scanner') if not isinstance(children_scanner, str): diff --git a/app/discover/scanner.py b/app/discover/scanner.py index 1fbcc68..8aac40b 100644 --- a/app/discover/scanner.py +++ b/app/discover/scanner.py @@ -26,6 +26,10 @@ from utils.ssh_connection import SshError class Scanner(Fetcher): + + ENV_TYPE_OPENSTACK = 'OpenStack' + ENV_TYPE_KUBERNETES = 'Kubernetes' + config = None environment = None env = None @@ -82,16 +86,21 @@ class Scanner(Fetcher): def check_type_env(self, type_to_fetch): # check if type is to be run in this environment - if "environment_condition" not in type_to_fetch: - return True - env_cond = type_to_fetch.get("environment_condition", {}) + basic_cond = {'environment_type': self.ENV_TYPE_OPENSTACK} + env_cond = type_to_fetch.get("environment_condition", {}) \ + if "environment_condition" in type_to_fetch \ + else basic_cond if not env_cond: - return True + env_cond = basic_cond + if 'environment_type' not in env_cond: + env_cond.update(basic_cond) if not isinstance(env_cond, dict): self.log.warn('illegal environment_condition given ' 'for type {}'.format(type_to_fetch['type'])) return True conf = self.config.get_env_config() + if 'environment_type' not in conf: + conf.update(basic_cond) for attr, required_val in env_cond.items(): if attr == "mechanism_drivers": if "mechanism_drivers" not in conf: @@ -120,6 +129,9 @@ class Scanner(Fetcher): # get Fetcher instance fetcher = type_to_fetch["fetcher"] + if not isinstance(fetcher, Fetcher): + type_to_fetch['fetcher'] = fetcher() # make it an instance + fetcher = type_to_fetch["fetcher"] fetcher.set_env(self.get_env()) # get children_scanner instance @@ -254,7 +266,6 @@ class Scanner(Fetcher): def load_link_finders_metadata(self): parser = FindLinksMetadataParser() - conf = self.config.get_env_config() finders_file = os.path.join(self.get_run_app_path(), 'config', FindLinksMetadataParser.FINDERS_FILE) diff --git a/app/install/calipso-installer.py b/app/install/calipso-installer.py index 84b10da..78bb927 100644 --- a/app/install/calipso-installer.py +++ b/app/install/calipso-installer.py @@ -292,6 +292,22 @@ def start_ui(host, dbuser, dbpassword, webport, dbport): environment=[root_url, mongo_url, LDAP_CONFIG]) +def start_test(): + name = "calipso-test" + if container_started(name): + return + print("\nstarting container {}...\n".format(name)) + image_name = "korenlev/calipso:test" + download_image(image_name) + ports = {'22/tcp': 10022} + DockerClient.containers.run(image_name, + detach=True, + name=name, + ports=ports, + restart_policy=RESTART_POLICY, + environment=[PYTHON_PATH, MONGO_CONFIG], + volumes=calipso_volume) + # check and stop a calipso container by given name def container_stop(container_name): if not container_started(container_name, print_message=False): @@ -395,7 +411,7 @@ else: container = "" action = "" -container_names = ["calipso-ui", "calipso-scan", "calipso-listen", +container_names = ["calipso-ui", "calipso-scan", "calipso-test", "calipso-listen", "calipso-ldap", "calipso-api", "calipso-sensu", "calipso-mongo"] container_actions = ["stop", "start"] while action not in container_actions: @@ -460,6 +476,9 @@ if action == "start": if container == "calipso-scan" or container == "all": start_scan() time.sleep(1) + if container == "calipso-test" or container == "all": + start_test() + time.sleep(1) if container == "calipso-sensu" or container == "all": start_sensu(args.uchiwaport, args.sensuport, args.rabbitport, args.rabbitmport) time.sleep(1) diff --git a/app/install/db/clique_types.json b/app/install/db/clique_types.json index 77e2d7d..624de70 100644 --- a/app/install/db/clique_types.json +++ b/app/install/db/clique_types.json @@ -26,6 +26,23 @@ ], "name" : "vservice" }, +{ + "environment" : "config_based_example", + "focal_point_type" : "vservice", + "link_types" : [ + "vservice-vnic", + "vnic-vedge", + "vedge-otep", + "otep-vconnector", + "vconnector-host_pnic", + "host_pnic-network" + ], + "name" : "vservice_config_based", + "distribution" : "Mirantis", + "distribution_version" : "6.0", + "mechanism_drivers" : "OVS", + "type_drivers" : "vxlan" +}, { "environment" : "ANY", "focal_point_type" : "network", @@ -135,5 +152,14 @@ "vnic-vservice" ], "name" : "network" +}, +{ + "name" : "instance", + "use_implicit_links" : true, + "link_types" : [ + "instance-network" + ], + "environment" : "implicit-links-ex", + "focal_point_type" : "instance" } ] diff --git a/app/install/db/constants.json b/app/install/db/constants.json index 6912eeb..8ea89e9 100644 --- a/app/install/db/constants.json +++ b/app/install/db/constants.json @@ -58,6 +58,27 @@ ], "name" : "log_levels" }, +{ + "data" : [ + { + "label" : "OpenStack", + "value" : "OpenStack" + }, + { + "label" : "Kubernetes", + "value" : "Kubernetes" + }, + { + "label" : "VMware", + "value" : "VMware" + }, + { + "label" : "Bare-metal", + "value" : "Bare-metal" + } + ], + "name" : "environment_types" +}, { "data" : [ { @@ -529,6 +550,10 @@ "value" : "10239", "label" : "10239" }, + { + "label" : "10307", + "value" : "10307" + }, { "value" : "10918", "label" : "10918" @@ -727,6 +752,14 @@ { "label" : "switch", "value" : "switch" + }, + { + "value" : "namespace", + "label" : "namespace" + }, + { + "value" : "namespaces_folder", + "label" : "namespaces_folder" } ] }, diff --git a/app/install/db/environments_config.json b/app/install/db/environments_config.json index d7157e7..80bc6aa 100644 --- a/app/install/db/environments_config.json +++ b/app/install/db/environments_config.json @@ -37,7 +37,8 @@ "server_name" : "sensu_server", "env_type" : "production", "provision" : "None", - "name" : "Monitoring", + "name" : "Monitoring", + "install_monitoring_client": false, "ssh_port" : "20022", "rabbitmq_pass" : "dummy_pwd", "ssh_password" : "dummy_pwd", @@ -55,7 +56,7 @@ } ], "enable_monitoring" : true, - "name" : "DEMO-ENVIRONMENT-SCHEME", + "name" : "DEMO-OpenStack", "distribution" : "Mirantis", "distribution_version" : "8.0", "last_scanned" : "filled-by-scanning", @@ -74,6 +75,93 @@ "wNLeBJxNDyw8G7Ssg" ] }, - "type" : "environment" + "type" : "environment", + "environment_type" : "OpenStack" +}, +{ + "user" : "wNLeBJxNDyw8G7Ssg", + "name" : "DEMO-Kubernetes", + "last_scanned" : "filled-by-scanning", + "auth" : { + "view-env" : [ + "wNLeBJxNDyw8G7Ssg" + ], + "edit-env" : [ + "wNLeBJxNDyw8G7Ssg" + ] + }, + "type_drivers" : "vxlan", + "distribution_version" : "8.0", + "enable_monitoring" : true, + "operational" : "stopped", + "mechanism_drivers" : [ + "OVS" + ], + "type" : "environment", + "distribution" : "Mirantis", + "listen" : true, + "configuration" : [ + { + "user" : "adminuser", + "name" : "OpenStack", + "pwd" : "dummy_pwd", + "host" : "10.0.0.1", + "admin_token" : "dummy_token", + "port" : "5000" + }, + { + "host" : "10.56.20.78", + "name" : "Kubernetes", + "user" : "koren", + "token" : "baba-token-xyz", + "port" : "6443" + }, + { + "user" : "mysqluser", + "name" : "mysql", + "pwd" : "dummy_pwd", + "port" : "3307", + "host" : "10.0.0.1" + }, + { + "user" : "sshuser", + "name" : "CLI", + "pwd" : "dummy_pwd", + "host" : "10.0.0.1" + }, + { + "user" : "rabbitmquser", + "name" : "AMQP", + "pwd" : "dummy_pwd", + "port" : "5673", + "host" : "10.0.0.1" + }, + { + "name" : "Monitoring", + "install_monitoring_client": false, + "api_port" : 4567, + "ssh_port" : "20022", + "rabbitmq_pass" : "dummy_pwd", + "env_type" : "production", + "rabbitmq_port" : "5671", + "server_ip" : "10.0.0.1", + "config_folder" : "/local_dir/sensu_config", + "type" : "Sensu", + "provision" : "None", + "ssh_user" : "root", + "ssh_password" : "dummy_pwd", + "rabbitmq_user" : "sensu", + "server_name" : "sensu_server" + }, + { + "user" : "admin", + "name" : "ACI", + "pwd" : "dummy_pwd", + "host" : "10.1.1.104" + } + ], + "app_path" : "/home/scan/calipso_prod/app", + "scanned" : false, + "environment_type" : "Kubernetes" } ] diff --git a/app/install/db/monitoring_config_templates.json b/app/install/db/monitoring_config_templates.json index 9bddfa2..b5c47df 100644 --- a/app/install/db/monitoring_config_templates.json +++ b/app/install/db/monitoring_config_templates.json @@ -310,6 +310,34 @@ "monitoring_system" : "sensu", "type" : "client_check_link_vnic-vconnector.json" }, +{ + "side" : "client", + "order" : "1", + "condition" : { + "mechanism_drivers" : [ + "OVS" + ] + }, + "config" : { + "checks" : { + "{objtype}_{objid}" : { + "interval" : 15, + "command" : "check_vconnector_ovs.py {name}", + "standalone" : true, + "type": "metric", + "subscribers" : [ + "base" + ], + "handlers" : [ + "file", + "osdna-monitor" + ] + } + } + }, + "monitoring_system" : "sensu", + "type" : "client_check_vconnector_ovs.json" +}, { "side" : "client", "order" : "1", @@ -394,5 +422,28 @@ }, "monitoring_system" : "sensu", "type" : "client_check_vservice.json" +}, +{ + "side" : "client", + "order" : "1", + "config" : { + "checks" : { + "{objtype}_{objid}" : { + "standalone" : true, + "interval" : 15, + "command" : "PYTHONPATH=/etc/sensu/plugins check_instance_communications.py {services_and_vnics}", + "handlers" : [ + "file", + "osdna-monitor" + ], + "type" : "metric", + "subscribers" : [ + "base" + ] + } + } + }, + "monitoring_system" : "sensu", + "type" : "client_check_instance.json" } ] diff --git a/app/install/db/supported_environments.json b/app/install/db/supported_environments.json index c2c376b..baa3150 100644 --- a/app/install/db/supported_environments.json +++ b/app/install/db/supported_environments.json @@ -12,6 +12,21 @@ "monitoring" : true } }, + { + "environment" : { + "distribution_version" : [ + "10307" + ], + "distribution" : "Mercury", + "type_drivers" : "vlan", + "mechanism_drivers" : "OVS" + }, + "features" : { + "scanning" : true, + "monitoring" : false, + "listening" : true + } + }, { "environment" : { "distribution" : "Devstack", diff --git a/app/monitoring/checks/check_instance_communictions.py b/app/monitoring/checks/check_instance_communictions.py new file mode 100644 index 0000000..d3a94b7 --- /dev/null +++ b/app/monitoring/checks/check_instance_communictions.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + +# find status of instance network +# For each instance vNIC - take the MAC address +# For each vService in the same network as the instance, +# use local_service_id attribute in the following command in the network node: +# "ip netns exec arp -n" +# look for the instance vNIC's mac_address to appear in the response +# for each mac_address: +# - if Flag 'C' = 'Complete' - mark result OK for that instance, +# - 'I' = 'Incomplete' - mark as 'warn', +# - no mac_address mark as 'error' + +import sys +import subprocess + +from binary_converter import binary2str + + +arp_headers = ['Address', 'HWtype', 'HWaddress', 'Flags', 'Mask', 'Iface'] +arp_mac_pos = arp_headers.index('HWaddress') +arp_flags_pos = arp_headers.index('Flags') + + +def check_vnic_tuple(vnic_and_service: str): + tuple_parts = vnic_and_service.split(',') + local_service_id = tuple_parts[0] + mac_address = tuple_parts[1] + check_output = None + try: + netns_cmd = 'ip netns exec {} arp -n'.format(local_service_id) + check_output = 'MAC={}, local_service_id={}\n'\ + .format(mac_address, local_service_id) + netns_out = subprocess.check_output([netns_cmd], + stderr=subprocess.STDOUT, + shell=True) + netns_out = binary2str(netns_out) + check_output += '{}\n'.format(netns_out) + netns_lines = netns_out.splitlines() + if not netns_lines or \ + netns_lines[0].endswith('No such file or directory'): + check_rc = 2 + else: + mac_found = False + flags = None + for l in netns_lines: + line_parts = l.split() + line_mac = line_parts[arp_mac_pos] + if len(line_parts) > arp_mac_pos and line_mac == mac_address: + mac_found = True + flags = line_parts[arp_flags_pos] + break + if mac_found: + check_rc = 1 if flags == 'I' else 0 + else: + check_rc = 2 + except subprocess.CalledProcessError as e: + check_output = str(e) + check_rc = 2 + return check_rc, check_output + + +if len(sys.argv) < 2: + print('usage: ' + sys.argv[0] + + ' ,[;<>,<>]...') + exit(1) + +rc = 0 +output = '' +vnics = str(sys.argv[1]).split(';') +for vnic_tuple in vnics: + tuple_ret, out = check_vnic_tuple(vnic_tuple) + rc = min(rc, tuple_ret) + output += out +print(output) +exit(rc) diff --git a/app/monitoring/checks/check_vconnector.py b/app/monitoring/checks/check_vconnector.py new file mode 100644 index 0000000..237a195 --- /dev/null +++ b/app/monitoring/checks/check_vconnector.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + +# find status of vconnector +# vconnector object name defines name of bridge +# use "brctl showmacs ", return ERROR if 'No such device' is returned + +import sys +import subprocess + +from binary_converter import binary2str + + +if len(sys.argv) < 2: + print('usage: ' + sys.argv[0] + ' ') + exit(1) +bridge_name = str(sys.argv[1]) + +rc = 0 + +cmd = None +out = '' +try: + cmd = "brctl showmacs {}".format(bridge_name) + out = subprocess.check_output([cmd], + stderr=subprocess.STDOUT, + shell=True) + out = binary2str(out) + lines = out.splitlines() + if not lines or lines[0].endswith('No such device'): + rc = 2 + else: + print(out) +except subprocess.CalledProcessError as e: + rc = 2 + out = str(e) + +if rc != 0: + print('Failed to find vConnector {}:\n{}\n' + .format(bridge_name, out)) + +exit(rc) diff --git a/app/monitoring/handlers/handle_vconnector.py b/app/monitoring/handlers/handle_vconnector.py new file mode 100644 index 0000000..85ee05f --- /dev/null +++ b/app/monitoring/handlers/handle_vconnector.py @@ -0,0 +1,28 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +# handle monitoring event for pNIC objects + +from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler +from utils.special_char_converter import SpecialCharConverter + + +class HandleVconnector(MonitoringCheckHandler): + + def handle(self, obj_id, check_result): + object_id = obj_id[:obj_id.index('-')] + mac = obj_id[obj_id.index('-')+1:] + converter = SpecialCharConverter() + mac_address = converter.decode_special_characters(mac) + object_id += '-' + mac_address + doc = self.doc_by_id(object_id) + if not doc: + return 1 + self.keep_result(doc, check_result) + return check_result['status'] diff --git a/app/monitoring/handlers/monitor.py b/app/monitoring/handlers/monitor.py index 9caed74..2495110 100755 --- a/app/monitoring/handlers/monitor.py +++ b/app/monitoring/handlers/monitor.py @@ -12,11 +12,15 @@ # handle monitoring events import argparse +import datetime import json import sys +from discover.configuration import Configuration +from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler from utils.inventory_mgr import InventoryMgr from utils.mongo_access import MongoAccess +from utils.special_char_converter import SpecialCharConverter from utils.util import ClassResolver @@ -32,7 +36,9 @@ class Monitor: MongoAccess.set_config_file(self.args.mongo_config) self.inv = InventoryMgr() self.inv.set_collections(self.args.inventory) + self.configuration = Configuration() self.input_text = None + self.converter = SpecialCharConverter() def get_args(self): parser = argparse.ArgumentParser() @@ -125,13 +131,83 @@ class Monitor: return handler def get_handler(self, check_type, obj_type): - basic_handling_types = ['vedge', 'vservice'] + basic_handling_types = ['instance', 'vedge', 'vservice', 'vconnector'] if obj_type not in basic_handling_types: return self.get_handler_by_type(check_type, obj_type) from monitoring.handlers.basic_check_handler \ import BasicCheckHandler return BasicCheckHandler(self.args) + def check_link_interdependency_for(self, + object_id: str, + from_type: str=None, + to_type: str=None): + if from_type is not None and to_type is not None or \ + from_type is None and to_type is None: + raise ValueError('check_link_interdependency: ' + 'supply one of from_type/to_type') + obj_id = self.converter.decode_special_characters(object_id) + obj = self.inv.get_by_id(environment=self.args.env, item_id=obj_id) + if not obj: + self.inv.log.error('check_link_interdependency: ' + 'failed to find object with ID: {}' + .format(object_id)) + return + if 'status' not in obj: + return + id_attr = 'source_id' if from_type is None else 'target_id' + link_type = '{}-{}'.format( + from_type if from_type is not None else obj['type'], + to_type if to_type is not None else obj['type']) + condition = { + 'environment': self.args.env, + 'link_type': link_type, + id_attr: obj_id + } + link = self.inv.find_one(search=condition, collection='links') + if not link: + self.inv.log.error('check_link_interdependency: ' + 'failed to find {} link with {}: {}' + .format(link_type, id_attr, obj_id)) + return + other_id_attr = '{}_id' \ + .format('source' if from_type is not None else 'target') + other_obj = self.inv.get_by_id(environment=self.args.env, + item_id=link[other_id_attr]) + if not other_obj: + self.inv.log.error('check_link_interdependency: ' + 'failed to find {} with ID: {} (link type: {})' + .format(other_id_attr, link[other_id_attr], + link_type)) + return + if 'status' not in other_obj: + return + status = 'Warning' + if obj['status'] == 'OK' and other_obj['status'] == 'OK': + status = 'OK' + elif obj['status'] == 'OK' and other_obj['status'] == 'OK': + status = 'OK' + link['status'] = status + time_format = MonitoringCheckHandler.TIME_FORMAT + timestamp1 = obj['status_timestamp'] + t1 = datetime.datetime.strptime(timestamp1, time_format) + timestamp2 = other_obj['status_timestamp'] + t2 = datetime.datetime.strptime(timestamp2, time_format) + timestamp = max(t1, t2) + link['status_timestamp'] = datetime.datetime.strftime(timestamp, + time_format) + self.inv.set(link, self.inv.collections['links']) + + def check_link_interdependency(self, object_id: str, object_type: str): + conf = self.configuration.get_env_config() + if 'OVS' in conf['mechanism_drivers']: + if object_type == 'vedge': + self.check_link_interdependency_for(object_id, + to_type='host_pnic') + if object_type == 'host_pnic': + self.check_link_interdependency_for(object_id, + from_type='vedge') + def process_input(self): check_result_full = json.loads(self.input_text) check_client = check_result_full['client'] @@ -142,14 +218,19 @@ class Monitor: monitor.find_object_type_and_id(name) if 'environment' in check_client: self.args.env = check_client['environment'] + else: + raise ValueError('Check client should contain environment name') + self.configuration.use_env(self.args.env) check_handler = self.get_handler(check_type, object_type) if check_handler: check_handler.handle(object_id, check_result) + self.check_link_interdependency(object_id, object_type) def process_check_result(self): self.read_input() self.process_input() + monitor = Monitor() monitor.process_check_result() diff --git a/app/monitoring/handlers/monitoring_check_handler.py b/app/monitoring/handlers/monitoring_check_handler.py index 1436a46..c1f70fb 100644 --- a/app/monitoring/handlers/monitoring_check_handler.py +++ b/app/monitoring/handlers/monitoring_check_handler.py @@ -21,13 +21,13 @@ from utils.logging.full_logger import FullLogger from utils.special_char_converter import SpecialCharConverter from utils.string_utils import stringify_datetime -TIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z' SOURCE_SYSTEM = 'Sensu' ERROR_LEVEL = ['info', 'warn', 'error'] class MonitoringCheckHandler(SpecialCharConverter): STATUS_LABEL = ['OK', 'Warning', 'Error'] + TIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z' def __init__(self, args): super().__init__() @@ -61,7 +61,7 @@ class MonitoringCheckHandler(SpecialCharConverter): else status if status_text: doc['status_text'] = status_text - doc['status_timestamp'] = strftime(TIME_FORMAT, timestamp) + doc['status_timestamp'] = strftime(self.TIME_FORMAT, timestamp) if 'link_type' in doc: self.inv.write_link(doc) else: diff --git a/app/monitoring/setup/monitoring_check_handler.py b/app/monitoring/setup/monitoring_check_handler.py index c453439..d1b863d 100644 --- a/app/monitoring/setup/monitoring_check_handler.py +++ b/app/monitoring/setup/monitoring_check_handler.py @@ -8,7 +8,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 # ############################################################################### from monitoring.setup.monitoring_handler import MonitoringHandler -from utils.inventory_mgr import InventoryMgr from utils.special_char_converter import SpecialCharConverter @@ -28,14 +27,13 @@ class MonitoringCheckHandler(MonitoringHandler, SpecialCharConverter): type_str = values['check_type'] if 'check_type' in values else \ (o['type'] if 'type' in o else 'link_' + o['link_type']) file_type = 'client_check_' + type_str + '.json' - host = o['host'] + host = values['host'] if 'host' in values else o['host'] sub_dir = '/host/' + host content = self.prepare_config_file( file_type, {'side': 'client', 'type': file_type}) # need to put this content inside client.json file client_file = 'client.json' - host = o['host'] client_file_content = self.get_config_from_db(host, client_file) # merge checks attribute from current content into client.json checks = client_file_content['config']['checks'] \ @@ -53,3 +51,14 @@ class MonitoringCheckHandler(MonitoringHandler, SpecialCharConverter): } content = client_file_content self.write_config_file(client_file, sub_dir, host, content) + + def get_check_from_db(self, o, postfix=''): + client_config = self.get_config_from_db(o. get('host', ''), + 'client.json') + if not client_config: + return {} + checks = client_config.get('config', {}).get('checks', {}) + objid = self.encode_special_characters(o.get('id', '')) + object_check_id = '{}_{}{}'.format(o.get('type'), objid, postfix) + check = checks.get(object_check_id, {}) + return check diff --git a/app/monitoring/setup/monitoring_host.py b/app/monitoring/setup/monitoring_host.py index 9450cf6..0b9f420 100644 --- a/app/monitoring/setup/monitoring_host.py +++ b/app/monitoring/setup/monitoring_host.py @@ -12,6 +12,7 @@ import os from os.path import join, sep from monitoring.setup.monitoring_handler import MonitoringHandler +from monitoring.setup.sensu_client_installer import SensuClientInstaller RABBITMQ_CONFIG_FILE = 'rabbitmq.json' RABBITMQ_CONFIG_ATTR = 'rabbitmq' @@ -27,13 +28,14 @@ class MonitoringHost(MonitoringHandler): # add monitoring setup for remote host def create_setup(self, o): + host_id = o.get('host', '') + self.install_sensu_on_host(host_id) sensu_host_files = [ 'transport.json', 'rabbitmq.json', 'client.json' ] server_ip = self.env_monitoring_config['server_ip'] - host_id = o['host'] sub_dir = join('/host', host_id) config = copy.copy(self.env_monitoring_config) env_name = self.configuration.env_name @@ -88,3 +90,10 @@ class MonitoringHost(MonitoringHandler): # this configuration requires SSL # keep the path of the files for later use self.fetch_ssl_files.append(path) + + def install_sensu_on_host(self, host_id): + auto_install = self.env_monitoring_config \ + .get('install_monitoring_client', False) + if auto_install: + installer = SensuClientInstaller(self.env, host_id) + installer.install() diff --git a/app/monitoring/setup/monitoring_instance.py b/app/monitoring/setup/monitoring_instance.py new file mode 100644 index 0000000..b376441 --- /dev/null +++ b/app/monitoring/setup/monitoring_instance.py @@ -0,0 +1,67 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from monitoring.setup.monitoring_simple_object import MonitoringSimpleObject + + +class MonitoringInstance(MonitoringSimpleObject): + + def __init__(self, env): + super().__init__(env) + + # monitoring setup for instance can only be done after vNIC is found + # and network for vNIC is set, so the first call will not do anything + def create_setup(self, instance: dict): + vnics = self.inv.find_items({ + 'environment': self.get_env(), + 'type': 'vnic', + 'vnic_type': 'instance_vnic', + 'id_path': {'$regex': '^{}/'.format(instance['id_path'])} + }) + for vnic in vnics: + self.add_instance_communication_monitoring(instance, vnic) + + # for instance we keep list of instance vNICs and services to use in call + # to check_instance_communications.py + # add this vNIC to the list with the corresponding + def add_instance_communication_monitoring(self, instance: dict, vnic: dict): + service = self.get_service_for_vnic(vnic) + if not service: + return + check = self.get_check_from_db(instance) + services_and_vnics = check.get('command', '') + if services_and_vnics: + services_and_vnics = \ + services_and_vnics[services_and_vnics.index('.py')+4:] + services_and_vnics_list = \ + services_and_vnics.split(';') if services_and_vnics \ + else [] + service_and_vnic = '{},{}'.format(service.get('local_service_id', ''), + vnic.get('id')) + if service_and_vnic in services_and_vnics_list: + return # we already have this tuple define + services_and_vnics_list.append(service_and_vnic) + values = { + 'objtype': 'instance', + 'objid': self.encode_special_characters(instance['id']), + 'host': service['host'], + 'services_and_vnics': ';'.join(services_and_vnics_list) + } + self.create_monitoring_for_object(instance, values) + + def get_service_for_vnic(self, vnic: dict) -> dict: + services = self.inv.find_items({'environment': self.get_env(), + 'type': 'vservice', + 'network': vnic.get('network', '')}) + if not services: + return {} + dhcp = next(s for s in services if s.get('service_type') == 'dhcp') + if dhcp: + return dhcp # If we have both DHCP and router, return the DHCP + return services[0] # currently only DHCP and router services diff --git a/app/monitoring/setup/monitoring_setup_manager.py b/app/monitoring/setup/monitoring_setup_manager.py index bc4fe01..8b7693a 100644 --- a/app/monitoring/setup/monitoring_setup_manager.py +++ b/app/monitoring/setup/monitoring_setup_manager.py @@ -11,12 +11,14 @@ from monitoring.setup.monitoring_handler import MonitoringHandler from monitoring.setup.monitoring_host import MonitoringHost +from monitoring.setup.monitoring_instance import MonitoringInstance from monitoring.setup.monitoring_link_vnic_vconnector \ import MonitoringLinkVnicVconnector from monitoring.setup.monitoring_pnic import MonitoringPnic from monitoring.setup.monitoring_otep import MonitoringOtep from monitoring.setup.monitoring_vedge import MonitoringVedge from monitoring.setup.monitoring_vnic import MonitoringVnic +from monitoring.setup.monitoring_vconnector import MonitoringVconnector from monitoring.setup.monitoring_vservice import MonitoringVservice @@ -31,7 +33,9 @@ class MonitoringSetupManager(MonitoringHandler): "otep": MonitoringOtep(env), "vedge": MonitoringVedge(env), "host_pnic": MonitoringPnic(env), + "instance": MonitoringInstance(env), "vnic": MonitoringVnic(env), + "vconnector": MonitoringVconnector(env), "vservice": MonitoringVservice(env), "vnic-vconnector": MonitoringLinkVnicVconnector(env)} diff --git a/app/monitoring/setup/monitoring_vconnector.py b/app/monitoring/setup/monitoring_vconnector.py new file mode 100644 index 0000000..9ddc6af --- /dev/null +++ b/app/monitoring/setup/monitoring_vconnector.py @@ -0,0 +1,24 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from monitoring.setup.monitoring_simple_object import MonitoringSimpleObject + + +class MonitoringVconnector(MonitoringSimpleObject): + + # add monitoring setup for remote host + def create_setup(self, o): + type = 'vconnector' + env_config = self.configuration.get_env_config() + vpp_or_ovs = 'vpp' if 'VPP' in env_config['mechanism_drivers'] \ + else 'ovs' + type_str = '{}_{}'.format(type, vpp_or_ovs) + self.setup(type, o, values={'check_type': type_str, + 'name': o['name']}) + diff --git a/app/monitoring/setup/sensu_client_installer.py b/app/monitoring/setup/sensu_client_installer.py new file mode 100644 index 0000000..72a8bbb --- /dev/null +++ b/app/monitoring/setup/sensu_client_installer.py @@ -0,0 +1,158 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import os.path +from pkg_resources import parse_version + +from monitoring.setup.monitoring_handler import MonitoringHandler +from utils.inventory_mgr import InventoryMgr + + +class SensuClientInstaller(MonitoringHandler): + + UBUNTU = 'ubuntu' + CENTOS = 'centos' + + INSTALL_CMD = { + UBUNTU: 'dpkg -i {}', + CENTOS: 'rpm -i {}' + } + PERMISSIONS_CMD = { + UBUNTU: '', + CENTOS: 'usermod -aG wheel sensu' + } + SUDOERS_FILE = '/etc/sudoers' + + available_downloads = {} + + def __init__(self, env: str, host_id: str): + super().__init__(env) + self.cli_ssh = self.get_ssh(host_id) + self.inv = InventoryMgr() + self.host = self.inv.get_by_id(env, host_id) + self.server = self.env_monitoring_config.get('server_ip') + self.server_cli_ssh = self.get_ssh(self.server) + self.ubuntu_dist = None + self.required_package = None + + def install(self): + pkg_to_install = self.get_pkg_to_install() + if not pkg_to_install: + return + try: + self.fetch_package(pkg_to_install) + self.install_package(pkg_to_install) + self.set_permissions() + except SystemError as e: + self.log.error('Sensu install on host {} failed: {}' + .format(self.host, str(e))) + return + + @staticmethod + def get_attr_from_output(output_lines: list, attr: str) -> str: + matches = [l for l in output_lines if l.startswith(attr)] + if not matches: + return '' + line = matches[0] + return SensuClientInstaller.get_attr_from_output_line(line) + + @staticmethod + def get_attr_from_output_line(output_line: str): + val = output_line[output_line.index(':')+1:].strip() + return val + + INSTALLED = 'Installed: ' + CANDIDATE = 'Candidate: ' + SENSU_DIR = '/opt/sensu' + SENSU_PKG_DIR = '/etc/sensu/pkg' + SENSU_PKG_DIR_LOCAL = '/tmp/sensu_pkg' + SENSU_VERSION_FILE = '/opt/sensu/version-manifest.txt' + + def find_available_downloads(self): + ls_output = self.server_cli_ssh.exec('ls -R {}' + .format(self.SENSU_PKG_DIR)) + ls_lines = ls_output.splitlines() + last_target_dir = None + for line in ls_lines: + if line[-4:] in ['/32:', '/64:']: + last_target_dir = line.replace(self.SENSU_PKG_DIR, '') + continue + elif last_target_dir: + target_dir = last_target_dir.strip(os.path.sep).strip(':') + self.available_downloads[target_dir] = line + last_target_dir = None + else: + last_target_dir = None + + def find_available_package(self, os_details: dict): + if not self.available_downloads: + self.find_available_downloads() + distribution = os_details['ID'] + version = os_details['version'].split()[-2].lower() + arch = os_details['architecure'][-2:] + download_dir = os.path.join(distribution, version, arch) + download_file = self.available_downloads.get(download_dir) + full_path = '' if not download_file \ + else os.path.join(self.SENSU_PKG_DIR, download_dir, download_file) + return download_file, full_path + + @staticmethod + def find_available_version(download_file: str) -> str: + ver = download_file.replace('sensu', '').strip('-_') + ver = ver[:ver.index('-')] + return ver + + def get_pkg_to_install(self) -> str: + if self.provision == self.provision_levels['none']: + return '' + if not self.host: + return '' + supported_os = [self.UBUNTU, self.CENTOS] + distribution = self.host['OS']['ID'] + if distribution not in [self.UBUNTU, self.CENTOS]: + self.log.error('Sensu client auto-install only supported for: {}' + .format(', '.join(supported_os))) + return '' + cmd = 'if [ -d {} ]; then head -1 {} | sed "s/sensu //"; fi' \ + .format(self.SENSU_DIR, self.SENSU_VERSION_FILE) + installed_version = self.cli_ssh.exec(cmd).strip() + os_details = self.host['OS'] + available_pkg, pkg_path = self.find_available_package(os_details) + available_version = self.find_available_version(available_pkg) + if parse_version(available_version) <= parse_version(installed_version): + return '' + return pkg_path + + def get_local_path(self, pkg_to_install: str): + return os.path.join(self.SENSU_PKG_DIR_LOCAL, + os.path.basename(pkg_to_install)) + + def fetch_package(self, pkg_to_install: str): + self.make_directory(self.SENSU_PKG_DIR_LOCAL) + self.get_file(self.server, pkg_to_install, + self.get_local_path(pkg_to_install)) + local_path = self.get_local_path(pkg_to_install) + self.copy_to_remote_host(self.host['host'], + local_path=local_path, + remote_path=local_path) + + def install_package(self, pkg_to_install): + local_path = self.get_local_path(pkg_to_install) + install_cmd = self.INSTALL_CMD[self.host['OS']['ID']] + self.cli_ssh.exec(install_cmd.format(local_path)) + + def set_permissions(self): + cmd = self.PERMISSIONS_CMD[self.host['OS']['ID']] + if cmd: + self.cli_ssh.exec(cmd) + # add to sudoers file + sudoer_permission = 'sensu ALL=(ALL) NOPASSWD: ALL' + sudoer_cmd = 'grep --silent -w sensu {} || echo "{}" >> {}'\ + .format(self.SUDOERS_FILE, sudoer_permission, self.SUDOERS_FILE) + self.cli_ssh.exec(sudoer_cmd) diff --git a/app/test/api/responders_test/resource/test_clique_types.py b/app/test/api/responders_test/resource/test_clique_types.py index f5e331e..5e52cea 100644 --- a/app/test/api/responders_test/resource/test_clique_types.py +++ b/app/test/api/responders_test/resource/test_clique_types.py @@ -17,10 +17,17 @@ from unittest.mock import patch class TestCliqueTypes(TestBase): - def test_get_clique_types_list_without_env_name(self): - self.validate_get_request(clique_types.URL, - params={}, - expected_code=base.BAD_REQUEST_CODE) + @patch(base.RESPONDER_BASE_READ) + def test_get_all_clique_types_list(self, read): + self.validate_get_request( + clique_types.URL, + params={}, + mocks={ + read: clique_types.CLIQUE_TYPES + }, + expected_code=base.SUCCESSFUL_CODE, + expected_response=clique_types.CLIQUE_TYPES_RESPONSE + ) def test_get_clique_types_with_invalid_filter(self): self.validate_get_request(clique_types.URL, @@ -53,6 +60,28 @@ class TestCliqueTypes(TestBase): expected_code=base.SUCCESSFUL_CODE ) + def test_get_clique_type_with_insufficient_configuration(self): + self.validate_get_request( + clique_types.URL, + params={ + "distribution_version": base.CORRECT_DIST_VER, + }, + expected_code=base.BAD_REQUEST_CODE + ) + + @patch(base.RESPONDER_BASE_READ) + def test_get_clique_type_with_correct_configuration(self, read): + self.validate_get_request( + clique_types.URL, + params=clique_types.TEST_CONFIGURATION, + mocks={ + read: clique_types.CLIQUE_TYPES_WITH_SPECIFIC_CONFIGURATION + }, + expected_response=clique_types. + CLIQUE_TYPES_WITH_SPECIFIC_CONFIGURATION_RESPONSE, + expected_code=base.SUCCESSFUL_CODE + ) + def test_get_clique_types_list_with_wrong_focal_point_type(self): self.validate_get_request(clique_types.URL, params={ @@ -204,9 +233,53 @@ class TestCliqueTypes(TestBase): body=json.dumps(clique_types.NON_DICT_CLIQUE_TYPE), expected_code=base.BAD_REQUEST_CODE) - def test_post_clique_type_without_env_name(self): + @patch(base.RESPONDER_BASE_CHECK_ENVIRONMENT_NAME) + def test_post_clique_type_with_reserved_env_name(self, check_env_name): + self.validate_post_request( + clique_types.URL, + mocks={ + check_env_name: True + }, + body=json.dumps(clique_types.CLIQUE_TYPE_WITH_RESERVED_NAME), + expected_code=base.BAD_REQUEST_CODE + ) + + def test_post_clique_type_without_env_name_and_configuration(self): + self.validate_post_request( + clique_types.URL, + body=json.dumps(clique_types.CLIQUE_TYPE_WITHOUT_ENV_NAME_AND_CONF), + expected_code=base.BAD_REQUEST_CODE + ) + + def test_post_clique_type_with_both_env_name_and_configuration(self): + self.validate_post_request( + clique_types.URL, + body=json.dumps( + clique_types.CLIQUE_TYPE_WITH_BOTH_ENV_AND_CONF), + expected_code=base.BAD_REQUEST_CODE + ) + + @patch(base.RESPONDER_BASE_CHECK_ENVIRONMENT_NAME) + def test_post_clique_type_with_insufficient_configuration(self, check_env_name): + self.validate_post_request( + clique_types.URL, + mocks={ + check_env_name: True + }, + body=json.dumps(clique_types.CLIQUE_TYPE_WITH_INSUFFICIENT_CONF), + expected_code=base.BAD_REQUEST_CODE + ) + + @patch(base.RESPONDER_BASE_READ) + def test_post_clique_type_with_duplicate_configuration(self, read): + data = clique_types.CLIQUE_TYPES_WITH_SPECIFIC_CONFIGURATION[0] + resp = clique_types.CLIQUE_TYPES_WITH_SPECIFIC_CONFIGURATION_RESPONSE + test_data = self.get_updated_data(data, deleted_keys=['id']) self.validate_post_request(clique_types.URL, - body=json.dumps(clique_types.CLIQUE_TYPE_WITHOUT_ENVIRONMENT), + body=json.dumps(test_data), + mocks={ + read: resp, + }, expected_code=base.BAD_REQUEST_CODE) @patch(base.RESPONDER_BASE_CHECK_ENVIRONMENT_NAME) @@ -231,6 +304,17 @@ class TestCliqueTypes(TestBase): CLIQUE_TYPE_WITH_WRONG_FOCAL_POINT_TYPE), expected_code=base.BAD_REQUEST_CODE) + @patch(base.RESPONDER_BASE_READ) + def test_post_clique_type_with_duplicate_focal_point_type(self, read): + test_data = self.get_updated_data(clique_types.CLIQUE_TYPE, + updates={'name': 'test-name'}) + self.validate_post_request(clique_types.URL, + body=json.dumps(test_data), + mocks={ + read: [clique_types.CLIQUE_TYPE], + }, + expected_code=base.BAD_REQUEST_CODE) + def test_post_clique_type_without_link_types(self): self.validate_post_request(clique_types.URL, body=json.dumps( @@ -255,6 +339,18 @@ class TestCliqueTypes(TestBase): body=json.dumps(clique_types.CLIQUE_TYPE_WITHOUT_NAME), expected_code=base.BAD_REQUEST_CODE) + def test_post_clique_type_with_wrong_mechanism_drivers(self): + self.validate_post_request(clique_types.URL, + body=json.dumps(clique_types. + CLIQUE_TYPE_WITH_WRONG_MECH_DRIVERS), + expected_code=base.BAD_REQUEST_CODE) + + def test_post_clique_type_with_wrong_type_drivers(self): + self.validate_post_request(clique_types.URL, + body=json.dumps(clique_types. + CLIQUE_TYPE_WITH_WRONG_TYPE_DRIVERS), + expected_code=base.BAD_REQUEST_CODE) + @patch(base.RESPONDER_BASE_CHECK_ENVIRONMENT_NAME) @patch(base.RESPONDER_BASE_WRITE) def test_post_clique_type(self, write, check_environment_name): diff --git a/app/test/api/responders_test/resource/test_environment_configs.py b/app/test/api/responders_test/resource/test_environment_configs.py index 6356f06..4405f2b 100644 --- a/app/test/api/responders_test/resource/test_environment_configs.py +++ b/app/test/api/responders_test/resource/test_environment_configs.py @@ -9,7 +9,9 @@ ############################################################################### import json +from api.responders.resource.environment_configs import EnvironmentConfigs from test.api.responders_test.test_data import base +from test.api.responders_test.test_data.base import CONSTANTS_BY_NAMES from test.api.test_base import TestBase from test.api.responders_test.test_data import environment_configs from utils.constants import EnvironmentFeatures @@ -23,35 +25,25 @@ class TestEnvironmentConfigs(TestBase): def test_get_environment_configs_list(self, read): self.validate_get_request(environment_configs.URL, params={}, - mocks={ - read: environment_configs.ENV_CONFIGS - }, + mocks={read: environment_configs.ENV_CONFIGS}, expected_code=base.SUCCESSFUL_CODE, expected_response=environment_configs. - ENV_CONFIGS_RESPONSE - ) + ENV_CONFIGS_RESPONSE) def test_get_environment_configs_list_with_invalid_filters(self): self.validate_get_request(environment_configs.URL, - params={ - "unknown": "unknown" - }, + params={"unknown": "unknown"}, expected_code=base.BAD_REQUEST_CODE) @patch(base.RESPONDER_BASE_READ) def test_get_environment_configs_list_with_name(self, read): + mocks = {read: environment_configs.ENV_CONFIGS_WITH_SPECIFIC_NAME} self.validate_get_request(environment_configs.URL, - params={ - "name": environment_configs.NAME - }, - mocks={ - read: environment_configs. - ENV_CONFIGS_WITH_SPECIFIC_NAME - }, + params={"name": environment_configs.NAME}, + mocks=mocks, expected_code=base.SUCCESSFUL_CODE, expected_response=environment_configs. - ENV_CONFIGS_WITH_SPECIFIC_NAME[0] - ) + ENV_CONFIGS_WITH_SPECIFIC_NAME[0]) @patch(base.RESPONDER_BASE_READ) def test_get_environment_configs_list_with_unknown_name(self, read): @@ -82,193 +74,151 @@ class TestEnvironmentConfigs(TestBase): @patch(base.RESPONDER_BASE_READ) def test_get_environment_configs_list_with_distribution(self, read): + config = environment_configs.ENV_CONFIGS_WITH_SPECIFIC_DISTRIBUTION + config_response = \ + environment_configs.ENV_CONFIGS_WITH_SPECIFIC_DISTRIBUTION_RESPONSE self.validate_get_request(environment_configs.URL, params={ "distribution": environment_configs. CORRECT_DISTRIBUTION }, - mocks={ - read: environment_configs. - ENV_CONFIGS_WITH_SPECIFIC_DISTRIBUTION - }, + mocks={read: config}, expected_code=base.SUCCESSFUL_CODE, - expected_response=environment_configs. - ENV_CONFIGS_WITH_SPECIFIC_DISTRIBUTION_RESPONSE) + expected_response=config_response) def test_get_environment_configs_list_with_wrong_mechanism_driver(self): + config = environment_configs.WRONG_MECHANISM_DRIVER self.validate_get_request(environment_configs.URL, - params={ - "mechanism_drivers": - environment_configs.WRONG_MECHANISM_DRIVER - }, + params={"mechanism_drivers": config}, expected_code=base.BAD_REQUEST_CODE) @patch(base.RESPONDER_BASE_READ) def test_get_environment_configs_list_with_mechanism_driver(self, read): + mechanism = environment_configs.CORRECT_MECHANISM_DRIVER + config = environment_configs.ENV_CONFIGS_WITH_SPECIFIC_MECHANISM_DRIVER + config_response = environment_configs.\ + ENV_CONFIGS_WITH_SPECIFIC_MECHANISM_DRIVER_RESPONSE self.validate_get_request(environment_configs.URL, - params={ - "mechanism_drivers": - environment_configs.CORRECT_MECHANISM_DRIVER - }, - mocks={ - read: environment_configs. - ENV_CONFIGS_WITH_SPECIFIC_MECHANISM_DRIVER - }, + params={"mechanism_drivers": mechanism}, + mocks={read: config}, expected_code=base.SUCCESSFUL_CODE, - expected_response=environment_configs. - ENV_CONFIGS_WITH_SPECIFIC_MECHANISM_DRIVER_RESPONSE - ) + expected_response=config_response) def test_get_environment_configs_list_with_wrong_type_driver(self): + driver = environment_configs.WRONG_TYPE_DRIVER self.validate_get_request(environment_configs.URL, - params={ - "type_drivers": - environment_configs.WRONG_TYPE_DRIVER - }, + params={"type_drivers": driver}, expected_code=base.BAD_REQUEST_CODE) @patch(base.RESPONDER_BASE_READ) def test_get_environment_configs_list_with_type_driver(self, read): + driver = environment_configs.CORRECT_TYPE_DRIVER + config = environment_configs.ENV_CONFIGS_WITH_SPECIFIC_TYPE_DRIVER + config_response = environment_configs.\ + ENV_CONFIGS_WITH_SPECIFIC_TYPE_DRIVER_RESPONSE self.validate_get_request(environment_configs.URL, - params={ - "type_drivers": - environment_configs.CORRECT_TYPE_DRIVER - }, - mocks={ - read: environment_configs. - ENV_CONFIGS_WITH_SPECIFIC_TYPE_DRIVER - }, + params={"type_drivers": driver}, + mocks={read: config}, expected_code=base.SUCCESSFUL_CODE, - expected_response=environment_configs. - ENV_CONFIGS_WITH_SPECIFIC_TYPE_DRIVER_RESPONSE + expected_response=config_response ) @patch(base.RESPONDER_BASE_READ) def test_get_environment_configs_list_with_user(self, read): + config = environment_configs.ENV_CONFIGS_WITH_SPECIFIC_USER + config_response = \ + environment_configs.ENV_CONFIGS_WITH_SPECIFIC_USER_RESPONSE self.validate_get_request(environment_configs.URL, - params={ - "user": environment_configs.USER - }, - mocks={ - read: environment_configs. - ENV_CONFIGS_WITH_SPECIFIC_USER - }, + params={"user": environment_configs.USER}, + mocks={read: config}, expected_code=base.SUCCESSFUL_CODE, - expected_response=environment_configs. - ENV_CONFIGS_WITH_SPECIFIC_USER_RESPONSE - ) + expected_response=config_response) def test_get_environment_configs_list_with_non_bool_listen(self): self.validate_get_request(environment_configs.URL, - params={ - "listen": environment_configs.NON_BOOL_LISTEN - }, + params={"listen": environment_configs. + NON_BOOL_LISTEN}, expected_code=base.BAD_REQUEST_CODE) @patch(base.RESPONDER_BASE_READ) def test_get_environment_configs_list_with_bool_listen(self, read): + config = environment_configs.ENV_CONFIGS_WITH_SPECIFIC_LISTEN + config_response = \ + environment_configs.ENV_CONFIGS_WITH_SPECIFIC_LISTEN_RESPONSE self.validate_get_request(environment_configs.URL, - params={ - "listen": environment_configs.BOOL_LISTEN - }, - mocks={ - read: environment_configs. - ENV_CONFIGS_WITH_SPECIFIC_LISTEN - }, + params={"listen": environment_configs. + BOOL_LISTEN}, + mocks={read: config}, expected_code=base.SUCCESSFUL_CODE, - expected_response=environment_configs. - ENV_CONFIGS_WITH_SPECIFIC_LISTEN_RESPONSE - ) + expected_response=config_response) def test_get_environment_configs_list_with_non_bool_scanned(self): self.validate_get_request(environment_configs.URL, - params={ - "scanned": environment_configs. - NON_BOOL_SCANNED - }, + params={"scanned": environment_configs. + NON_BOOL_SCANNED}, expected_code=base.BAD_REQUEST_CODE) @patch(base.RESPONDER_BASE_READ) def test_get_environment_configs_list_with_bool_scanned(self, read): + config = environment_configs.ENV_CONFIGS_WITH_SPECIFIC_SCANNED + config_response = \ + environment_configs.ENV_CONFIGS_WITH_SPECIFIC_SCANNED_RESPONSE self.validate_get_request(environment_configs.URL, - params={ - "scanned": environment_configs.BOOL_SCANNED - }, - mocks={ - read: environment_configs. - ENV_CONFIGS_WITH_SPECIFIC_SCANNED - }, + params={"scanned": environment_configs. + BOOL_SCANNED}, + mocks={read: config}, expected_code=base.SUCCESSFUL_CODE, - expected_response=environment_configs. - ENV_CONFIGS_WITH_SPECIFIC_SCANNED_RESPONSE + expected_response=config_response ) - def test_get_environment_configs_list_with_non_bool_monitoring_setup_done(self): + def test_get_env_configs_list_with_non_bool_monitoring_setup_done(self): self.validate_get_request(environment_configs.URL, - params={ - "listen": environment_configs. - NON_BOOL_MONITORING_SETUP_DONE - }, + params={"listen": environment_configs. + NON_BOOL_MONITORING_SETUP_DONE}, expected_code=base.BAD_REQUEST_CODE) @patch(base.RESPONDER_BASE_READ) - def test_get_environment_configs_list_with_bool_monitoring_setup_done(self, read): + def test_get_environment_configs_list_with_bool_monitoring_setup_done(self, + read): + config = environment_configs.\ + ENV_CONFIGS_WITH_SPECIFIC_MONITORING_SETUP_DONE + config_response = environment_configs.\ + ENV_CONFIGS_WITH_SPECIFIC_MONITORING_SETUP_DONE_RESPONSE self.validate_get_request(environment_configs.URL, - params={ - "scanned": environment_configs. - BOOL_MONITORING_SETUP_DONE - }, - mocks={ - read: environment_configs. - ENV_CONFIGS_WITH_SPECIFIC_MONITORING_SETUP_DONE - }, + params={"scanned": environment_configs. + BOOL_MONITORING_SETUP_DONE}, + mocks={read: config}, expected_code=base.SUCCESSFUL_CODE, - expected_response=environment_configs. - ENV_CONFIGS_WITH_SPECIFIC_MONITORING_SETUP_DONE_RESPONSE - ) + expected_response=config_response) def test_get_environment_configs_list_with_non_int_page(self): self.validate_get_request(environment_configs.URL, - params={ - "page": base.NON_INT_PAGE - }, + params={"page": base.NON_INT_PAGE}, expected_code=base.BAD_REQUEST_CODE) @patch(base.RESPONDER_BASE_READ) def test_get_environment_configs_list_with_int_page(self, read): + config_response = environment_configs.ENV_CONFIGS_RESPONSE self.validate_get_request(environment_configs.URL, - params={ - "page": base.INT_PAGE - }, - mocks={ - read: environment_configs.ENV_CONFIGS - }, + params={"page": base.INT_PAGE}, + mocks={read: environment_configs.ENV_CONFIGS}, expected_code=base.SUCCESSFUL_CODE, - expected_response=environment_configs. - ENV_CONFIGS_RESPONSE - ) + expected_response=config_response) def test_get_environment_configs_list_with_non_int_page_size(self): self.validate_get_request(environment_configs.URL, - params={ - "page_size": base.NON_INT_PAGESIZE - }, + params={"page_size": base.NON_INT_PAGESIZE}, expected_code=base.BAD_REQUEST_CODE) @patch(base.RESPONDER_BASE_READ) def test_get_environment_configs_list_with_int_page_size(self, read): + config_response = environment_configs.ENV_CONFIGS_RESPONSE self.validate_get_request(environment_configs.URL, - params={ - "page_size": base.INT_PAGESIZE - }, - mocks={ - read: environment_configs.ENV_CONFIGS - }, + params={"page_size": base.INT_PAGESIZE}, + mocks={read: environment_configs.ENV_CONFIGS}, expected_code=base.SUCCESSFUL_CODE, - expected_response=environment_configs. - ENV_CONFIGS_RESPONSE - ) + expected_response=config_response) def test_post_environment_config_without_app_path(self): test_data = self.get_updated_data(environment_configs.ENV_CONFIG, @@ -292,8 +242,9 @@ class TestEnvironmentConfigs(TestBase): expected_code=base.BAD_REQUEST_CODE) def test_post_environment_config_with_wrong_distribution(self): + dist = environment_configs.WRONG_DISTRIBUTION test_data = self.get_updated_data(environment_configs.ENV_CONFIG, - updates={"distribution": environment_configs.WRONG_DISTRIBUTION}) + updates={"distribution": dist}) self.validate_post_request(environment_configs.URL, body=json.dumps(test_data), expected_code=base.BAD_REQUEST_CODE) @@ -306,8 +257,9 @@ class TestEnvironmentConfigs(TestBase): expected_code=base.BAD_REQUEST_CODE) def test_post_environment_config_with_wrong_listen(self): + listen_val = environment_configs.NON_BOOL_LISTEN test_data = self.get_updated_data(environment_configs.ENV_CONFIG, - updates={"listen": environment_configs.NON_BOOL_LISTEN}) + updates={"listen": listen_val}) self.validate_post_request(environment_configs.URL, body=json.dumps(test_data), expected_code=base.BAD_REQUEST_CODE) @@ -320,10 +272,10 @@ class TestEnvironmentConfigs(TestBase): expected_code=base.BAD_REQUEST_CODE) def test_post_environment_config_with_wrong_mechanism_driver(self): + mechanism = environment_configs.WRONG_MECHANISM_DRIVER test_data = self.get_updated_data(environment_configs.ENV_CONFIG, updates={ - "mechanism_drivers": - [environment_configs.WRONG_MECHANISM_DRIVER] + "mechanism_drivers": [mechanism] }) self.validate_post_request(environment_configs.URL, body=json.dumps(test_data), @@ -344,19 +296,17 @@ class TestEnvironmentConfigs(TestBase): expected_code=base.BAD_REQUEST_CODE) def test_post_environment_config_with_wrong_scanned(self): + scanned_val = environment_configs.NON_BOOL_SCANNED test_data = self.get_updated_data(environment_configs.ENV_CONFIG, - updates={ - "scanned": environment_configs.NON_BOOL_SCANNED - }) + updates={"scanned": scanned_val}) self.validate_post_request(environment_configs.URL, body=json.dumps(test_data), expected_code=base.BAD_REQUEST_CODE) def test_post_environment_config_with_wrong_last_scanned(self): + scanned_val = base.WRONG_FORMAT_TIME test_data = self.get_updated_data(environment_configs.ENV_CONFIG, - updates={ - "last_scanned": base.WRONG_FORMAT_TIME - }) + updates={"last_scanned": scanned_val}) self.validate_post_request(environment_configs.URL, body=json.dumps(test_data), expected_code=base.BAD_REQUEST_CODE) @@ -376,16 +326,81 @@ class TestEnvironmentConfigs(TestBase): expected_code=base.BAD_REQUEST_CODE) def test_post_environment_config_with_wrong_type_drivers(self): + driver = environment_configs.WRONG_TYPE_DRIVER + test_data = self.get_updated_data(environment_configs.ENV_CONFIG, + updates={"type_drivers": [driver]}) + self.validate_post_request(environment_configs.URL, + body=json.dumps(test_data), + expected_code=base.BAD_REQUEST_CODE) + + def test_post_environment_config_with_duplicate_configurations(self): + test_data = self.get_updated_data(environment_configs.ENV_CONFIG) + test_data["configuration"].append({ + "name": "OpenStack" + }) + self.validate_post_request(environment_configs.URL, + body=json.dumps(test_data), + expected_code=base.BAD_REQUEST_CODE) + + def test_post_environment_config_with_empty_configuration(self): + test_data = self.get_updated_data(environment_configs.ENV_CONFIG) + test_data["configuration"].append({}) + self.validate_post_request(environment_configs.URL, + body=json.dumps(test_data), + expected_code=base.BAD_REQUEST_CODE) + + def test_post_environment_config_with_unknown_configuration(self): + test_data = self.get_updated_data(environment_configs.ENV_CONFIG) + test_data["configuration"].append({ + "name": "Unknown configuration", + }) + self.validate_post_request(environment_configs.URL, + body=json.dumps(test_data), + expected_code=base.BAD_REQUEST_CODE) + + def test_post_environment_config_without_required_configurations(self): + for env_type in CONSTANTS_BY_NAMES["environment_types"]: + required_conf_list = ( + EnvironmentConfigs.REQUIRED_CONFIGURATIONS_NAMES.get(env_type, + []) + ) + if required_conf_list: + test_data = \ + self.get_updated_data(environment_configs.ENV_CONFIG) + test_data['environment_type'] = env_type + test_data['configuration'] = [ + c + for c in test_data['configuration'] + if c['name'] != required_conf_list[0] + ] + + self.validate_post_request(environment_configs.URL, + body=json.dumps(test_data), + expected_code=base.BAD_REQUEST_CODE) + + def test_post_environment_config_with_incomplete_configuration(self): test_data = self.get_updated_data(environment_configs.ENV_CONFIG, updates={ - "type_drivers": [environment_configs.WRONG_TYPE_DRIVER] + "configuration": [{ + "host": "10.56.20.239", + "name": "mysql", + "user": "root" + }, { + "name": "OpenStack", + "host": "10.56.20.239", + }, { + "host": "10.56.20.239", + "name": "CLI", + "user": "root" + }] }) self.validate_post_request(environment_configs.URL, body=json.dumps(test_data), expected_code=base.BAD_REQUEST_CODE) - def mock_validate_env_config_with_supported_envs(self, scanning, - monitoring, listening): + @staticmethod + def mock_validate_env_config_with_supported_envs(scanning, monitoring, + listening): InventoryMgr.is_feature_supported_in_env = \ lambda self, matches, feature: { EnvironmentFeatures.SCANNING: scanning, @@ -396,11 +411,12 @@ class TestEnvironmentConfigs(TestBase): @patch(base.RESPONDER_BASE_WRITE) def test_post_environment_config(self, write): self.mock_validate_env_config_with_supported_envs(True, True, True) + post_body = json.dumps(environment_configs.ENV_CONFIG) self.validate_post_request(environment_configs.URL, mocks={ write: None }, - body=json.dumps(environment_configs.ENV_CONFIG), + body=post_body, expected_code=base.CREATED_CODE) def test_post_unsupported_environment_config(self): @@ -421,10 +437,11 @@ class TestEnvironmentConfigs(TestBase): "listening": False } ] + mock_validate = self.mock_validate_env_config_with_supported_envs + config = environment_configs.ENV_CONFIG for test_case in test_cases: - self.mock_validate_env_config_with_supported_envs(test_case["scanning"], - test_case["monitoring"], - test_case["listening"]) + mock_validate(test_case["scanning"], test_case["monitoring"], + test_case["listening"]) self.validate_post_request(environment_configs.URL, - body=json.dumps(environment_configs.ENV_CONFIG), + body=json.dumps(config), expected_code=base.BAD_REQUEST_CODE) diff --git a/app/test/api/responders_test/test_data/base.py b/app/test/api/responders_test/test_data/base.py index b99d5bb..6d2422a 100644 --- a/app/test/api/responders_test/test_data/base.py +++ b/app/test/api/responders_test/test_data/base.py @@ -16,14 +16,14 @@ UNAUTHORIZED_CODE = "401" CREATED_CODE = "201" ENV_NAME = "Mirantis-Liberty-API" -UNKNOWN_ENV = "Unkown-Environment" +UNKNOWN_ENV = "Unknown-Environment" NON_INT_PAGE = 1.4 INT_PAGE = 1 NON_INT_PAGESIZE = 2.4 INT_PAGESIZE = 2 WRONG_LINK_TYPE = "instance-host" -CORRECT_LINK_TYPE= "instance-vnic" +CORRECT_LINK_TYPE = "instance-vnic" WRONG_LINK_STATE = "wrong" CORRECT_LINK_STATE = "up" @@ -41,7 +41,7 @@ WRONG_TYPE_DRIVER = "wrong_type" CORRECT_TYPE_DRIVER = "local" WRONG_MECHANISM_DRIVER = "wrong-mechanism-dirver" -CORRECT_MECHANISM_DRIVER = "ovs" +CORRECT_MECHANISM_DRIVER = "OVS" WRONG_LOG_LEVEL = "wrong-log-level" CORRECT_LOG_LEVEL = "critical" @@ -71,16 +71,32 @@ NON_DICT_OBJ = "" CONSTANTS_BY_NAMES = { "link_types": [ "instance-vnic", - "otep-vconnector", - "otep-host_pnic", + "vnic-instance", + "vnic-vconnector", + "vconnector-vnic", + "vconnector-vedge", + "vedge-vconnector", + "vedge-host_pnic", + "host_pnic-vedge", "host_pnic-network", + "network-host_pnic", "vedge-otep", - "vnic-vconnector", + "otep-vedge", + "otep-vconnector", + "vconnector-otep", + "otep-host_pnic", + "host_pnic-otep", "vconnector-host_pnic", - "vconnector-vedge", + "host_pnic-vconnector", "vnic-vedge", - "vedge-host_pnic", - "vservice-vnic" + "vedge-vnic", + "vservice-vnic", + "vnic-vservice", + "switch_pnic-host_pnic", + "host_pnic-switch_pnic", + "switch_pnic-switch_pnic", + "switch_pnic-switch", + "switch-switch_pnic" ], "link_states": [ "up", @@ -117,9 +133,9 @@ CONSTANTS_BY_NAMES = { "flat" ], "mechanism_drivers": [ - "ovs", - "vpp", - "LinuxBridge", + "OVS", + "VPP", + "LXB", "Arista", "Nexus" ], @@ -155,6 +171,10 @@ CONSTANTS_BY_NAMES = { "Mirantis", "RDO" ], + "distribution_versions": [ + "8.0", + "9.0" + ], "environment_operational_status": [ "stopped", "running", @@ -168,6 +188,30 @@ CONSTANTS_BY_NAMES = { ], "environment_monitoring_types": [ "Sensu" + ], + "scans_statuses": [ + "draft", + "pending", + "running", + "completed", + "completed_with_errors", + "failed", + "aborted" + ], + "configuration_targets": [ + "AMQP", + "CLI", + "ACI", + "mysql", + "OpenStack", + "Monitoring", + "Kubernetes" + ], + "environment_types": [ + "OpenStack", + "Kubernetes", + "VMware", + "Bare-metal" ] } @@ -175,7 +219,8 @@ CONSTANTS_BY_NAMES = { RESPONDER_BASE_PATH = "api.responders.responder_base.ResponderBase" RESPONDER_BASE_GET_OBJECTS_LIST = RESPONDER_BASE_PATH + ".get_objects_list" RESPONDER_BASE_GET_OBJECT_BY_ID = RESPONDER_BASE_PATH + ".get_object_by_id" -RESPONDER_BASE_CHECK_ENVIRONMENT_NAME = RESPONDER_BASE_PATH + ".check_environment_name" +RESPONDER_BASE_CHECK_ENVIRONMENT_NAME = \ + RESPONDER_BASE_PATH + ".check_environment_name" RESPONDER_BASE_READ = RESPONDER_BASE_PATH + ".read" RESPONDER_BASE_WRITE = RESPONDER_BASE_PATH + ".write" RESPONDER_BASE_AGGREGATE = RESPONDER_BASE_PATH + ".aggregate" diff --git a/app/test/api/responders_test/test_data/clique_types.py b/app/test/api/responders_test/test_data/clique_types.py index ae962ce..0791bdf 100644 --- a/app/test/api/responders_test/test_data/clique_types.py +++ b/app/test/api/responders_test/test_data/clique_types.py @@ -8,13 +8,18 @@ # http://www.apache.org/licenses/LICENSE-2.0 # ############################################################################### from test.api.responders_test.test_data import base - +from test.api.responders_test.test_data.base import WRONG_MECHANISM_DRIVER, \ + CORRECT_MECHANISM_DRIVER, CORRECT_TYPE_DRIVER, WRONG_TYPE_DRIVER, \ + CORRECT_DISTRIBUTION, CORRECT_DIST_VER URL = "/clique_types" WRONG_ID = base.WRONG_OBJECT_ID NONEXISTENT_ID = "58ca73ae3a8a836d10ff3b44" CORRECT_ID = base.CORRECT_OBJECT_ID +SAMPLE_IDS = ['58ca73ae3a8a836d10ff3b80', '58ca73ae3a8a836d10ff3b81'] + +RESERVED_ENV_NAME = 'ANY' WRONG_FOCAL_POINT_TYPE = base.WRONG_OBJECT_TYPE CORRECT_FOCAL_POINT_POINT_TYPE = base.CORRECT_OBJECT_TYPE @@ -23,25 +28,52 @@ WRONG_LINK_TYPE = base.WRONG_LINK_TYPE NONEXISTENT_LINK_TYPE = "otep-host_pnic" CORRECT_LINK_TYPE = base.CORRECT_LINK_TYPE +CLIQUE_TYPE = { + "environment": "Mirantis-Liberty-API", + "name": "instance_vconnector_clique", + "link_types": [ + "instance-vnic", + "vnic-vconnector" + ], + "focal_point_type": "instance" +} + +TEST_CONFIGURATION = { + "distribution": CORRECT_DISTRIBUTION, + "distribution_version": CORRECT_DIST_VER, + "mechanism_drivers": CORRECT_MECHANISM_DRIVER, + "type_drivers": CORRECT_TYPE_DRIVER +} + + +def get_payload(update: dict = None, delete: list = None): + payload = CLIQUE_TYPE.copy() + if update: + payload.update(update) + if delete: + for k in delete: + del payload[k] + return payload + + CLIQUE_TYPES_WITH_SPECIFIC_ID = [ - { - "environment": "Mirantis-Liberty-API", - "focal_point_type": "host_pnic", - "id": CORRECT_ID - } + get_payload(update={'id': CORRECT_ID}) +] + +CLIQUE_TYPES_WITH_SPECIFIC_CONFIGURATION = [ + get_payload(update={'id': SAMPLE_IDS[0], + **TEST_CONFIGURATION}, + delete=['environment']) ] +CLIQUE_TYPES_WITH_SPECIFIC_CONFIGURATION_RESPONSE = { + "clique_types": CLIQUE_TYPES_WITH_SPECIFIC_CONFIGURATION +} + CLIQUE_TYPES_WITH_SPECIFIC_FOCAL_POINT_TYPE = [ - { - "environment": "Mirantis-Liberty-API", - "focal_point_type": CORRECT_FOCAL_POINT_POINT_TYPE, - "id": "58ca73ae3a8a836d10ff3b80" - }, - { - "environment": "Mirantis-Liberty-API", - "focal_point_type": CORRECT_FOCAL_POINT_POINT_TYPE, - "id": "58ca73ae3a8a836d10ff3b81" - } + get_payload(update={'id': _id, + 'focal_point_type': CORRECT_FOCAL_POINT_POINT_TYPE}) + for _id in SAMPLE_IDS ] CLIQUE_TYPES_WITH_SPECIFIC_FOCAL_POINT_TYPE_RESPONSE = { @@ -49,20 +81,9 @@ CLIQUE_TYPES_WITH_SPECIFIC_FOCAL_POINT_TYPE_RESPONSE = { } CLIQUE_TYPES_WITH_SPECIFIC_LINK_TYPE = [ - { - "environment": "Mirantis-Liberty-API", - "link_types": [ - CORRECT_LINK_TYPE - ], - "id": "58ca73ae3a8a836d10ff3b80" - }, - { - "environment": "Mirantis-Liberty-API", - "link_types": [ - CORRECT_LINK_TYPE - ], - "id": "58ca73ae3a8a836d10ff3b81" - } + get_payload(update={'id': _id, + 'link_types': [CORRECT_LINK_TYPE]}) + for _id in SAMPLE_IDS ] CLIQUE_TYPES_WITH_SPECIFIC_LINK_TYPE_RESPONSE = { @@ -70,16 +91,7 @@ CLIQUE_TYPES_WITH_SPECIFIC_LINK_TYPE_RESPONSE = { } CLIQUE_TYPES = [ - { - "environment": "Mirantis-Liberty-API", - "focal_point_type": "vnic", - "id": "58ca73ae3a8a836d10ff3b80" - }, - { - "environment": "Mirantis-Liberty-API", - "focal_point_type": "vnic", - "id": "58ca73ae3a8a836d10ff3b81" - } + get_payload(update={'id': _id}) for _id in SAMPLE_IDS ] CLIQUE_TYPES_RESPONSE = { @@ -88,83 +100,48 @@ CLIQUE_TYPES_RESPONSE = { NON_DICT_CLIQUE_TYPE = base.NON_DICT_OBJ -CLIQUE_TYPE_WITHOUT_ENVIRONMENT = { - "name": "instance_vconnector_clique", - "link_types": [ - "instance-vnic", - "vnic-vconnector" - ], - "focal_point_type": "instance" -} +CLIQUE_TYPE_WITH_RESERVED_NAME = get_payload( + update={'environment': RESERVED_ENV_NAME} +) -CLIQUE_TYPE_WITH_UNKNOWN_ENVIRONMENT = { - "environment": base.UNKNOWN_ENV, - "id": "589a3969761b0555a3ef6093", - "name": "instance_vconnector_clique", - "link_types": [ - "instance-vnic", - "vnic-vconnector" - ], - "focal_point_type": "instance" -} +CLIQUE_TYPE_WITHOUT_ENV_NAME_AND_CONF = get_payload( + delete=['environment'] +) -CLIQUE_TYPE_WITHOUT_FOCAL_POINT_TYPE = { - "environment": "Mirantis-Liberty-API", - "name": "instance_vconnector_clique", - "link_types": [ - "instance-vnic", - "vnic-vconnector" - ] -} +CLIQUE_TYPE_WITH_BOTH_ENV_AND_CONF = get_payload( + update=TEST_CONFIGURATION +) -CLIQUE_TYPE_WITH_WRONG_FOCAL_POINT_TYPE = { - "environment": "Mirantis-Liberty-API", - "name": "instance_vconnector_clique", - "link_types": [ - "instance-vnic", - "vnic-vconnector" - ], - "focal_point_type": WRONG_FOCAL_POINT_TYPE -} +CLIQUE_TYPE_WITH_INSUFFICIENT_CONF = get_payload( + update={'distribution_version': CORRECT_DIST_VER} +) -CLIQUE_TYPE_WITHOUT_LINK_TYPES = { - "environment": "Mirantis-Liberty-API", - "name": "instance_vconnector_clique", - "focal_point_type": "instance" -} +CLIQUE_TYPE_WITH_UNKNOWN_ENVIRONMENT = get_payload( + update={'environment': base.UNKNOWN_ENV} +) -CLIQUE_TYPE_WITH_NON_LIST_LINK_TYPES = { - "environment": "Mirantis-Liberty-API", - "name": "instance_vconnector_clique", - "link_types": "instance-vnic", - "focal_point_type": "instance" -} +CLIQUE_TYPE_WITHOUT_FOCAL_POINT_TYPE = get_payload(delete=['focal_point_type']) -CLIQUE_TYPE_WITH_WRONG_LINK_TYPE = { - "environment": "Mirantis-Liberty-API", - "name": "instance_vconnector_clique", - "link_types": [ - WRONG_LINK_TYPE, - "vnic-vconnector" - ], - "focal_point_type": "instance" -} +CLIQUE_TYPE_WITH_WRONG_FOCAL_POINT_TYPE = get_payload( + update={'focal_point_type': WRONG_FOCAL_POINT_TYPE} +) -CLIQUE_TYPE_WITHOUT_NAME = { - "environment": "Mirantis-Liberty-API", - "link_types": [ - "instance-vnic", - "vnic-vconnector", - ], - "focal_point_type": "instance" -} +CLIQUE_TYPE_WITHOUT_LINK_TYPES = get_payload(delete=['link_types']) -CLIQUE_TYPE = { - "environment": "Mirantis-Liberty-API", - "name": "instance_vconnector_clique", - "link_types": [ - "instance-vnic", - "vnic-vconnector" - ], - "focal_point_type": "instance" -} +CLIQUE_TYPE_WITH_NON_LIST_LINK_TYPES = get_payload( + update={'link_types': "instance-vnic"} +) + +CLIQUE_TYPE_WITH_WRONG_LINK_TYPE = get_payload( + update={'link_types': [WRONG_LINK_TYPE, "vnic-vconnector"]} +) + +CLIQUE_TYPE_WITHOUT_NAME = get_payload(delete=['name']) + +CLIQUE_TYPE_WITH_WRONG_MECH_DRIVERS = get_payload( + update={'mechanism_drivers': WRONG_MECHANISM_DRIVER} +) + +CLIQUE_TYPE_WITH_WRONG_TYPE_DRIVERS = get_payload( + update={'type_drivers': WRONG_TYPE_DRIVER} +) \ No newline at end of file diff --git a/app/test/api/responders_test/test_data/environment_configs.py b/app/test/api/responders_test/test_data/environment_configs.py index 4cea105..3e976ec 100644 --- a/app/test/api/responders_test/test_data/environment_configs.py +++ b/app/test/api/responders_test/test_data/environment_configs.py @@ -201,6 +201,7 @@ ENV_CONFIG = { "provision": "None", "env_type": "development", "name": "Monitoring", + "install_monitoring_client": True, "api_port": "4567", "rabbitmq_port": "5671", "rabbitmq_pass": "sensuaccess", @@ -218,12 +219,13 @@ ENV_CONFIG = { "last_scanned": "2017-03-16T11:14:54Z", "listen": True, "mechanism_drivers": [ - "ovs" + "OVS" ], "name": "Mirantis-Liberty", "operational": "running", "scanned": True, "type": "environment", "type_drivers": "vxlan", - "user": "WS7j8oTbWPf3LbNne" + "user": "WS7j8oTbWPf3LbNne", + "environment_type": "OpenStack" } diff --git a/app/test/api/test_base.py b/app/test/api/test_base.py index 33185ec..edc59ae 100644 --- a/app/test/api/test_base.py +++ b/app/test/api/test_base.py @@ -34,8 +34,10 @@ class TestBase(TestCase): self.original_auth_method = AuthenticationMiddleware.process_request AuthenticationMiddleware.process_request = mock_auth_method - ResponderBase.get_constants_by_name = MagicMock(side_effect= - lambda name: base.CONSTANTS_BY_NAMES[name]) + ResponderBase.get_constants_by_name = MagicMock( + side_effect=lambda name: base.CONSTANTS_BY_NAMES[name] + ) + # mock mongo access MongoAccess.mongo_connect = MagicMock() MongoAccess.db = MagicMock() @@ -47,8 +49,8 @@ class TestBase(TestCase): log_level = 'debug' self.app = App(log_level=log_level).get_app() - def validate_get_request(self, url, params={}, headers=None, mocks={}, - side_effects={}, + def validate_get_request(self, url, params=None, headers=None, mocks=None, + side_effects=None, expected_code=base.SUCCESSFUL_CODE, expected_response=None): self.validate_request("GET", url, params, headers, "", @@ -59,25 +61,27 @@ class TestBase(TestCase): def validate_request(self, action, url, params, headers, body, mocks, side_effects, expected_code, expected_response): - for mock_method, mock_data in mocks.items(): - mock_method.return_value = mock_data + if mocks: + for mock_method, mock_data in mocks.items(): + mock_method.return_value = mock_data - for mock_method, side_effect in side_effects.items(): - mock_method.side_effect = side_effect + if side_effects: + for mock_method, side_effect in side_effects.items(): + mock_method.side_effect = side_effect result = self.simulate_request(action, url, params=params, headers=headers, body=body) self.assertEqual(result.status, expected_code) if expected_response: self.assertEqual(result.json, expected_response) - def validate_post_request(self, url, headers={}, body="", mocks={}, - side_effects={}, + def validate_post_request(self, url, headers=None, body="", mocks=None, + side_effects=None, expected_code=base.CREATED_CODE, expected_response=None): self.validate_request("POST", url, {}, headers, body, mocks, side_effects, expected_code, expected_response) - def validate_delete_request(self, url, params={}, headers={}, mocks={}, - side_effects={}, + def validate_delete_request(self, url, params=None, headers=None, mocks=None, + side_effects=None, expected_code=base.SUCCESSFUL_CODE, expected_response=None): self.validate_request("DELETE", url, params, headers, "", mocks, side_effects, diff --git a/app/test/fetch/api_fetch/test_api_access.py b/app/test/fetch/api_fetch/test_api_access.py index 0effc0e..440b730 100644 --- a/app/test/fetch/api_fetch/test_api_access.py +++ b/app/test/fetch/api_fetch/test_api_access.py @@ -7,9 +7,9 @@ # which accompanies this distribution, and is available at # # http://www.apache.org/licenses/LICENSE-2.0 # ############################################################################### -from unittest.mock import MagicMock, Mock - +import copy import requests +from unittest.mock import MagicMock, Mock from discover.fetchers.api.api_access import ApiAccess from test.fetch.api_fetch.test_data.api_access import * @@ -35,38 +35,45 @@ class TestApiAccess(TestFetch): def test_parse_illegal_time(self): time = self.api_access.parse_time(ILLEGAL_TIME) - self.assertEqual(time, None, "Can't get None when the time format is wrong") + self.assertEqual(time, None, + "Can't get None when the time format is wrong") def test_get_existing_token(self): self.api_access.tokens = VALID_TOKENS token = self.api_access.get_existing_token(PROJECT) - self.assertNotEqual(token, VALID_TOKENS[PROJECT], "Can't get existing token") + self.assertNotEqual(token, VALID_TOKENS[PROJECT], + "Can't get existing token") def test_get_nonexistent_token(self): self.api_access.tokens = EMPTY_TOKENS token = self.api_access.get_existing_token(TEST_PROJECT) - self.assertEqual(token, None, "Can't get None when the token doesn't " + - "exist in tokens") + self.assertEqual(token, None, + "Can't get None when the token doesn't exist " + "in tokens") def test_v2_auth(self): self.api_access.get_existing_token = MagicMock(return_value=None) self.response.json = Mock(return_value=CORRECT_AUTH_CONTENT) # mock authentication info from OpenStack Api - token_details = self.api_access.v2_auth(TEST_PROJECT, TEST_HEADER, TEST_BODY) + token_details = self.api_access.v2_auth(TEST_PROJECT, TEST_HEADER, + TEST_BODY) self.assertNotEqual(token_details, None, "Can't get the token details") def test_v2_auth_with_error_content(self): self.api_access.get_existing_token = MagicMock(return_value=None) self.response.json = Mock(return_value=ERROR_AUTH_CONTENT) # authentication content from OpenStack Api will be incorrect - token_details = self.api_access.v2_auth(TEST_PROJECT, TEST_HEADER, TEST_BODY) - self.assertIs(token_details, None, "Can't get None when the content is wrong") + token_details = self.api_access.v2_auth(TEST_PROJECT, TEST_HEADER, + TEST_BODY) + self.assertIs(token_details, None, + "Can't get None when the content is wrong") def test_v2_auth_with_error_token(self): self.response.status_code = requests.codes.bad_request self.response.json = Mock(return_value=ERROR_TOKEN_CONTENT) # authentication info from OpenStack Api will not contain token info - token_details = self.api_access.v2_auth(TEST_PROJECT, TEST_HEADER, TEST_BODY) + token_details = self.api_access.v2_auth(TEST_PROJECT, TEST_HEADER, + TEST_BODY) self.assertIs(token_details, None, "Can't get None when the content " + "doesn't contain any token info") @@ -78,12 +85,13 @@ class TestApiAccess(TestFetch): # the time will not be parsed self.api_access.parse_time = MagicMock(return_value=None) - token_details = self.api_access.v2_auth(TEST_PROJECT, TEST_HEADER, TEST_BODY) + token_details = self.api_access.v2_auth(TEST_PROJECT, TEST_HEADER, + TEST_BODY) # reset original parse_time method self.api_access.parse_time = original_method - self.assertIs(token_details, None, "Can't get None when the time in token " + - "can't be parsed") + self.assertIs(token_details, None, + "Can't get None when the time in token can't be parsed") def test_v2_auth_pwd(self): self.response.json = Mock(return_value=CORRECT_AUTH_CONTENT) @@ -92,20 +100,30 @@ class TestApiAccess(TestFetch): self.assertNotEqual(token, None, "Can't get token") def test_get_url(self): - self.response.json = Mock(return_value=GET_CONTENT) + get_response = copy.deepcopy(self.response) + get_response.status_code = requests.codes.ok + self.requests_get = requests.get + requests.get = MagicMock(return_value=get_response) + get_response.json = Mock(return_value=GET_CONTENT) result = self.api_access.get_url(TEST_URL, TEST_HEADER) # check whether it returns content message when the response is correct self.assertNotEqual(result, None, "Can't get content when the " "response is correct") + requests.get = self.requests_get def test_get_url_with_error_response(self): - self.response.status_code = requests.codes.bad_request - self.response.json = Mock(return_value=None) - self.response.text = "Bad request" + get_response = copy.deepcopy(self.response) + get_response.status_code = requests.codes.bad_request + get_response.text = "Bad request" + get_response.json = Mock(return_value=GET_CONTENT) + self.requests_get = requests.get + requests.get = MagicMock(return_value=get_response) + # the response will be wrong result = self.api_access.get_url(TEST_URL, TEST_HEADER) self.assertEqual(result, None, "Result returned" + "when the response status is not 200") + requests.get = self.requests_get def test_get_region_url(self): region_url = self.api_access.get_region_url(REGION_NAME, SERVICE_NAME) @@ -120,23 +138,30 @@ class TestApiAccess(TestFetch): def test_get_region_url_without_service_endpoint(self): # error service doesn't exist in region service endpoints - region_url = self.api_access.get_region_url(REGION_NAME, ERROR_SERVICE_NAME) - self.assertIs(region_url, None, "Can't get None with wrong service name") + region_url = self.api_access.get_region_url(REGION_NAME, + ERROR_SERVICE_NAME) + self.assertIs(region_url, None, + "Can't get None with wrong service name") def test_region_url_nover(self): - # mock return value of get_region_url, which has something starting from v2 + # mock return value of get_region_url, + # which has something starting from v2 self.api_access.get_region_url = MagicMock(return_value=REGION_URL) - region_url = self.api_access.get_region_url_nover(REGION_NAME, SERVICE_NAME) + region_url = self.api_access.get_region_url_nover(REGION_NAME, + SERVICE_NAME) # get_region_nover will remove everything from v2 - self.assertNotIn("v2", region_url, "Can't get region url without v2 info") + self.assertNotIn("v2", region_url, + "Can't get region url without v2 info") def test_get_service_region_endpoints(self): region = REGIONS[REGION_NAME] - result = self.api_access.get_service_region_endpoints(region, SERVICE_NAME) + result = self.api_access.get_service_region_endpoints(region, + SERVICE_NAME) self.assertNotEqual(result, None, "Can't get service endpoint") def test_get_service_region_endpoints_with_nonexistent_service(self): region = REGIONS[REGION_NAME] - result = self.api_access.get_service_region_endpoints(region, ERROR_SERVICE_NAME) + get_endpoints = self.api_access.get_service_region_endpoints + result = get_endpoints(region, ERROR_SERVICE_NAME) self.assertIs(result, None, "Can't get None when the service name " + "doesn't exist in region's services") diff --git a/app/test/fetch/api_fetch/test_api_fetch_project_hosts.py b/app/test/fetch/api_fetch/test_api_fetch_project_hosts.py index da3df17..784079e 100644 --- a/app/test/fetch/api_fetch/test_api_fetch_project_hosts.py +++ b/app/test/fetch/api_fetch/test_api_fetch_project_hosts.py @@ -7,6 +7,7 @@ # which accompanies this distribution, and is available at # # http://www.apache.org/licenses/LICENSE-2.0 # ############################################################################### +import copy from unittest.mock import MagicMock from discover.fetchers.api.api_fetch_project_hosts import ApiFetchProjectHosts from test.fetch.test_fetch import TestFetch @@ -36,23 +37,28 @@ class TestApiFetchProjectHosts(TestFetch): "type in host_type") def test_add_host_type_with_existent_host_type(self): + fetch_host_os_details = self.fetcher.fetch_host_os_details + self.fetcher.fetch_host_os_details = MagicMock() # add nonexistent host type to host type HOST_DOC["host_type"] = [NONEXISTENT_TYPE] # try to add existing host type self.fetcher.add_host_type(HOST_DOC, NONEXISTENT_TYPE, HOST_ZONE) - self.assertEqual(len(HOST_DOC['host_type']), 1, "Add duplicate host type") + self.assertEqual(len(HOST_DOC['host_type']), 1, + "Add duplicate host type") + self.fetcher.fetch_host_os_details = fetch_host_os_details def test_add_compute_host_type(self): - HOST_DOC['host_type'] = [] + doc = copy.deepcopy(HOST_DOC) + doc['host_type'] = [] # clear zone - HOST_DOC['zone'] = None + doc['zone'] = None # add compute host type - self.fetcher.add_host_type(HOST_DOC, COMPUTE_TYPE, HOST_ZONE) + self.fetcher.add_host_type(doc, COMPUTE_TYPE, HOST_ZONE) # for compute host type, zone information will be added - self.assertEqual(HOST_DOC['zone'], HOST_ZONE, "Can't update zone " + - "name for compute node") - self.assertEqual(HOST_DOC['parent_id'], HOST_ZONE, "Can't update parent_id " + - "for compute node") + self.assertEqual(doc['zone'], HOST_ZONE, + "Can't update zone name for compute node") + self.assertEqual(doc['parent_id'], HOST_ZONE, + "Can't update parent_id for compute node") def test_fetch_compute_node_ip_address(self): # mock ip address information fetched from DB @@ -78,16 +84,24 @@ class TestApiFetchProjectHosts(TestFetch): def test_get_host_details(self): # test node have nova-conductor attribute, controller type will be added + fetch_host_os_details = self.fetcher.fetch_host_os_details + self.fetcher.fetch_host_os_details = MagicMock() result = self.fetcher.get_host_details(AVAILABILITY_ZONE, HOST_NAME) self.assertIn("Controller", result['host_type'], "Can't put controller type " + "in the compute node host_type") + self.fetcher.fetch_host_os_details = fetch_host_os_details def test_get_hosts_from_az(self): + fetch_host_os_details = self.fetcher.fetch_host_os_details + self.fetcher.fetch_host_os_details = MagicMock() result = self.fetcher.get_hosts_from_az(AVAILABILITY_ZONE) self.assertNotEqual(result, [], "Can't get hosts information from " "availability zone") + self.fetcher.fetch_host_os_details = fetch_host_os_details def test_get_for_region(self): + fetch_host_os_details = self.fetcher.fetch_host_os_details + self.fetcher.fetch_host_os_details = MagicMock() # mock region url for nova node self.fetcher.get_region_url = MagicMock(return_value=REGION_URL) # mock the response from OpenStack Api @@ -96,6 +110,7 @@ class TestApiFetchProjectHosts(TestFetch): result = self.fetcher.get_for_region(self.region, TOKEN) self.assertNotEqual(result, [], "Can't get hosts information for region") + self.fetcher.fetch_host_os_details = fetch_host_os_details def test_get_for_region_without_token(self): self.fetcher.get_region_url = MagicMock(return_value=REGION_URL) @@ -112,6 +127,8 @@ class TestApiFetchProjectHosts(TestFetch): self.assertEqual(result, [], "Can't get [] when the response is wrong") def test_get_for_region_with_error_hypervisors_response(self): + fetch_host_os_details = self.fetcher.fetch_host_os_details + self.fetcher.fetch_host_os_details = MagicMock() self.fetcher.get_region_url = MagicMock(return_value=REGION_URL) # mock error hypervisors response from OpenStack Api side_effect = [AVAILABILITY_ZONE_RESPONSE, HYPERVISORS_ERROR_RESPONSE] @@ -120,6 +137,7 @@ class TestApiFetchProjectHosts(TestFetch): result = self.fetcher.get_for_region(self.region, TOKEN) self.assertNotEqual(result, [], "Can't get hosts information when " + "the hypervisors response is wrong") + self.fetcher.fetch_host_os_details = fetch_host_os_details def test_get(self): original_method = self.fetcher.get_for_region @@ -140,6 +158,15 @@ class TestApiFetchProjectHosts(TestFetch): result = self.fetcher.get(PROJECT_NAME) self.assertEqual(result, [], "Can't get [] when the token is invalid") + def test_fetch_host_os_details(self): + original_method = self.fetcher.run + self.fetcher.run = MagicMock(return_value=OS_DETAILS_INPUT) + doc = {'host': 'host1'} + self.fetcher.fetch_host_os_details(doc) + self.assertEqual(doc.get('OS', {}), OS_DETAILS) + self.fetcher.run = original_method + + def tearDown(self): super().tearDown() ApiFetchProjectHosts.v2_auth_pwd = self._v2_auth_pwd diff --git a/app/test/fetch/api_fetch/test_data/api_fetch_host_project_hosts.py b/app/test/fetch/api_fetch/test_data/api_fetch_host_project_hosts.py index 3ef1ac7..ba42590 100644 --- a/app/test/fetch/api_fetch/test_data/api_fetch_host_project_hosts.py +++ b/app/test/fetch/api_fetch/test_data/api_fetch_host_project_hosts.py @@ -223,3 +223,24 @@ GET_FOR_REGION_INFO = [ "zone": "osdna-zone" } ] + +OS_DETAILS_INPUT = """ +NAME="Ubuntu" +VERSION="16.04 LTS (Xenial Xerus)" +ID=ubuntu +ID_LIKE=debian +PRETTY_NAME="Ubuntu 16.04 LTS" +VERSION_ID="16.04" +HOME_URL="http://www.ubuntu.com/" +SUPPORT_URL="http://help.ubuntu.com/" +BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/" +UBUNTU_CODENAME=xenial +ARCHITECURE=x86_64 +""" +OS_DETAILS = { + 'name': 'Ubuntu', + 'version': '16.04 LTS (Xenial Xerus)', + 'ID': 'ubuntu', + 'ID_LIKE': 'debian', + 'architecure': 'x86_64' +} diff --git a/app/test/fetch/link_finders/__init__.py b/app/test/fetch/link_finders/__init__.py new file mode 100644 index 0000000..b0637e9 --- /dev/null +++ b/app/test/fetch/link_finders/__init__.py @@ -0,0 +1,9 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### diff --git a/app/test/fetch/link_finders/test_data/__init__.py b/app/test/fetch/link_finders/test_data/__init__.py new file mode 100644 index 0000000..b0637e9 --- /dev/null +++ b/app/test/fetch/link_finders/test_data/__init__.py @@ -0,0 +1,9 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### diff --git a/app/test/fetch/link_finders/test_data/test_find_implicit_links.py b/app/test/fetch/link_finders/test_data/test_find_implicit_links.py new file mode 100644 index 0000000..aef20f6 --- /dev/null +++ b/app/test/fetch/link_finders/test_data/test_find_implicit_links.py @@ -0,0 +1,303 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +ENV = 'env1' +CLIQUE_CONSTRAINTS = [ + { + 'focal_point_type': 'instance', + 'constraints': ['network'] + }, + { + 'focal_point_type': 'dummy1', + 'constraints': [] + }, + { + 'focal_point_type': 'dummy2', + 'constraints': ['network', 'dummy_constraint'] + }, + { + 'focal_point_type': 'dummy3', + 'constraints': ['dummy_constraint2'] + } +] +CONSTRAINTS = ['network', 'dummy_constraint', 'dummy_constraint2'] + +LINK_ATTRIBUTES_NONE = {} +LINK_ATTRIBUTES_NONE_2 = {} +LINK_ATTRIBUTES_EMPTY = {'attributes': []} +LINK_ATTR_V1 = {'attributes': {'network': 'v1'}} +LINK_ATTR_V1_2 = {'attributes': {'network': 'v1'}} +LINK_ATTR_V2 = {'attributes': {'network': 'v2'}} +LINK_ATTR_V1_AND_A2V2 = {'attributes': {'network': 'v1', 'attr2': 'v2'}} + +LINK_TYPE_1 = { + 'link_type': 'instance-vnic', + 'source_id': 'instance1', + 'target_id': 'vnic1' +} +LINK_TYPE_1_REVERSED = { + 'link_type': 'instance-vnic', + 'source_id': 'vnic1', + 'target_id': 'instance1' +} +LINK_TYPE_1_2 = { + 'link_type': 'instance-vnic', + 'source_id': 'instance1', + 'target_id': 'vnic2' +} +LINK_TYPE_2 = { + 'link_type': 'vnic-vconnector', + 'source_id': 'vnic1', + 'target_id': 'vconnector1' +} +LINK_TYPE_3 = { + 'implicit': True, + 'link_type': 'instance-vconnector', + 'source_id': 'instance1', + 'target_id': 'vconnector1' +} +LINK_TYPE_4_NET1 = { + 'environment': ENV, + 'implicit': True, + 'link_type': 'instance-host_pnic', + 'source': 'instance1_dbid', + 'source_id': 'instance1', + 'target': 'host_pnic1_dbid', + 'target_id': 'host_pnic1', + 'host': 'host1', + 'link_name': '', + 'state': 'up', + 'source_label': '', + 'target_label': '', + 'link_weight': 0, + 'attributes': {'network': 'netID1'} +} +LINK_TYPE_5_NET2 = { + 'environment': ENV, + 'link_type': 'host_pnic-switch', + 'source_id': 'host_pnic1', + 'target': 'switch1_dbid', + 'target_id': 'switch1', + 'host': 'host2', + 'link_name': '', + 'state': 'up', + 'source_label': '', + 'target_label': '', + 'link_weight': 0, + 'attributes': {'network': 'netID2'} +} +LINK_TYPE_6_NET1 = { + 'environment': ENV, + 'link_type': 'host_pnic-switch', + 'source': 'host_pnic1_dbid', + 'source_id': 'host_pnic1', + 'target': 'switch2_dbid', + 'target_id': 'switch2', + 'host': 'host1', + 'link_name': '', + 'state': 'up', + 'source_label': '', + 'target_label': '', + 'link_weight': 0, + 'attributes': {'network': 'netID1'} +} +LINK_TYPE_7_NET1 = { + 'environment': ENV, + 'implicit': True, + 'link_type': 'instance-switch', + 'source': 'instance1_dbid', + 'source_id': 'instance1', + 'target': 'switch2_dbid', + 'target_id': 'switch2', + 'host': 'host1', + 'link_name': '', + 'state': 'up', + 'source_label': '', + 'target_label': '', + 'link_weight': 0, + 'attributes': {'network': 'netID1'} +} + +LINK_FULL_A2B = { + 'environment': ENV, + 'link_type': 'instance-vnic', + 'source': 'instance1_dbid', + 'source_id': 'instance1', + 'target': 'vnic1_dbid', + 'target_id': 'vnic1', + 'host': 'host1', + 'link_name': '', + 'state': 'up', + 'source_label': '', + 'target_label': '', + 'link_weight': 0, + 'attributes': {'network': 'netID1'} +} +LINK_FULL_B2C = { + 'environment': ENV, + 'link_type': 'vnic-vconnector', + 'source': 'vnic1_dbid', + 'source_id': 'vnic1', + 'target': 'vconnector1_dbid', + 'target_id': 'vconnector1', + 'host': 'host1', + 'link_name': '', + 'state': 'up', + 'source_label': '', + 'target_label': '', + 'link_weight': 0, + 'attributes': {'network': 'netID1'} +} +LINK_FULL_C2D = { + 'environment': ENV, + 'link_type': 'vconnector-vedge', + 'source': 'vconnector1_dbid', + 'source_id': 'vconnector1', + 'target': 'vedge1_dbid', + 'target_id': 'vedge1', + 'host': 'host1', + 'link_name': '', + 'state': 'up', + 'source_label': '', + 'target_label': '', + 'link_weight': 0, + 'attributes': {'network': 'netID1'} +} +LINK_FULL_D2E = { + 'environment': ENV, + 'link_type': 'vedge-otep', + 'source': 'vedge1_dbid', + 'source_id': 'vedge1', + 'target': 'otep1_dbid', + 'target_id': 'otep1', + 'host': 'host1', + 'link_name': '', + 'state': 'up', + 'source_label': '', + 'target_label': '', + 'link_weight': 0, + 'attributes': {'network': 'netID1'} +} +LINK_FULL_A2C = { + 'environment': ENV, + 'implicit': True, + 'link_type': 'instance-vconnector', + 'source': 'instance1_dbid', + 'source_id': 'instance1', + 'target': 'vconnector1_dbid', + 'target_id': 'vconnector1', + 'host': 'host1', + 'link_name': '', + 'state': 'up', + 'source_label': '', + 'target_label': '', + 'link_weight': 0, + 'attributes': {'network': 'netID1'} +} +LINK_FULL_B2D = { + 'environment': ENV, + 'implicit': True, + 'link_type': 'vnic-vedge', + 'source': 'vnic1_dbid', + 'source_id': 'vnic1', + 'target': 'vedge1_dbid', + 'target_id': 'vedge1', + 'host': 'host1', + 'link_name': '', + 'state': 'up', + 'source_label': '', + 'target_label': '', + 'link_weight': 0, + 'attributes': {'network': 'netID1'} +} +LINK_FULL_C2E = { + 'environment': ENV, + 'implicit': True, + 'link_type': 'vconnector-otep', + 'source': 'vconnector1_dbid', + 'source_id': 'vconnector1', + 'target': 'otep1_dbid', + 'target_id': 'otep1', + 'host': 'host1', + 'link_name': '', + 'state': 'up', + 'source_label': '', + 'target_label': '', + 'link_weight': 0, + 'attributes': {'network': 'netID1'} +} +LINK_FULL_A2D = { + 'environment': ENV, + 'implicit': True, + 'link_type': 'instance-vedge', + 'source': 'instance1_dbid', + 'source_id': 'instance1', + 'target': 'vedge1_dbid', + 'target_id': 'vedge1', + 'host': 'host1', + 'link_name': '', + 'state': 'up', + 'source_label': '', + 'target_label': '', + 'link_weight': 0, + 'attributes': {'network': 'netID1'} +} +LINK_FULL_B2E = { + 'environment': ENV, + 'implicit': True, + 'link_type': 'vnic-otep', + 'source': 'vnic1_dbid', + 'source_id': 'vnic1', + 'target': 'otep1_dbid', + 'target_id': 'otep1', + 'host': 'host1', + 'link_name': '', + 'state': 'up', + 'source_label': '', + 'target_label': '', + 'link_weight': 0, + 'attributes': {'network': 'netID1'} +} +LINK_FULL_A2E = { + 'environment': ENV, + 'implicit': True, + 'link_type': 'instance-otep', + 'source': 'instance1_dbid', + 'source_id': 'instance1', + 'target': 'otep1_dbid', + 'target_id': 'otep1', + 'host': 'host1', + 'link_name': '', + 'state': 'up', + 'source_label': '', + 'target_label': '', + 'link_weight': 0, + 'attributes': {'network': 'netID1'} +} +BASE_LINKS = [ + {'pass': 0, 'link': LINK_FULL_A2B}, + {'pass': 0, 'link': LINK_FULL_B2C}, + {'pass': 0, 'link': LINK_FULL_C2D}, + {'pass': 0, 'link': LINK_FULL_D2E}, +] +IMPLICIT_LINKS = [ + [ + {'pass': 1, 'link': LINK_FULL_A2C}, + {'pass': 1, 'link': LINK_FULL_B2D}, + {'pass': 1, 'link': LINK_FULL_C2E}, + ], + [ + {'pass': 2, 'link': LINK_FULL_A2D}, + {'pass': 2, 'link': LINK_FULL_B2E}, + ], + [ + {'pass': 3, 'link': LINK_FULL_A2E}, + ], + [] +] diff --git a/app/test/fetch/link_finders/test_find_implicit_links.py b/app/test/fetch/link_finders/test_find_implicit_links.py new file mode 100644 index 0000000..9931688 --- /dev/null +++ b/app/test/fetch/link_finders/test_find_implicit_links.py @@ -0,0 +1,107 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import bson + +from discover.link_finders.find_implicit_links import FindImplicitLinks +from test.fetch.test_fetch import TestFetch +from unittest.mock import MagicMock +from test.fetch.link_finders.test_data.test_find_implicit_links import * + +from utils.inventory_mgr import InventoryMgr + + +class TestFindImplicitLinks(TestFetch): + + def setUp(self): + super().setUp() + self.configure_environment() + self.fetcher = FindImplicitLinks() + self.fetcher.set_env(ENV) + self.fetcher.constraint_attributes = ['network'] + self.original_write_link = self.inv.write_link + self.inv.write_link = lambda x: x + self.original_objectid = bson.ObjectId + bson.ObjectId = lambda x: x + + def tearDown(self): + super().tearDown() + bson.ObjectId = self.original_objectid + self.inv.write_link = self.original_write_link + + def test_get_constraint_attributes(self): + original_find = InventoryMgr.find + InventoryMgr.find = MagicMock(return_value=CLIQUE_CONSTRAINTS) + constraint_types = self.fetcher.get_constraint_attributes() + self.assertEqual(sorted(constraint_types), sorted(CONSTRAINTS)) + InventoryMgr.find = original_find + + def test_constraints_match(self): + matcher = self.fetcher.constraints_match + self.assertTrue(matcher(LINK_ATTRIBUTES_NONE, LINK_ATTRIBUTES_NONE_2)) + self.assertTrue(matcher(LINK_ATTRIBUTES_NONE, LINK_ATTRIBUTES_EMPTY)) + self.assertTrue(matcher(LINK_ATTRIBUTES_NONE, LINK_ATTR_V1)) + self.assertTrue(matcher(LINK_ATTRIBUTES_EMPTY, LINK_ATTR_V1)) + self.assertTrue(matcher(LINK_ATTR_V1, LINK_ATTR_V1_2)) + self.assertTrue(matcher(LINK_ATTR_V1, + LINK_ATTR_V1_AND_A2V2)) + self.assertFalse(matcher(LINK_ATTR_V1, LINK_ATTR_V2)) + + def test_links_match(self): + matcher = self.fetcher.links_match + self.assertFalse(matcher(LINK_TYPE_1, LINK_TYPE_1_2)) + self.assertFalse(matcher(LINK_TYPE_1, LINK_TYPE_1_REVERSED)) + self.assertFalse(matcher(LINK_TYPE_4_NET1, LINK_TYPE_5_NET2)) + self.assertFalse(matcher(LINK_TYPE_1_2, LINK_TYPE_2)) + self.assertTrue(matcher(LINK_TYPE_1, LINK_TYPE_2)) + + def test_get_link_constraint_attributes(self): + getter = self.fetcher.get_link_constraint_attributes + self.assertEqual(getter(LINK_TYPE_1, LINK_TYPE_1_2), {}) + self.assertEqual(getter(LINK_TYPE_1, LINK_TYPE_4_NET1), + LINK_TYPE_4_NET1.get('attributes')) + self.assertEqual(getter(LINK_TYPE_4_NET1, LINK_TYPE_1), + LINK_TYPE_4_NET1.get('attributes')) + self.assertEqual(getter(LINK_TYPE_1, LINK_TYPE_5_NET2), + LINK_TYPE_5_NET2.get('attributes')) + self.assertEqual(getter(LINK_TYPE_4_NET1, LINK_TYPE_6_NET1), + LINK_TYPE_4_NET1.get('attributes')) + + def test_get_attr(self): + getter = self.fetcher.get_attr + self.assertIsNone(getter('host', {}, {})) + self.assertIsNone(getter('host', {'host': 'v1'}, {'host': 'v2'})) + self.assertEqual(getter('host', {'host': 'v1'}, {}), 'v1') + self.assertEqual(getter('host', {}, {'host': 'v2'}), 'v2') + self.assertEqual(getter('host', {'host': 'v1'}, {'host': 'v1'}), 'v1') + + def test_add_implicit_link(self): + original_write_link = self.inv.write_link + self.inv.write_link = lambda x: x + original_objectid = bson.ObjectId + bson.ObjectId = lambda x: x + add_func = self.fetcher.add_implicit_link + self.assertEqual(add_func(LINK_TYPE_4_NET1, LINK_TYPE_6_NET1), + LINK_TYPE_7_NET1) + bson.ObjectId = original_objectid + self.inv.write_link = original_write_link + + def test_get_transitive_closure(self): + self.fetcher.links = [ + {'pass': 0, 'link': LINK_FULL_A2B}, + {'pass': 0, 'link': LINK_FULL_B2C}, + {'pass': 0, 'link': LINK_FULL_C2D}, + {'pass': 0, 'link': LINK_FULL_D2E}, + ] + self.fetcher.get_transitive_closure() + for pass_no in range(1, len(IMPLICIT_LINKS)): + implicit_links = [l for l in self.fetcher.links + if l['pass'] == pass_no] + self.assertEqual(implicit_links, IMPLICIT_LINKS[pass_no-1], + 'incorrect links for pass #{}'.format(pass_no)) diff --git a/app/test/scan/test_data/configurations.py b/app/test/scan/test_data/configurations.py index 96dbc23..044ff0b 100644 --- a/app/test/scan/test_data/configurations.py +++ b/app/test/scan/test_data/configurations.py @@ -47,6 +47,7 @@ CONFIGURATIONS = { "provision": "Deploy", "env_type": "development", "name": "Monitoring", + "install_monitoring_client": True, "rabbitmq_port": "5672", "rabbitmq_pass": "osdna", "rabbitmq_user": "sensu", diff --git a/app/test/scan/test_data/scanner.py b/app/test/scan/test_data/scanner.py index 23838aa..500021d 100644 --- a/app/test/scan/test_data/scanner.py +++ b/app/test/scan/test_data/scanner.py @@ -17,6 +17,19 @@ METADATA = { "scanners_package": "discover", "scanners": {} } +LINK_FINDERS_METADATA = { + "finders_package": "discover.link_finders", + "base_finder": "FindLinks", + "link_finders": [ + "FindLinksForInstanceVnics", + "FindLinksForOteps", + "FindLinksForVconnectors", + "FindLinksForVedges", + "FindLinksForVserviceVnics", + "FindLinksForPnics", + "FindImplicitLinks" + ] +} TYPE_TO_FETCH = { "type": "host_pnic", diff --git a/app/test/scan/test_scan_metadata_parser.py b/app/test/scan/test_scan_metadata_parser.py index 91c11ef..5d91306 100644 --- a/app/test/scan/test_scan_metadata_parser.py +++ b/app/test/scan/test_scan_metadata_parser.py @@ -104,6 +104,8 @@ class TestScanMetadataParser(TestScan): 'input': METADATA_SCANNER_INCORRECT_FETCHER, 'msg': 'failed to find fetcher class f1 ' 'in scanner ScanAggregate type #1' + ' (could not import module discover.fetchers.f1.f1: ' + 'No module named \'discover.fetchers.f1\')' }, { 'input': METADATA_SCANNER_WITH_INCORRECT_CHILD, diff --git a/app/test/scan/test_scanner.py b/app/test/scan/test_scanner.py index 4a7536e..e93a35b 100644 --- a/app/test/scan/test_scanner.py +++ b/app/test/scan/test_scanner.py @@ -10,6 +10,9 @@ from discover.scanner import Scanner from test.scan.test_scan import TestScan from unittest.mock import MagicMock, patch + +from discover.link_finders.find_links_metadata_parser \ + import FindLinksMetadataParser from discover.scan_metadata_parser import ScanMetadataParser from test.scan.test_data.scanner import * from monitoring.setup.monitoring_setup_manager import MonitoringSetupManager @@ -19,7 +22,10 @@ class TestScanner(TestScan): def setUp(self): super().setUp() - ScanMetadataParser.parse_metadata_file = MagicMock(return_value=METADATA) + ScanMetadataParser.parse_metadata_file = \ + MagicMock(return_value=METADATA) + FindLinksMetadataParser.parse_metadata_file = \ + MagicMock(return_value=LINK_FINDERS_METADATA) self.scanner = Scanner() self.scanner.set_env(self.env) MonitoringSetupManager.create_setup = MagicMock() diff --git a/app/test/verify.sh b/app/test/verify.sh index a7ac9a2..681b5ed 100755 --- a/app/test/verify.sh +++ b/app/test/verify.sh @@ -11,4 +11,8 @@ set -o errexit set -o nounset set -o pipefail +PYTHONPATH=$PWD/app python3 -m unittest discover -s app/test/api +PYTHONPATH=$PWD/app python3 -m unittest discover -s app/test/event_based_scan PYTHONPATH=$PWD/app python3 -m unittest discover -s app/test/fetch +PYTHONPATH=$PWD/app python3 -m unittest discover -s app/test/scan +PYTHONPATH=$PWD/app python3 -m unittest discover -s app/test/utils diff --git a/app/utils/api_access_base.py b/app/utils/api_access_base.py new file mode 100644 index 0000000..31f50b4 --- /dev/null +++ b/app/utils/api_access_base.py @@ -0,0 +1,51 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import requests + +from discover.configuration import Configuration +from discover.fetcher import Fetcher + + +class ApiAccessBase(Fetcher): + + CONNECT_TIMEOUT = 5 + + def __init__(self, api_name=None, config=None): + super().__init__() + if api_name is None: + raise ValueError('ApiAccessBase: api_name must be defined') + self.config = {api_name: config} if config else Configuration() + self.api_config = self.config.get(api_name) + if self.api_config is None: + raise ValueError('ApiAccessBase: section "{}" missing in config' + .format(api_name)) + self.host = self.api_config.get('host', '') + self.port = self.api_config.get('port', '') + if not (self.host and self.port): + raise ValueError('Missing definition of host or port ' + + 'for {} API access' + .format(api_name)) + + def get_rel_url(self, relative_url, headers): + req_url = self.base_url + relative_url + return self.get_url(req_url, headers) + + def get_url(self, req_url, headers): + response = requests.get(req_url, headers=headers) + if response.status_code != requests.codes.ok: + # some error happened + if 'reason' in response: + msg = ', reason: {}'.format(response.reason) + else: + msg = ', response: {}'.format(response.text) + self.log.error('req_url: {} {}'.format(req_url, msg)) + return None + ret = response.json() + return ret diff --git a/app/utils/inventory_mgr.py b/app/utils/inventory_mgr.py index 722d0aa..bbc5542 100644 --- a/app/utils/inventory_mgr.py +++ b/app/utils/inventory_mgr.py @@ -265,6 +265,7 @@ class InventoryMgr(MongoAccess, metaclass=Singleton): # source_label, target_label: labels for the ends of the link (optional) def create_link(self, env, src, source_id, target, target_id, link_type, link_name, state, link_weight, + implicit=False, source_label="", target_label="", host=None, switch=None, extra_attributes=None): @@ -282,6 +283,7 @@ class InventoryMgr(MongoAccess, metaclass=Singleton): "link_weight": link_weight, "source_label": source_label, "target_label": target_label, + "implicit": implicit, "attributes": extra_attributes if extra_attributes else {} } if host: @@ -347,16 +349,18 @@ class InventoryMgr(MongoAccess, metaclass=Singleton): if not env_config: return False - # Workaround for mechanism_drivers field type - mechanism_driver = env_config['mechanism_drivers'][0] \ - if isinstance(env_config['mechanism_drivers'], list) \ - else env_config['mechanism_drivers'] + # Workarounds for mechanism_drivers and distribution_version field types + mechanism_driver = env_config.get('mechanism_drivers') + if isinstance(mechanism_driver, list): + mechanism_driver = mechanism_driver[0] + env_distribution_version = env_config.get('distribution_version') + if isinstance(env_distribution_version, list): + env_distribution_version = env_distribution_version[0] full_env = { - 'environment.distribution': env_config['distribution'], - 'environment.distribution_version': - {"$in": [env_config['distribution_version']]}, - 'environment.type_drivers': env_config['type_drivers'], + 'environment.distribution': env_config.get('distribution'), + 'environment.distribution_version': env_distribution_version, + 'environment.type_drivers': env_config.get('type_drivers'), 'environment.mechanism_drivers': mechanism_driver } return self.is_feature_supported_in_env(full_env, feature) @@ -394,8 +398,10 @@ class InventoryMgr(MongoAccess, metaclass=Singleton): self.log.error("failed to find master parent " + master_parent_id) return False - folder_id_path = "/".join((master_parent["id_path"], o["parent_id"])) - folder_name_path = "/".join((master_parent["name_path"], o["parent_text"])) + folder_id_path = "/".join((master_parent["id_path"], + o["parent_id"])) + folder_name_path = "/".join((master_parent["name_path"], + o["parent_text"])) folder = { "environment": parent["environment"], "parent_id": master_parent_id, diff --git a/app/utils/mongo_access.py b/app/utils/mongo_access.py index d4599f1..75c265c 100644 --- a/app/utils/mongo_access.py +++ b/app/utils/mongo_access.py @@ -36,8 +36,10 @@ class MongoAccess(DictNamingConverter): def __init__(self): super().__init__() - self.log_file = os.path.join(FileLogger.LOG_DIRECTORY, - MongoAccess.LOG_FILENAME) + log_dir = FileLogger.LOG_DIRECTORY \ + if os.path.isdir(FileLogger.LOG_DIRECTORY) \ + else os.path.abspath('.') + self.log_file = os.path.join(log_dir, MongoAccess.LOG_FILENAME) try: self.log = FileLogger(self.log_file) diff --git a/app/utils/ssh_connection.py b/app/utils/ssh_connection.py index e9dd39a..b9b1cde 100644 --- a/app/utils/ssh_connection.py +++ b/app/utils/ssh_connection.py @@ -22,6 +22,7 @@ class SshConnection(BinaryConverter): max_call_count_per_con = 100 timeout = 15 # timeout for exec in seconds + CONNECT_TIMEOUT = 5 DEFAULT_PORT = 22 @@ -118,7 +119,8 @@ class SshConnection(BinaryConverter): pkey=k, port=self.port if self.port is not None else self.DEFAULT_PORT, - password=self.pwd, timeout=30) + password=self.pwd, + timeout=self.CONNECT_TIMEOUT) else: port = None try: @@ -127,7 +129,7 @@ class SshConnection(BinaryConverter): username=self.user, password=self.pwd, port=port, - timeout=30) + timeout=self.CONNECT_TIMEOUT) except paramiko.ssh_exception.AuthenticationException: self.log.error('Failed SSH connect to host {}, port={}' .format(self.host, port)) diff --git a/app/utils/util.py b/app/utils/util.py index ae7b518..17a31c0 100644 --- a/app/utils/util.py +++ b/app/utils/util.py @@ -47,7 +47,6 @@ class ClassResolver: class_name = ''.join(name_parts) return class_name - @staticmethod def get_fully_qualified_class(class_name: str = None, package_name: str = "discover", @@ -58,8 +57,9 @@ class ClassResolver: module_name = ".".join(module_parts) try: class_module = importlib.import_module(module_name) - except ImportError: - raise ValueError('could not import module {}'.format(module_name)) + except ImportError as e: + raise ValueError('could not import module {}: {}' + .format(module_name, str(e))) clazz = getattr(class_module, class_name) return clazz @@ -74,7 +74,8 @@ class ClassResolver: class_name = ClassResolver.get_class_name_by_module(module_name) if class_name in ClassResolver.instances: return 'instance', ClassResolver.instances[class_name] - clazz = ClassResolver.get_fully_qualified_class(class_name, package_name, + clazz = ClassResolver.get_fully_qualified_class(class_name, + package_name, module_name) return 'class', clazz diff --git a/docs/release/Calipso-usage-stories.rst b/docs/release/Calipso-usage-stories.rst new file mode 100644 index 0000000..4c0c753 --- /dev/null +++ b/docs/release/Calipso-usage-stories.rst @@ -0,0 +1,446 @@ +***The following are fake stories, although providing real examples of +real problems that are faced today by cloud providers, and showing +possible resolutions provided by Calipso:*** + +***Enterprise use-case story (Calipso ‘S’ release):*** + +Moz is a website publishing and managing product, Moz provides +reputation and popularity tracking, helps with distributions, listing, +and ratings and provides content distributions for industry marketing. + +Moz considers moving their main content distribution application to be +hosted on https://www.dreamhost.com/, which provides shared and +dedicated IaaS and PaaS hosting based on OpenStack. + +As a major milestone for Moz’s due diligence for choosing Dreamhost, Moz +acquires a shared hosting facility from Dreamhost, that is +cost-effective and stable, it includes 4 mid-sized Web-servers, 4 +large-sized Application-servers and 2 large-sized DB servers, connected +using several networks, with some security services. + +Dreamhost executives instruct their infrastructure operations department +to make sure proper SLA and Monitoring is in-place so the due diligence +and final production deployment of Moz’s services in the Dreamhost +datacenter goes well and that Moz’s engineers receive excellent service +experience. + +Moz received the following SLA with their current VPS contract: + +- 97-day money back guarantee, in case of a single service down event + or any dissatisfaction. + +- 99.5 % uptime/availability with a weekly total downtime of 30 + minutes. + +- 24/7/365 on-call service with a total of 6 hours MTTR. + +- Full HA for all networking services. + +- Managed VPS using own Control Panel IaaS provisioning with overall + health visibility. + +- Scalable RAM, starts with 1GB can grow per request to 16GB from + within control panel. + +- Guaranteed usage of SSD or equivalent speeds, storage capacity from + 30GB to 240GB. + +- Backup service based on cinder-backup and Ceph’s dedicated backup + volumes, with restoration time below 4 hours. + +Dreamhost‘s operations factored all requirement and has decided to +include real-time monitoring and analysis for the VPS for Moz. + +One of the tools used now for Moz environment in Dreamhost is Calipso +for virtual networking. + +Here are some benefits provided by Calipso for Dreamhost operations +during service cycles: + +*Reporting:* + +Special handling of virtual networking is in place: + +- Dreamhost designed a certain virtual networking setup and + connectivity that provides the HA and performance required by the SLA + and decided on several physical locations for Moz’s virtual servers + in different availability zones. + +- Scheduling of discovery has been created, Calipso takes a snapshot of + Moz’s environment every Sunday at midnight, reporting on connectivity + among all 20 servers (10 main and 10 backups) and overall health of + that connectivity. + +- Every Sunday morning at 8am, before the week’s automatic + snapshotting, the NOC administrator runs a manual discovery and saves + that snapshot, she then runs a comparison check against last week’s + snapshot and against initial design to find any gaps or changes that + might happen due to other shared services deployments, virtual + instances and their connectivity are analyzed and reported with + Calipso’s topology and health monitoring. + +- Reports are saved for a bi-weekly reporting sent to Moz’s networking + engineers. + + *Change management:* + + If infrastructure changes needs to happen on any virtual service + (routers, switches, firewalls etc.) or on any physical server or + physical switch the following special guidelines apply: + +- Run a search on Calipso for the name of the virtual service, switch + or host. Lookup if Moz environment is using this object (using the + object’s attributes). + +- Using Calipso’s impact analysis, fill a report stating all Moz’s + objects, on which host, connected to which switch that is affected by + the planed change. + +- Run clique-type scan, using the specific object as ‘focal-point’ to + create a dedicated topology with accompanied health report before + conducting the change itself, use this a *pre snapshot*. + +- Simulate the change, using Moz’s testing environment only, make sure + HA services are in places and downtime is confirmed to be in the SLA + boundaries. + +- Using all reports provided by Calipso, along with application and + storage reports, send a detailed change request to NOC and later to + the end-customer for review. + +- During the change, make sure HA is operational, by running the same + clique-type snapshotting every 10 minutes and running a comparison. + +- NOC, while waiting for the change to complete, looks at Calipso’s + dashboard focused on MOZ’s environment, monitoring results for + service down event (as expected), impact on other objects in the + service chain - the entire Calipso clique for that object (as + expected). + +- Once operations has reported back to NOC about change done, run the + same snapshotting again as *post snapshot* and run a comparison to + make sure all virtual networking are back to the ‘as designed’ stage + and all networking services are back. + +**Example snapshot taken at one stage on Calipso for the Moz virtual +networking:** + +|image0| + + *Troubleshooting:* + + Dreamhost NOC uses Calipso dashboards for Moz’s environment for + their daily health-check. Troubleshooting starts in two cases: + +1. When a failure is detected on Calipso for any of Moz’s objects on + their virtual networking topologies, + +2. When a service case has been opened by Moz with “High Priority, + service down” flag. + +3. Networking department needs to know which virtual services are + connected to which ACI switches ports. + + The following actions are taken, using Calipso dashboards: + +- Kickoff a discovery through Calipso API for all objects related to + Moz. + +- For a service request with no Calipso error detected: using Calipso’s + impact analysis, create all cliques for all objects as focal point. + +- For an error detected by Calipso: using Calipso’s impact analysis, + create cliques for objects with errors as focal point. + +- Resulted cliques are then analyzed using detailed messaging facility + in Calipso (looking deeply into any message generated regarding the + related objects). + +- Report with ACI ports to virtual services mappings is sent to + networking department for further analysis. + + |image1| + +- If this is a failure on any physical device (host or switch) and/or + on any physical NIC (switch or host side), Calipso immediately points + this out and using the specific set of messages generated the + administrator can figure out the root cause (like optical failure, + driver, disconnect etc.). + +- In virtual object failures Calipso saves time pinpointing the servers + where erroneous objects are running, and their previous and new + connectivity details. + +- Calipso alerts on dependencies for : + +1. All related objects in the clique for that objects. + +2. Related hosts + +3. Related projects and networks + +4. Related application (\* in case Murano app has been added) + +- Administrators connects directly to the specific servers and now, + using the specific object attributes can start he’s manual + troubleshooting (actual fixing of the software issues is not + currently part of the Calipso features). + +- The NOC operators approves closing the service ticket only when all + related Calipso cliques are showing up as healthy and connectivity is + back to it’s original “as designed” stage, using Calipso older + snapshots. + +**Lookup of message – to – graph object in messaging facility:** + +|image2| + +**Finding the right object related to a specific logging/monitoring +message**: + +|image3| + +***Service Provider use-case story (Calipso ‘P’ release):*** + +BoingBoing is a specialized video casting service and blogging site. It +is using several locations to run their service (regional hubs and +central corporate campus, some hosted and some are private). + +BoingBoing contracted AT&T to build an NFV service for them, deployed on +2 new hosted regional hubs, to be brought up dynamically for special +sporting, news or cloture events. On each one of the 2 hosted virtual +environments the following service chain is created: + +1. Two vyatta 5600 virtual routers are front-end routing aggregation + function. + +2. Two Steelhead virtual wan acceleration appliances connected to + central campus for accelerating and caching of video casting + services. + +3. Two f5 BIG-IP Traffic Management (load balancing) virtual appliances. + +4. Two Cisco vASA for virtual firewall and remote-access VPN services. + +As a major milestone for BoingBoing’s due diligence for choosing AT&T +NFV service, BoingBoing acquires 2 shared hosting facilities and +automatic service from AT&T, that is cost-effective and stable, it +includes This NFV service consist of a total of 16 virtual appliance +across those 2 sites, to be created on-demand and maintained with a +certain SLA once provisioned, all NFV devices are connected using +several networks, provisioned using VPP ml2 on an OpenStack based +environment.. + +AT&T executives instruct their infrastructure operations department to +make sure proper SLA and Monitoring is in-place so the due diligence and +final production deployment of BoingBoing’s services in the AT&T +datacenters goes well and that BoingBoing’s engineers receive excellent +service experience. + +BoingBoing received the following SLA with their current VPS contract: + +- 30-day money back guarantee, in case of a single service down event + or any dissatisfaction. + +- 99.9 % uptime/availability with a weekly total downtime of 10 + minutes. + +- 24/7/365 on-call service with a total of 2 hours MTTR. + +- Full HA for all networking services. + +- Managed service using Control Panel IaaS provisioning with overall + health visibility. + +- Dedicated RAM, from16GB to 64GB from within control panel. + +- Guaranteed usage of SSD or equivalent speeds, storage capacity from + 10GB to 80GB. + +- Backup service based on cinder-backup and Ceph’s dedicated backup + volumes, with restoration time below 4 hours. + +- End-to-end throughput from central campus to dynamically created + regional sites to be always above 2Gbps, including all devices on the + service chain and the virtual networking in place. + +AT&T’s operations factored all requirement and has decided to include +real-time monitoring and analysis for the NFV environment for +BoingBoing. + +One of the tools used now for BoingBoing environment in AT&T is Calipso +for virtual networking. + +Here are some benefits provided by Calipso for AT&T operations during +service cycles: + +*Reporting:* + +Special handling of virtual networking is in place: + +- AT&T designed a certain virtual networking (SFC) setup and + connectivity that provides the HA and performance required by the SLA + and decided on several physical locations for BoingBoing’s virtual + appliances in different availability zones. + +- Scheduling of discovery has been created, Calipso takes a snapshot of + BoingBoing’s environment every Sunday at midnight, reporting on + connectivity among all 16 instances (8 per regional site, 4 pairs on + each) and overall health of that connectivity. + +- Every Sunday morning at 8am, before the week’s automatic + snapshotting, the NOC administrator runs a manual discovery and saves + that snapshot, she then runs a comparison check against last week’s + snapshot and against initial design to find any gaps or changes that + might happen due to other shared services deployments, virtual + instances and their connectivity are analyzed and reported with + Calipso’s topology and health monitoring. + +- Reports are saved for a bi-weekly reporting sent to BoingBoing’s + networking engineers. + +- Throughput is measured by a special traffic sampling technology + inside the VPP virtual switches and sent back to Calipso for + references to virtual objects and topological inventory. Dependencies + are analyzed so SFC topologies are now visualized across all sites + and includes graphing facility on the Calipso UI to visualize the + throughput. + + *Change management:* + + If infrastructure changes needs to happen on any virtual service + (NFV virtual appliances, internal routers, switches, firewalls etc.) + or on any physical server or physical switch the following special + guidelines apply: + +- Run a lookup on Calipso search-engine for the name of the virtual + service, switch or host, including names of NFV appliances as updated + in the Calipso inventory by the NFV provisioning application. Lookup + if BoingBoing environment is using this object (using the object’s + attributes). + + **Running a lookup on Calipso search-engine** + +|image4| + +- Using Calipso’s impact analysis, fill a report stating all + BoingBoing’s objects, on which host, connected to which switch that + is affected by the planed change. + +- Run clique-type scan, using the specific object as ‘focal-point’ to + create a dedicated topology with accompanied health report before + conducting the change itself, use this a *pre snapshot*. + +- Simulate the change, using BoingBoing’s testing environment only, + make sure HA services are in places and downtime is confirmed to be + in the SLA boundaries. + +- Using all reports provided by Calipso, along with application and + storage reports, send a detailed change request to NOC and later to + the end-customer for review. + +- During the change, make sure HA is operational, by running the same + clique-type snapshotting every 10 minutes and running a comparison. + +- NOC, while waiting for the change to complete, looks at Calipso’s + dashboard focused on BoingBoing’s environment, monitoring results for + SFC service down event (as expected), impact on other objects in the + service chain - the entire Calipso clique for that object (as + expected). + +- Once operations has reported back to NOC about change done, run the + same snapshotting again as *post snapshot* and run a comparison to + make sure all virtual networking are back to the ‘as designed’ stage + and all networking services are back. + +**Example snapshot taken at one stage for the BoingBoing virtual +networking and SFC:** + +|image5| + + *Troubleshooting:* + + AT&T NOC uses Calipso dashboards for BoingBoing’s environment for + their daily health-check. Troubleshooting starts in two cases: + +1. When a failure is detected on Calipso for any of BoingBoing’s objects + on their virtual networking topologies, + +2. When a service case has been opened by BoingBoing with “High + Priority, SFC down” flag. + + The following actions are taken, using Calipso dashboards: + +- Kickoff a discovery through Calipso API for all objects related to + BoingBoing. + +- For a service request with no Calipso error detected: using Calipso’s + impact analysis, create all cliques for all objects as focal point. + +- For an error detected by Calipso: using Calipso’s impact analysis, + create cliques for objects with errors as focal point. + +- Resulted cliques are then analyzed using detailed messaging facility + in Calipso (looking deeply into any message generated regarding the + related objects). + +- If this is a failure on any physical device (host or switch) and/or + on any physical NIC (switch or host side), Calipso immediately points + this out and using the specific set of messages generated the + administrator can figure out the root cause (like optical failure, + driver, disconnect etc.). + +- In virtual object failures Calipso saves time pinpointing the servers + where erroneous objects are running, and their previous and new + connectivity details. + +- \*Sources of alerts ...OpenStack, Calipso’s and Sensu are built-in + sources, other NFV related monitoring and alerting sources can be + added to Calipso messaging system. + +- Calipso alerts on dependencies for : + +1. All related objects in the clique for that objects. + +2. Related hosts + +3. Related projects and networks + +4. Related NFV service and SFC (\* in case NFV tacker has been added) + +- Administrators connects directly to the specific servers and now, + using the specific object attributes can start he’s manual + troubleshooting (actual fixing of the software issues is not + currently part of the Calipso features). + +- The NOC operators approves closing the service ticket only when all + related Calipso cliques are showing up as healthy and connectivity is + back to it’s original “as designed” stage, using Calipso older + snapshots. + +**Calipso’s monitoring dashboard shows virtual services are back to +operational state:** + +|image6| + +.. |image0| image:: media/image101.png + :width: 7.14372in + :height: 2.84375in +.. |image1| image:: media/image102.png + :width: 6.99870in + :height: 2.87500in +.. |image2| image:: media/image103.png + :width: 6.50000in + :height: 0.49444in +.. |image3| image:: media/image104.png + :width: 6.50000in + :height: 5.43472in +.. |image4| image:: media/image105.png + :width: 7.24398in + :height: 0.77083in +.. |image5| image:: media/image106.png + :width: 6.50000in + :height: 3.58611in +.. |image6| image:: media/image107.png + :width: 7.20996in + :height: 2.94792in diff --git a/docs/release/apex-scenario-guide.rst b/docs/release/apex-scenario-guide.rst new file mode 100644 index 0000000..c240b0a --- /dev/null +++ b/docs/release/apex-scenario-guide.rst @@ -0,0 +1,282 @@ +| Calipso.io +| Installation Guide + +|image0| + +Project “Calipso” tries to illuminate complex virtual networking with +real time operational state visibility for large and highly distributed +Virtual Infrastructure Management (VIM). + +We believe that Stability is driven by accurate Visibility. + +Calipso provides visible insights using smart discovery and virtual +topological representation in graphs, with monitoring per object in the +graph inventory to reduce error vectors and troubleshooting, maintenance +cycles for VIM operators and administrators. + +Table of Contents + +Calipso.io Installation Guide 1 + +1 Pre Requisites 3 + +1.1 Pre Requisites for Calipso “all in one” application 3 + +1.2 Pre Requisites for Calipso UI application 3 + +2 Installation Option used with Apex 4 + +2.1 Micro Services App, single line install 4 + +3 OPNFV Scenario 5 + +3.1 APEX automatic configurator and setup 5 + +3.2 Apex scenario 5 + +3.3 Calipso functest 6 + +TBD 6 + +Pre Requisites +=============== + +Pre Requisites for Calipso “all in one” application +---------------------------------------------------- + + Calipso’s main application is written with Python3.5 for Linux + Servers, tested successfully on Centos 7.3 and Ubuntu 16.04. When + running using micro-services many of the required software packages + and libraries are delivered per micro service, but for an “all in + one” application case there are several dependencies. + + Here is a list of the required software packages, and the official + supported steps required to install them: + +1. Python3.5.x for Linux : + https://docs.python.org/3.5/using/unix.html#on-linux + +2. Pip for Python3 : https://docs.python.org/3/installing/index.html + +3. Python3 packages to install using pip3 : + + **sudo pip3 install falcon (>1.1.0)** + + **sudo pip3 install pymongo (>3.4.0)** + + **sudo pip3 install gunicorn (>19.6.0)** + + **sudo pip3 install ldap3 (>2.1.1)** + + **sudo pip3 install setuptools (>34.3.2)** + + **sudo pip3 install python3-dateutil (>2.5.3-2)** + + **sudo pip3 install bcrypt (>3.1.1)** + + **sudo pip3 install bson** + + **sudo pip3 install websocket** + + **sudo pip3 install datetime** + + **sudo pip3 install typing** + + **sudo pip3 install kombu** + + **sudo pip3 install boltons** + + **sudo pip3 install paramiko** + + **sudo pip3 install requests ** + + **sudo pip3 install httplib2** + + **sudo pip3 install mysql.connector** + + **sudo pip3 install xmltodict** + + **sudo pip3 install cryptography** + + **sudo pip3 install docker** + +1. Git : https://git-scm.com/book/en/v2/Getting-Started-Installing-Git + +2. Docker : https://docs.docker.com/engine/installation/ + +Pre Requisites for Calipso UI application +------------------------------------------ + + Calipso UI is developed and maintained using Meteor Framework + (https://www.meteor.com/tutorials). For stability and manageability + reasons we decided to always build the latest Calipso UI as a Docker + container pre-parameterized for stable and supported behavior. The + required steps for installing the Calipso UI with several options + are listed below. + +Installation Option used with Apex +================================== + +Micro Services App, single line install +--------------------------------------- + + For most users, this will be the fastest and more reliable install + option. We currently have Calipso divided into 7 major containers, + those are installed using a single installer. The Calipso containers + are pre-packaged and fully customized per our design needs. Here are + the required steps for installation using this option: + +1. Follow steps 1- 5 per section 2.1 above. + +2. Install Docker : https://docs.docker.com/engine/installation/ + +3. Install the following python3 libraries using pip3 : docker, pymongo + +4. Although Calipso installer can download all needed containers, if + they doesn’t exist locally already, we recommend doing a manual + download of all 7 containers, providing better control and logging: + + **sudo docker login** # use your DockerHub username and password to + login. + + **sudo docker pull korenlev/calipso:scan** # scan container used to + scan VIM + + **sudo docker pull korenlev/calipso:listen** # listen container to + attach to VIM’s BUS. + + **sudo docker pull korenlev/calipso:api** # api container for + application integration + + **sudo docker pull korenlev/calipso:sensu** # sensu server container + for monitoring + + **sudo docker pull korenlev/calipso:mongo** # calipso mongo DB + container + + **sudo docker pull korenlev/calipso:ui** # calipso ui container + + **sudo docker pull korenlev/calipso:ldap** # calipso ldap container + +1. Check that all containers were downloaded and registered + successfully: + + **sudo docker images** + + Expected results (As of Aug 2017): + + **REPOSITORY TAG IMAGE ID CREATED SIZE** + + **korenlev/calipso listen 12086aaedbc3 6 hours ago 1.05GB** + + **korenlev/calipso api 34c4c6c1b03e 6 hours ago 992MB** + + **korenlev/calipso scan 1ee60c4e61d5 6 hours ago 1.1GB** + + **korenlev/calipso sensu a8a17168197a 6 hours ago 1.65GB** + + **korenlev/calipso mongo 17f2d62f4445 22 hours ago 1.31GB** + + **korenlev/calipso ui ab37b366e812 11 days ago 270MB** + + **korenlev/calipso ldap 316bc94b25ad 2 months ago 269MB** + +1. Run the calipso installer using single line arguments: + + **python3 calipso/app/install/calipso-installer.py--command + start-all --copy q** + + This should launch all calipso modules in sequence along with all + needed configuration files placed in /home/calipso. + +OPNFV Scenario +=============== + +Although calipso is designed for any VIM and for enterprise use-cases +too, service providers can use additional capability to install calipso +with Apex for OPNFV. + +APEX automatic configurator and setup +------------------------------------- + + When using apex to install OPNFV, the Triple-O based OpenStack is + installed automatically and calipso installation can be initiated + automatically after apex completes the VIM installation process for + a certain scenario. + + In this case setup\_apex\_environment.py can be used for creating a + new environment automatically into mongoDB and UI of Calipso + (instead of using the calipso UI to do that as typical user would + do), then detailed scanning can start immediately, the following + options are available for setup\_apex\_environment.py: + + **-m [MONGO\_CONFIG], --mongo\_config [MONGO\_CONFIG]** + + **name of config file with MongoDB server access details** + + **(Default: /local\_dir/calipso\_mongo\_access.conf)** + + **-d [CONFIG\_DIR], --config\_dir [CONFIG\_DIR]** + + **path to directory with config data (Default:** + + **/home/calipso/apex\_setup\_files)** + + **-i [INSTALL\_DB\_DIR], --install\_db\_dir [INSTALL\_DB\_DIR]** + + **path to directory with DB data (Default:** + + **/home/calipso/Calipso/app/install/db)** + + **-a [APEX], --apex [APEX]** + + **name of environment to Apex host** + + **-e [ENV], --env [ENV]** + + **name of environment to create(Default: Apex-Euphrates)** + + **-l [LOGLEVEL], --loglevel [LOGLEVEL]** + + **logging level (default: "INFO")** + + **-f [LOGFILE], --logfile [LOGFILE]** + + **log file (default:** + + **"/home/calipso/log/apex\_environment\_fetch.log")** + + **-g [GIT], --git [GIT]** + + **URL to clone Git repository (default:** + + **https://git.opnfv.org/calipso)** + +Apex scenario +------------- + + Starting Euphrates 1.0 the following scenario added with Apex + installer: + + **os-nosdn-calipso-noha** + + Following CI jobs defined: + + https://build.opnfv.org/ci/job/calipso-verify-euphrates/ + + https://build.opnfv.org/ci/job/apex-testsuite-os-nosdn-calipso-noha-baremetal-euphrates/ + + https://build.opnfv.org/ci/job/apex-os-nosdn-calipso-noha-baremetal-euphrates/ + + Note: destination deploy server needs to have pre-requisites + detailed above. + +Calipso functest +---------------- + +TBD +---- + +.. |image0| image:: media/image1.png + :width: 6.50000in + :height: 4.27153in diff --git a/docs/release/developer-guide.pdf b/docs/release/developer-guide.pdf new file mode 100644 index 0000000..2ed302e Binary files /dev/null and b/docs/release/developer-guide.pdf differ diff --git a/docs/release/developer-guide.rst b/docs/release/developer-guide.rst new file mode 100644 index 0000000..0de3f57 --- /dev/null +++ b/docs/release/developer-guide.rst @@ -0,0 +1,1338 @@ +| Calipso +| Developer Guide + +|image0| + +Project “Calipso” tries to illuminate complex virtual networking with +real time operational state visibility for large and highly distributed +Virtual Infrastructure Management (VIM). + +We believe that Stability is driven by accurate Visibility. + +Calipso provides visible insights using smart discovery and virtual +topological representation in graphs, with monitoring per object in the +graph inventory to reduce error vectors and troubleshooting, maintenance +cycles for VIM operators and administrators. + +Project architecture +==================== + +Calipso comprises two major parts: application and UI. We’ll focus on +the former in this developer guide. + +Current project structure is as follows: + +- root/ + + - app/ + + - api/ + + - responders/ + + - auth/ + + - resource/ + + - *server.py* + + - config/ + + - *events.json* + + - *scanners.json* + + - discover/ + + - events/ + + - listeners/ + + - *default\_listener.py* + + - *listener\_base.py* + + - handlers/ + + - *event\_base.py* + + - *event\_\*.py* + + - fetchers/ + + - aci/ + + - api/ + + - cli/ + + - db/ + + - *event\_manager.py* + + - *scan.py* + + - *scan\_manager.py* + + - monitoring/ + + - checks/ + + - handlers/ + + - *monitor.py* + + - setup/ + + - *monitoring\_setup\_manager.py* + + - test/ + + - api/ + + - event\_based\_scan/ + + - fetch/ + + - scan/ + + - utils/ + + - ui/ + +Application structure +--------------------- + +‘API’ package +~~~~~~~~~~~~~ + +Calipso API is designed to be used by native and third-party +applications that are planning to use Calipso discovery application. + +***api/responders*** + +This package contains all exposed API endpoint handlers: + +*auth* package contains token management handlers, + +*resource* package contains resource handlers. + +***server.py*** + +API server startup script. In order for it to work correctly, connection +arguments for a Mongo database used by a Calipso application instance +are required: + +-m [MONGO\_CONFIG], --mongo\_config [MONGO\_CONFIG] + +name of config file with mongo access details + +--ldap\_config [LDAP\_CONFIG] + +name of the config file with ldap server config + +details + +-l [LOGLEVEL], --loglevel [LOGLEVEL] + +logging level (default: 'INFO') + +-b [BIND], --bind [BIND] + +binding address of the API server (default + +127.0.0.1:8000) + +-y [INVENTORY], --inventory [INVENTORY] + +name of inventory collection (default: 'inventory') + +-t [TOKEN\_LIFETIME], --token-lifetime [TOKEN\_LIFETIME] + +lifetime of the token + +For detailed reference and endpoints guide, see the API Guide document. + +‘Discover’ package +~~~~~~~~~~~~~~~~~~ + +‘Discover’ package contains the core Calipso functionality which +involves: + +- scanning a network topology using a defined suite of scanners (see + `Scanning concepts <#scanning-concepts>`__, `Scanners configuration + file structure <#the-scanners-configuration-file-structure>`__) that + use fetchers to get all needed data on objects of the topology; + +- tracking live events that modifies the topology in any way (by adding + new object, updating existing or deleting them) using a suite of + event handlers and event listeners; + +- managing the aforementioned suites using specialized manager scripts + (*scan\_manager.py* and *event\_manager.py*) + +‘Tests’ package +~~~~~~~~~~~~~~~ + +‘Tests’ package contains unit tests for main Calipso components: API, +event handlers, fetchers, scanners and utils. + +Other packages +~~~~~~~~~~~~~~ + +***Install*** + +Installation and deployment scripts (with initial data for Calipso +database). + +***Monitoring*** + +Monitoring configurations, checks and handlers (see +`Monitoring <#monitoring>`__ section and Monitoring Guide document). + +***Utils*** + +Utility modules for app-wide use (inventory manager, mongo access, +loggers, etc.). + +Scanning Guide +============== + + Introduction to scanning +------------------------- + +Architecture overview +~~~~~~~~~~~~~~~~~~~~~ + +Calipso backend will scan any OpenStack environment to discover the +objects that it is made of, and place the objects it discovered in a +MongoDB database. + +Following discovery of objects, Calipso will: + +| Find what links exist between these objects, and save these links to + MongoDB as well. +| For example, it will create a pnic-network link from a pNIC (physical + NIC) and the network it is connected to. + +Based on user definitions, it will create a 'clique' for each object +using the links it previously found. These cliques are later used to +present graphs for objects being viewed in the Calipso UI. This is not a +clique by graph theory definition, but more like the social definition +of clique: a graph of related, interconnected nodes. + +  + +OpenStack Scanning is done using the following methods, in order of +preference: + +1. OpenStack API + +2. MySQL DB - fetch any extra detail we can from the infrastructure + MySQL DB used by OpenStack + +3. CLI - connect by SSH to the hosts in the OpenStack environment to run + commands, e.g. ifconfig, that will provide the most in-depth details. + +  + +| *Note*: 'environment' in Calipso means a single deployment of + OpenStack, possibly containing multiple tenants (projects), hosts and + instances (VMs). A single Calipso instance can handle multiple + OpenStack environments.  +| However, we expect that typically Calipso will reside inside an + OpenStack control node and will handle just that node's OpenStack + environment. + +  + +***Environment*** + +| The Calipso scan script, written in Python, is called scan.py. +| It uses Python 3, along with the following libraries: + +- pymongo - for MongoDB access + +- mysql-connector - For MySQL DB access + +- paramiko - for SSH access + +- requests - For handling HTTP requests and responses to the OpenStack + API + +- xmltodict - for handling XML output of CLI commands + +- cryptography - used by Paramiko + +See Calipso installation guide for environment setup instructions. + +***Configuration*** + +The configuration for accessing the OpenStack environment, by API, DB or +SSH, is saved in the Calipso MongoDB *“environments\_config”* +collection. + +Calipso can work with a remote MongoDB instance, the details of which +are read from a configuration file (default: */etc/calipso/mongo.conf*). + +| The first column is the configuration key while the second is the + configuration value, in the case the value is the server host name or + IP address. +| Other possible keys for MongoDB access: + +- port: IP port number + +- Other parameters for the PyMongo MongoClient class constructor + +Alternate file location can be specified using the CLI -m parameter. + +Scanning concepts +~~~~~~~~~~~~~~~~~ + +***DB Schema*** + +Objects are stored in the inventory collection, named *“inventory”* by +default, along with the accompanying collections, named by +default: \ *"links", "cliques", "clique\_types" and +"clique\_constraints"*. For development, separate sets of collections +can be defined per environment (collection names are created by +appending the default collection name to the alternative inventory +collection name). + +The inventory, links and cliques collections are all designed to work +with a multi-environment scenario, so documents are marked with an +*"environment"* attribute. + +The clique\_types collection allows Calipso users (typically +administrators) to define how the "clique" graphs are to be defined.  + +It defines a set of link types to be traversed when an object such as an +instance is clicked in the UI (therefore referred to as the focal +point). See "Clique Scanning" below. This definition can differ between +environments. + +Example: for focal point type "instance", the link types are often set +to + +- instance-vnic + +- vnic-vconnector + +- vconnector-vedge + +- vedge-pnic + +- pnic-network + +| The clique\_constraints collection defines a constraint on links + traversed for a specific clique when starting from a given focal + point.  +| For example: instance cliques are constrained to a specific + network. If we wouldn't have this constraint, the resulting graph + would stretch to include objects from neighboring networks that are + not really related to the instance. + +\ ***Hierarchy of Scanning*** + +The initial scanning is done hierarchically, starting from the +environment level and discovering lower levels in turn. + +Examples: + +- Under environment we scan for regions and projects (tenants). + +- Under availability zone we have hosts, and under hosts we have + instances and host services + +The actual scanning order is not always same as the logical hierarchical +order of objects, to improve scanning performance. + +Some objects are referenced multiple times in the hierarchy. For +example, hosts are always in an availability zone, but can also be part +of a host aggregate. Such extra references are saved as references to +the main object. + +***Clique Scanning*** + +| For creating cliques based on the discovered objects and links, clique + types need to be defined for the given environment. +| A clique type specifies the list of link types used in building a + clique for a specific focal point object type. + +For example, it can define that for instance objects we want to have the +following link types: + +- instance-vnic + +- vnic-vconnector + +- vconnector-vedge + +- vedge-pnic + +- pnic-network + +  + +As in many cases the same clique types are used, default clique types +will be provided with a new Calipso deployment. + +\ ***Clique creation algorithm*** + +- For each clique type CT: + + - For each focal point object F of the type specified as the clique + type focal point type: + + - Create a new clique C + + - Add F to the list of objects included in the clique + + - For each link type X-Y of the link types in CT: + + - Find all the source objects of type x that are already + included in the clique + + - For each such source object S: + + - for all links L of type X-Y that have S as their source + + - Add the object T of type Y that is the target in L to the + list of objects included in the clique + + - Add L to the list of links in the clique C + +How to run scans +---------------- + +For running environment scans Calipso uses a specialized daemon script +called *scan manager*. If Calipso application is deployed in docker +containers, scan manager will run inside the *calipso-scan* container. + +Scan manager uses MongoDB connection to fetch requests for environment +scans and execute them by running a *scan* script. It also performs +extra checks and procedures connected to scan failure/completion, such +as marking *environment* as scanned and reporting errors (see +`details <#scan-manager>`__). + +Scan script workflow: + +1. Loads specific scanners definitions from a predefined metadata file + (which can be extended in order to support scanning of new object + types). + +2. Runs the root scanner and then children scanners recursively (see + `Hierarchy of scanning <#Hierarchy_of_scanning>`__) + + a. Scanners do all necessary work to insert objects in *inventory*. + +3. Finalizes the scan and publishes successful scan completion. + +Scan manager +~~~~~~~~~~~~ + +Scan manager is a script which purpose is to manage the full lifecycle +of scans requested through API. It runs indefinitely while: + +1. Polling the database (*scans* and *scheduled\_scans* collections) for + new and scheduled scan requests; + +2. Parsing their configurations; + +3. Running the scans; + +4. Logging the results. + +Scan manager can be run in a separate container provided that it has +connection to the database and the topology source system. + +Monitoring +---------- + +***Monitoring Subsystem Overview*** + +Calipso monitoring uses Sensu to remotely track actual state of hosts. + +A Sensu server is installed as a Docker image along with the other +Calipso components. + +  + +Remote hosts send check events to the Sensu server.  + +We use a filtering of events such that the first occurrence of a check +is always used, after that cases where status is unchanged are ignored. + +When handling a check event, the Calipso Sensu handlers will find the +matching Calipso object, and update its status. + +We also keep the timestamp of the last status update, along with the +full check output. + +Setup of checks and handlers code on the server and the remote hosts can +be done by Calipso. It is also possible to have this done using another +tool, e.g. Ansible or Puppet. + +More info is available in Monitoring Guide document. + +  + +***Package Structure*** + +Monitoring package is divided like this: + +1.      Checks: these are the actual check scripts that will be run on +the hosts; + +2.      Handlers: the code that does handling of check events; + +3.      Setup: code for setting up handlers and checks. + +Events Guide +============ + +Introduction +------------ + +Events +~~~~~~ + +Events in general sense are any changes to the monitored topology +objects that are trackable by Calipso. We currently support subscription +to Neutron notification queues for several OpenStack distributions as a +source of events. + +The two core concepts of working with events are *listening to events* +and *event handling*, so the main module groups in play are the *event +listener* and *event handlers*. + +Event listeners +~~~~~~~~~~~~~~~ + +An event listener is a module that handles connection to the event +source, listening to the new events and routing them to respective event +handlers. + +An event listener class should be designed to run indefinitely in +foreground or background (daemon) while maintaining a connection to the +source of events (generally a message queue like RabbitMQ or Apache +Kafka). Each incoming event is examined and, if it has the correct +format, is routed to the corresponding event handler class. The routing +can be accomplished through a dedicated event router class using a +metadata file and a metadata parser (see `Metadata +parsers <#metadata-parsers>`__). + +Event handlers +~~~~~~~~~~~~~~ + +An event handler is a specific class that parses the incoming event +payload and performs a certain CUD (Create/Update/Delete) operation on +zero or more database objects. Event handler should be independent of +the event listener implementation. + +Event manager +~~~~~~~~~~~~~ + +Event manager is a script which purpose is to manage event listeners. It +runs indefinitely and performs the following operations: + +1. Starts a process for each valid entry in *environments\_config* + collection that is scanned (*scanned == true*) and has the *listen* + flag set to *true*; + +2. Checks the *operational* statuses of event listeners and updating + them in *environments\_config* collection; + +3. Stops the event listeners that no longer qualify for listening (see + step 1); + +4. Restarts the event listeners that quit unexpectedly; + +5. Repeats steps 1-5 + +Event manager can be run in a separate container provided that it has +connection to the database and to all events source systems that event +listeners use. + +Contribution +~~~~~~~~~~~~ + +You can contribute to Calipso *events* system in several ways: + +- create custom event handlers for an existing listener; + +- create custom event listeners and reuse existing handlers; + +- create custom event handlers and listeners. + +See `Creating new event handlers <#creating-new-event-handlers>`__ and +`Creating new event listeners <#creating-new-event-listeners>`__ for the +respective guides. + +Contribution +============ + +This section covers the designed approach to contribution to Calipso. + +The main scenario of contribution consists of introducing a new *object* +type to the discovery engine, defining *links* that connect this new +object to existing ones, and describing a *clique* (or cliques) that +makes use of the object and its links. Below we describe how this +scenario should be implemented, step-by-step. + +*Note*: Before writing any new code, you need to create your own +environment using UI (see User Guide document) or API (see the API guide +doc). Creating an entry directly in *“environments\_config”* collection +is not recommended. + +Creating new object types +------------------------- + +Before you proceed with creation of new object type, you need to make +sure the following requirements are met: + +- New object type has a unique name and purpose + +- New object type has an existing parent object type + +First of all, you need to create a fetcher that will take care of +getting info on objects of the new type, processing it and adding new +entries in Calipso database. + +Creating new fetchers +~~~~~~~~~~~~~~~~~~~~~ + +A fetcher is a common name for a class that handles fetching of all +objects of a certain type that have a common parent object. The source +of this data may be already implemented in Calipso (like OpenStack API, +CLI and DB sources) or you may create one yourself. + +***Common fetchers*** + +Fetchers package structure should adhere to the following pattern (where +*%source\_name%* is a short prefix like *api, cli, db*): + +- app + + - discover + + - fetchers + + - *%source\_name%* + + - *%source\_name%*\ \_\ *%fetcher\_name%.*\ py + +If you reuse the existing data source, your new fetcher should subclass +the class located in *%source\_name%\_access* module inside the +*%source\_name%* directory. + +Fetcher class name should repeat the module name, except in CamelCase +instead of snake\_case. + +Example: if you are adding a new cli fetcher, you should subclass +*CliAccess* class found by *app/discover/fetchers/cli/cli\_access.py* +path. If the module is named *cli\_fetch\_new\_objects.py*, fetcher +class should be named *CliFetchNewObjects*. + +If you are creating a fetcher that uses new data source, you may +consider adding an “access” class for this data source to store +convenience methods. In this case, the “access” class should subclass +the base Fetcher class (found in *app/discover/fetcher.py*) and the +fetcher class should subclass the “access” class. + +All business logic of a fetcher should be defined inside the overridden +method from base Fetcher class *get(self, parent\_id)*. You should use +the second argument that is automatically passed by parent scanner to +get the parent entity from database and any data you may need. This +method has to return a list of new objects (dicts) that need to be +inserted in Calipso database. Their parent object should be passed along +other fields (see example). + +*Note*: types of returned objects should match the one their fetcher is +designed for. + +***Example***: + +**app/discover/fetchers/cli/cli\_fetch\_new\_objects.py** + + | **from** discover.fetchers.cli.cli\_access **import** CliAccess + | **from** utils.inventory\_mgr **import** InventoryMgr + | **class** CliFetchNewObjects(CliAccess): + | **def** \_\_init\_\_(self): + | super().\_\_init\_\_() + | self.inv = InventoryMgr() + | **def** get(self, parent\_id): + | parent = self.inv.get\_by\_id(self.env, parent\_id) + | *# do something + *\ objects = [{**"type"**: **"new\_type"**, **"id"**: **"1234"**, + **"parent"**: parent}, + | {**"type"**: **"new\_type"**, **"id"**: **"2345"**, **"parent"**: + parent}] + | **return** objects + +This is an example of a fetcher that deals with the objects of type +*“new\_type”*. It uses the parent id to fetch the parent object, then +performs some operations in order to fetch the new objects and +ultimately returns the objects list, at which point it has gathered all +required information. + +\ ***Folder fetcher*** + +A special type of fetchers is the folder fetcher. It serves as a dummy +object used to aggregate objects in a specific point in objects +hierarchy. If you would like to logically separate children objects from +parent, you may use folder fetcher found at +*app/discover/fetchers/folder\_fetcher.py*. + +Usage is described `here <#Folder_scanner>`__. + +The scanners configuration file structure +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Scanners.json** (full path *app/config/scanners.json*) is an essential +configuration file that defines scanners hierarchy. It has a forest +structure, meaning that it is a set of trees, where each tree has a +*root* scanner, potentially many levels of *children* scanners and +pointers from parent scanners to children scanners. Scanning hierarchy +is described `here <#Hierarchy_of_scanning>`__. + +A scanner is essentially a list of fetchers with configuration (we’ll +call those **Fetch types**). Fetch types can be **Simple** and +**Folder**, described below. + +***Simple fetch type*** + +A simple fetch type looks like this: + + | { + | **"type"**: **"project"**, + | **"fetcher"**: **"ApiFetchProjects"**, + | **"object\_id\_to\_use\_in\_child"**: **"name"**, + + | **"environment\_condition"**: { + | **"mechanism\_drivers"**: **"VPP" + ** }, + | **"children\_scanner"**: **"ScanProject" + **} + +Supported fields include: + +- *“fetcher”* – class name of fetcher that the scanner uses; + +- *“type”* – object type that the fetcher works with; + +- *“children\_scanner”* – (optional) full name of a scanner that should + run after current one finishes; + +- *“environment\_condition”* – (optional) specific constraints that + should be checked against the environment in *environments\_config* + collection before execution; + +- *“object\_id\_to\_use\_in\_child*\ ” – (optional) which parent field + should be passed as parent id to the fetcher (default: “id”). + + \ ***Folder fetch type*** + +Folder fetch types deal with folder fetchers (described +`here <#Folder_fetcher>`__) and have a slightly different structure: + + | { + | **"type"**: **"aggregates\_folder"**, + | **"fetcher"**: { + | **"folder"**: **true**, + | **"types\_name"**: **"aggregates"**, + | **"parent\_type"**: **"region" + **}, + + **"object\_id\_to\_use\_in\_child"**: **"name"**, + + | **"environment\_condition"**: { + | **"mechanism\_drivers"**: **"VPP" + ** }, + | **"children\_scanner"**: **"ScanAggregatesRoot" + **} + +The only difference is that *“fetcher”* field is now a dictionary with +the following fields: + +- *“folder”* – should always be **true**; + +- *“types\_name”* – type name in plural (with added ‘s’) of objects + that serve as folder’s children + +- *“parent\_type”* – folder’s parent type (basically the parent type of + folder’s objects). + +Updating scanners +~~~~~~~~~~~~~~~~~ + +After creating a new fetcher, you should integrate it into scanners +hierarchy. There are several possible courses of action: + +***Add new scanner as a child of an existing one*** + +If the parent type of your newly added object type already has a +scanner, you can add your new scanner as a child of an existing one. +There are two ways to do that: + +1. Add new scanner as a *“children\_scanner”* field to parent scanner + + ***Example*** + + Before: + + | **"ScanHost"**: [ + | { + | **"type"**: **"host"**, + | **"fetcher"**: **"ApiFetchProjectHosts"**, + | } + | ], + + After: + + | **"ScanHost"**: [ + | { + | **"type"**: **"host"**, + | **"fetcher"**: **"ApiFetchProjectHosts"**, + | **"children\_scanner"**: **"NewTypeScanner" + **} + | ], + | **"NewTypeScanner"**: [ + | { + | **"type"**: **"new\_type"**, + | **"fetcher"**: **"CliFetchNewType" + **} + | ] + +1. Add new fetch type to parent scanner (in case if children scanner + already exists) + + ***Example*** + + Before: + + | **"ScanHost"**: [ + | { + | **"type"**: **"host"**, + | **"fetcher"**: **"ApiFetchProjectHosts"**, + | **"children\_scanner"**: **"ScanHostPnic" + **} + | ], + + After: + + | **"ScanHost"**: [ + | { + | **"type"**: **"host"**, + | **"fetcher"**: **"ApiFetchProjectHosts"**, + | **"children\_scanner"**: **"ScanHostPnic" + **}, + | { + | **"type"**: **"new\_type"**, + | **"fetcher"**: **"CliFetchNewType" + **} + | ], + +***Add new scanner and set an existing one as a child*** + +***Example*** + + Before: + + | **"ScanHost"**: [ + | { + | **"type"**: **"host"**, + | **"fetcher"**: **"ApiFetchProjectHosts"**, + | **"children\_scanner"**: **"ScanHostPnic" + **} + | ], + + After: + + | **"NewTypeScanner"**: [ + | { + | **"type"**: **"new\_type"**, + | **"fetcher"**: **"CliFetchNewType"**, + | **"children\_scanner"**: **"ScanHost" + **} + | ] + + | **"ScanHost"**: [ + | { + | **"type"**: **"host"**, + | **"fetcher"**: **"ApiFetchProjectHosts"**, + | **"children\_scanner"**: **"ScanHostPnic" + **} + | ], + +***Other cases*** + +You may choose to combine approaches or use none of them and create an +isolated scanner if needed. + +Updating constants collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Before testing your new scanner and fetcher you need to add the newly +created object type to *“constants”* collection in Calipso database: + +1. **constants.object\_types** document + + Append a *{“value”: “new\_type”, “label”: “new\_type”}* object to + **data** list. + +1. **constants.scan\_object\_types** document + + Append a *{“value”: “new\_type”, “label”: “new\_type”}* object to + **data** list. + +1. **constants.object\_types\_for\_links** document + + If you’re planning to build links using this object type (you + probably are), append a *{“value”: “new\_type”, “label”: + “new\_type”}* object to **data** list. + +Setting up monitoring +~~~~~~~~~~~~~~~~~~~~~ + +In order to setup monitoring for the new object type you have defined, +you’ll need to add a Sensu check: + +1. Add a check script in app/monitoring/checks: + + a. | Checks should return the following values: + | 0: **OK** + | 1: **Warning** + | 2: **Error** + + b. Checks can print the underlying query results to stdout. Do so + within reason, as this output is later stored in the DB, so avoid + giving too much output; + + c. Test your script on a remote host: + + i. Write it in */etc/sensu/plugins* directory; + + ii. Update the Sensu configuration on the remote host to run this + check; + + iii. Add the check in the “checks” section of + */etc/sensu/conf.d/client.json*; + + iv. The name under which you save the check will be used by the + handler to determine the DB object that it relates to; + + v. Restart the client with the command: *sudo service + sensu-client restart*; + + vi. Check the client log file to see the check is run and + produces the expected output (in */var/log/sensu* directory). + + d. Add the script to the source directory (*app/monitoring/checks*). + +2. Add a handler in app/monitoring/handlers: + + a. If you use a standard check naming scheme and check an object, the + *BasicCheckHandler* can take care of this, but add the object type + in *basic\_handling\_types* list in *get\_handler()*; + + b. If you have a more complex naming scheme, override + MonitoringCheckHandler. See HandleOtep for example. + +3. If you deploy monitoring using Calipso: + + a. Add the check in the *monitoring\_config\_templates* collection. + +*Check Naming* + +The check name should start with the type of the related object, +followed by an underscore (“\_”). For example, the name for a check +related to an OTEP (type “otep”)  will start with “otep\_“. It should +then be followed by the object ID. + +  + +For checks related to links, the check name will have this +format:
link\_\_\_ + +Creating new link types +----------------------- + +After you’ve added a new object type you may consider adding new link +types to connect objects of new type to existing objects in topology. +Your new object type may serve as a *source* and/or *target* type for +the new link type. + +The process of new link type creation includes several steps: + +1. Write a link finder class; + +2. Add the link finder class to the link finders configuration file; + +3. Update *“constants”* collection with the new link types. + +Writing link finder classes +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A new link finder class should: + +1. Subclass *app.discover.link\_finders.FindLinks* class; + +2. Be located in the *app.discover.link\_finders* package; + +3. Define an instance method called *add\_links(self)* with no + additional arguments. This method is the only entry point for link + finder classes. + +*FindLinks* class provides access to inventory manager to its subclasses +which they should use to their advantage. It also provides a convenience +method *create\_links(self, …)* for saving links to database. It is +reasonable to call this method at the end of *add\_links* method. + +You may opt to add more than one link type at a time in a single link +finder. + +***Example*** + + | **from** discover.find\_links **import** FindLinks + | **class** FindLinksForNewType(FindLinks): + | **def** add\_links(self): + | new\_objects = self.inv.find\_items({\ **"environment"**: + self.get\_env(), + | **"type"**: **"new\_type"**}) + | **for** new\_object **in** new\_objects: + | old\_object = self.inv.get\_by\_id(environment=self.get\_env(), + | item\_id=new\_object[**"old\_object\_id"**]) + | link\_type = **"old\_type-new\_type" + **\ link\_name = **"{}-{}"**.format(old\_object[**"name"**], + new\_object[**"name"**]) + | state = **"up"** *# TBD + *\ link\_weight = 0 *# TBD + *\ self.create\_link(env=self.get\_env(), + | source=old\_object[**"\_id"**], + | source\_id=old\_object[**"id"**], + | target=new\_object[**"\_id"**], + | target\_id=new\_object[**"id"**], + | link\_type=link\_type, + | link\_name=link\_name, + | state=state, + | link\_weight=link\_weight) + +Updating the link finders configuration file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default link finders configuration file can be found at +*/app/config/link\_finders.json* and has the following structure: + + | { + | **"finders\_package"**: **"discover.link\_finders"**, + | **"base\_finder"**: **"FindLinks"**, + | **"link\_finders"**: [ + | **"FindLinksForInstanceVnics"**, + | **"FindLinksForOteps"**, + | **"FindLinksForPnics"**, + | **"FindLinksForVconnectors"**, + | **"FindLinksForVedges"**, + | **"FindLinksForVserviceVnics" + **] + | } + +File contents: + +- *finders\_package* – python path to the package that contains link + finders (relative to $PYTHONPATH environment variable); + +- *base\_finder* – base link finder class name; + +- *link\_finders* – class names of actual link finders. + +If your new fetcher meets the requirements described in `Writing link +finder classes <#writing-link-finder-classes>`__ section, you can append +its name to the *“link\_finders”* list in *link\_finders.json* file. + +Updating constants collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Before testing your new links finder, you need to add the newly created +link types to *“constants”* collection in Calipso database: + +1. **constants.link\_types** document + + Append a *{“value”: “source\_type-target\_type”, “label”: + “source\_type-target\_type”}* object to **data** list for each new + link type. + +Creating custom link finders configuration file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you consider writing a custom list finders configuration file, you +should also follow the guidelines from 4.2.1-4.2.3 while designing link +finder classes and including them in the new link finders source file. + +The general approach is the following: + +1. Custom configuration file should have the same key structure with the + basic one; + +2. You should create a *base\_finder* class that subclasses the basic + FindLinks class (see `Writing link finder + classes <#writing-link-finder-classes>`__); + +3. Your link finder classes should be located in the same package with + your *base\_finder* class; + +4. Your link finder classes should subclass your *base\_finder* class + and override the *add\_links(self)* method. + +Creating new clique types +------------------------- + +Two steps in creating new clique types and including them in clique +finder are: + +1. Designing new clique types + +2. Updating clique types collection + +Designing new clique types +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A clique type is basically a list of links that will be traversed during +clique scans (see `Clique creation algorithm <#clique_creation>`__). The +process of coming up with clique types involves general networking +concepts knowledge as well as expertise in monitored system details +(e.g. OpenStack distribution specifics). In a nutshell, it is not a +trivial process, so the clique design should be considered carefully. + +The predefined clique types (in *clique\_types* collection) may give you +some idea about the rationale behind clique design. + +Updating clique types collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +After designing the new clique type, you need to update the +*clique\_types* collection in order for the clique finder to use it. For +this purpose, you should add a document of the following structure: + + {  + +     "environment": "ANY",  + +     "link\_types": [ + +         "instance-vnic",  + +         "vnic-vconnector",  + +         "vconnector-vedge",  + +         "vedge-otep",  + +         "otep-vconnector",  + +         "vconnector-host\_pnic",  + +         "host\_pnic-network" + +     ],  + +     "name": "instance",  + +     "focal\_point\_type": "instance" + + } + +Document fields are: + +- *environment* – can either hold the environment name, for which the + new clique type is designed, or **“ANY”** if the new clique type + should be added to all environments; + +- *name* – display name for the new clique type; + +- *focal\_point\_type* – the aggregate object type for the new clique + type to use as a starting point; + +- *link\_types* – a list of links that constitute the new clique type. + +Creating new event handlers +--------------------------- + +There are three steps to creating a new event handler: + +1. Determining *event types* that will be handled by the new handler; + +2. Writing the new handler module and class; + +3. Adding the (event type -> handler) mapping to the event handlers + configuration file. + +Writing custom handler classes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Each event handler should adhere to the following design: + +1. Event handler class should subclass + the app.discover.events.event\_base.EventBase class; + +2. Event handler class should override handle method of EventBase. + Business logic of the event handler should be placed inside + the handle method; + + a. Handle method accepts two arguments: environment name (str) and + notification contents (dict). No other event data will be provided + to the method; + + b. Handle method returns an EventResult object, which accepts the + following arguments in its constructor: + + i. *result* (mandatory) - determines whether the event handling + was successful; + + ii. *retry* (optional) - determines whether the message should be + put back in the queue in order to be processed later. This + argument is checked only if result was set to False; + + iii. *message* (optional) - (Currently unused) a string comment on + handling status; + + iv. *related\_object* (optional) – id of the object related to + the handled event; + + v. *display\_context* (optional) – (Calipso UI requirement). + +3. Module containing event handler class should have the same name as + the relevant handler class except translated + from UpperCamelCase to snake\_case. + + ***Example:*** + + **app/discover/events/event\_new\_object\_add.py** + + | **from** discover.events.event\_base **import** EventBase, + EventResult + | **class** EventNewObjectAdd(EventBase): + | **def** handle(self, env: str, notification: dict) -> EventResult: + | obj\_id = notification[**'payload'**][**'new\_object'**][**'id'**] + | obj = { + | **'id'**: obj\_id, + | **'type'**: **'new\_object' + **} + | self.inv.set(obj) + | **return** EventResult(result=\ **True**) + + Modifications in *events.json*: + + <...> + + | **"event\_handlers"**: { + | <...> + | **"new\_object.create"**: **"EventNewObjectAdd"**, + | <...>** + **} + + <...> + +After these changes are implemented, any event of type +new\_object.create will be consumed by the event listener and the +payload will be passed to EventNewObjectAdd handler which will insert a +new document in the database. + +Event handlers configuration file structure +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Events.json** (full path *app/config/events.json*) is a configuration +file that contains information about events and event handlers, +including: + +- Event subscription details (queues and exchanges for Neutron + notifications); + +- Location of event handlers package; + +- Mappings between event types and respective event handlers. + +The structure of *events.json* is as following: + + | { + | **"handlers\_package"**: **"discover.events"**, + | **"queues"**: [ + | { + | **"queue"**: **"notifications.nova"**, + | **"exchange"**: **"nova" + **}, + | <…> + | ], + | **"event\_handlers"**: { + | **"compute.instance.create.end"**: **"EventInstanceAdd"**, + | **"compute.instance.update"**: **"EventInstanceUpdate"**, + | **"compute.instance.delete.end"**: **"EventInstanceDelete"**, + | **"network.create.end"**: **"EventNetworkAdd"**, + + | <…>** + **} + | } + +The root object contains the following fields: + +- **handlers\_package** - python path to the package that contains + event handlers (relative to $PYTHONPATH environment variable) + +- **queues –** RabbitMQ queues and exchanges to consume messages from + (for Neutron notifications case) + +- **event\_handlers** – mappings of event types to the respective + handlers. The structure suggests that any event can have only one + handler. + +In order to add a new event handler to the configuration file, you +should add another mapping to the event\_handlers object, where key is +the event type being handled and value is the handler class name (module +name will be determined automatically). + +If your event is being published to a queue and/or exchange that the +listener is not subscribed to, you should add another entry to the +queues list. + +Creating new event listeners +---------------------------- + +At the moment, the only guideline for creation of new event listeners is +that they should subclass the *ListenerBase* class (full path +*app/discover/events/listeners/listener\_base.py*) and override the +*listen(self)* method that listens to incoming events indefinitely +(until terminated by a signal). + +In future versions, a comprehensive guide to listeners structure is +planned. + +Metadata parsers +---------------- + +Metadata parsers are specialized classes that are designed to verify +metadata files (found in *app/*\ config directory), use data from them +to load instances of implementation classes (e.g. scanners, event +handlers, link finders) in memory, and supply them by request. Scanners +and link finders configuration files are used in scanner, event handlers +configuration file – in event listener. + +In order to create a new metadata parser, you should consider +subclassing *MetadataParser* class (found in +*app/utils/metadata\_parser.py*). *MetadataParser* supports parsing and +validating of json files out of the box. Entry point for the class is +the *parse\_metadata\_file* method, which requires the abstract +*get\_required\_fields* method to be overridden in subclasses. This +method should return a list of keys that the metadata file is required +to contain. + +For different levels of customization you may consider: + +1. Overriding *validate\_metadata* method to provide more precise + validation of metadata; + +2. Overriding *parse\_metadata\_file* to provide custom metadata + handling required by your use case. + +.. |image0| image:: media/image1.png + :width: 6.50000in + :height: 4.27153in diff --git a/docs/release/media/image101.png b/docs/release/media/image101.png new file mode 100644 index 0000000..b0a8a5c Binary files /dev/null and b/docs/release/media/image101.png differ diff --git a/docs/release/media/image102.png b/docs/release/media/image102.png new file mode 100644 index 0000000..8c8d413 Binary files /dev/null and b/docs/release/media/image102.png differ diff --git a/docs/release/media/image103.png b/docs/release/media/image103.png new file mode 100644 index 0000000..cc65824 Binary files /dev/null and b/docs/release/media/image103.png differ diff --git a/docs/release/media/image104.png b/docs/release/media/image104.png new file mode 100644 index 0000000..2418dcf Binary files /dev/null and b/docs/release/media/image104.png differ diff --git a/docs/release/media/image105.png b/docs/release/media/image105.png new file mode 100644 index 0000000..1d7fc26 Binary files /dev/null and b/docs/release/media/image105.png differ diff --git a/docs/release/media/image106.png b/docs/release/media/image106.png new file mode 100644 index 0000000..029589a Binary files /dev/null and b/docs/release/media/image106.png differ diff --git a/docs/release/media/image107.png b/docs/release/media/image107.png new file mode 100644 index 0000000..7ac129d Binary files /dev/null and b/docs/release/media/image107.png differ -- cgit 1.2.3-korg