From 7e83d0876ddb84a45e130eeba28bc40ef53c074b Mon Sep 17 00:00:00 2001 From: Yaron Yogev Date: Thu, 27 Jul 2017 09:02:54 +0300 Subject: Calipso initial release for OPNFV Change-Id: I7210c244b0c10fa80bfa8c77cb86c9d6ddf8bc88 Signed-off-by: Yaron Yogev --- app/test/scan/__init__.py | 10 + app/test/scan/config/__init__.py | 9 + app/test/scan/config/test_config.py | 17 ++ app/test/scan/main.py | 17 ++ app/test/scan/mock_module.py | 37 +++ app/test/scan/test_data/__init__.py | 9 + app/test/scan/test_data/configurations.py | 69 +++++ app/test/scan/test_data/metadata.py | 318 +++++++++++++++++++++ app/test/scan/test_data/scan.py | 435 +++++++++++++++++++++++++++++ app/test/scan/test_data/scanner.py | 355 +++++++++++++++++++++++ app/test/scan/test_scan.py | 46 +++ app/test/scan/test_scan_controller.py | 215 ++++++++++++++ app/test/scan/test_scan_metadata_parser.py | 152 ++++++++++ app/test/scan/test_scanner.py | 355 +++++++++++++++++++++++ 14 files changed, 2044 insertions(+) create mode 100644 app/test/scan/__init__.py create mode 100644 app/test/scan/config/__init__.py create mode 100644 app/test/scan/config/test_config.py create mode 100644 app/test/scan/main.py create mode 100644 app/test/scan/mock_module.py create mode 100644 app/test/scan/test_data/__init__.py create mode 100644 app/test/scan/test_data/configurations.py create mode 100644 app/test/scan/test_data/metadata.py create mode 100644 app/test/scan/test_data/scan.py create mode 100644 app/test/scan/test_data/scanner.py create mode 100644 app/test/scan/test_scan.py create mode 100644 app/test/scan/test_scan_controller.py create mode 100644 app/test/scan/test_scan_metadata_parser.py create mode 100644 app/test/scan/test_scanner.py (limited to 'app/test/scan') diff --git a/app/test/scan/__init__.py b/app/test/scan/__init__.py new file mode 100644 index 0000000..1e85a2a --- /dev/null +++ b/app/test/scan/__init__.py @@ -0,0 +1,10 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + diff --git a/app/test/scan/config/__init__.py b/app/test/scan/config/__init__.py new file mode 100644 index 0000000..b0637e9 --- /dev/null +++ b/app/test/scan/config/__init__.py @@ -0,0 +1,9 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### diff --git a/app/test/scan/config/test_config.py b/app/test/scan/config/test_config.py new file mode 100644 index 0000000..176fd48 --- /dev/null +++ b/app/test/scan/config/test_config.py @@ -0,0 +1,17 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +# local config info for test. + + +MONGODB_CONFIG = 'your-mongo-config-path-here' + +ENV_CONFIG = 'your-env-name-here' + +COLLECTION_CONFIG = 'your-inventory-collection-name-here' diff --git a/app/test/scan/main.py b/app/test/scan/main.py new file mode 100644 index 0000000..fb8c4b5 --- /dev/null +++ b/app/test/scan/main.py @@ -0,0 +1,17 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import unittest + +from test.scan.test_scanner import TestScanner +from test.scan.test_scan_controller import TestScanController +from test.scan.test_scan_metadata_parser import TestScanMetadataParser + +if __name__=='__main__': + unittest.main() diff --git a/app/test/scan/mock_module.py b/app/test/scan/mock_module.py new file mode 100644 index 0000000..e7aeb13 --- /dev/null +++ b/app/test/scan/mock_module.py @@ -0,0 +1,37 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +class ScanEnvironment: + + run_scan_count = 0 + scan_links_count = 0 + scan_cliques_count = 0 + result = [] + + def set_result(self, result): + self.result = result + + def run_scan(self, *args): + ScanEnvironment.run_scan_count += 1 + return self.result + + def scan_links(self, *args): + ScanEnvironment.scan_links_count += 1 + + def scan_cliques(self, *args): + ScanEnvironment.scan_cliques_count += 1 + + def set_env(self, env): + pass + + @classmethod + def reset_counts(cls): + cls.run_scan_count = 0 + cls.scan_cliques_count = 0 + cls.scan_links_count = 0 \ No newline at end of file diff --git a/app/test/scan/test_data/__init__.py b/app/test/scan/test_data/__init__.py new file mode 100644 index 0000000..b0637e9 --- /dev/null +++ b/app/test/scan/test_data/__init__.py @@ -0,0 +1,9 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### diff --git a/app/test/scan/test_data/configurations.py b/app/test/scan/test_data/configurations.py new file mode 100644 index 0000000..da68dd1 --- /dev/null +++ b/app/test/scan/test_data/configurations.py @@ -0,0 +1,69 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +CONFIGURATIONS = { + "app_path": "/home/scan/calipso_prod/app/", + "scanners_file": "/home/yarony/osdna_dev/app/discover/scanners.json", + "configuration": [ + { + "mock": "True", + "host": "10.56.20.239", + "name": "mysql", + "password": "102QreDdiD5sKcvNf9qbHrmr", + "port": 3307.0, + "user": "root", + "schema": "nova" + }, + { + "name": "OpenStack", + "host": "10.56.20.239", + "admin_token": "38MUh19YWcgQQUlk2VEFQ7Ec", + "port": "5000", + "user": "admin", + "pwd": "admin" + }, + { + "host": "10.56.20.239", + "key": "/Users/ngrandhi/.ssh/id_rsa", + "name": "CLI", + "pwd": "", + "user": "root" + }, + { + "name": "AMQP", + "host": "10.56.20.239", + "port": "5673", + "user": "nova", + "password": "NF2nSv3SisooxPkCTr8fbfOa" + }, + { + "config_folder": "/tmp/sensu_config", + "provision": "Deploy", + "env_type": "development", + "name": "Monitoring", + "rabbitmq_port": "5672", + "rabbitmq_pass": "osdna", + "rabbitmq_user": "sensu", + "ssh_port": "20022", + "ssh_user": "scan", + "ssh_password": "scan", + "server_ip": "korlev-osdna-staging1.cisco.com", + "server_name": "osdna-sensu", + "type": "Sensu" + } + ], + "distribution": "Mirantis-8.0", + "last_scanned:": "5/8/16", + "name": "Mirantis-Liberty-Nvn", + "mechanism_drivers": [ + "OVS" + ], + "operational": "yes", + "type": "environment" +} diff --git a/app/test/scan/test_data/metadata.py b/app/test/scan/test_data/metadata.py new file mode 100644 index 0000000..ed47c80 --- /dev/null +++ b/app/test/scan/test_data/metadata.py @@ -0,0 +1,318 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +METADATA_EMPTY = {} + +METADATA_SCANNERS_MISSING = {"scanners_package": "discover.fetchers"} + +METADATA_NO_PACKAGE = { + "scanners": {} +} + +METADATA_NO_SCANNERS = { + "scanners_package": "discover.fetchers" +} + +METADATA_ZERO_SCANNERS = { + "scanners_package": "discover.fetchers", + "scanners": {} +} + +METADATA_SIMPLE_SCANNER = { + "scanners_package": "discover.fetchers", + "scanners": { + "ScanAggregate": [ + { + "type": "host_ref", + "fetcher": "DbFetchAggregateHosts" + } + ] + } +} + +METADATA_SCANNER_UNKNOWN_ATTRIBUTE = { + "scanners_package": "discover.fetchers", + "scanners": { + "ScanAggregate": [ + { + "xyz": "123", + "type": "host_ref", + "fetcher": "DbFetchAggregateHosts" + } + ] + } +} + +METADATA_SCANNER_NO_TYPE = { + "scanners_package": "discover.fetchers", + "scanners": { + "ScanAggregate": [ + { + "fetcher": "DbFetchAggregateHosts" + } + ] + } +} + +METADATA_SCANNER_NO_FETCHER = { + "scanners_package": "discover.fetchers", + "scanners": { + "ScanAggregate": [ + { + "type": "host_ref" + } + ] + } +} + +METADATA_SCANNER_INCORRECT_TYPE = { + "scanners_package": "discover.fetchers", + "scanners": { + "ScanAggregate": [ + { + "type": "t1", + "fetcher": "DbFetchAggregateHosts" + } + ] + } +} + +METADATA_SCANNER_INCORRECT_FETCHER = { + "scanners_package": "discover.fetchers", + "scanners": { + "ScanAggregate": [ + { + "type": "host_ref", + "fetcher": "f1" + } + ] + } +} + +METADATA_SCANNER_WITH_CHILD = { + "scanners_package": "discover.fetchers", + "scanners": { + "ScanAggregatesRoot": [ + { + "type": "aggregate", + "fetcher": "DbFetchAggregates", + "children_scanner": "ScanAggregate" + } + ], + "ScanAggregate": [ + { + "type": "host_ref", + "fetcher": "DbFetchAggregateHosts" + } + ] + } +} + +METADATA_SCANNER_WITH_INCORRECT_CHILD = { + "scanners_package": "discover.fetchers", + "scanners": { + "ScanAggregatesRoot": [ + { + "type": "aggregate", + "fetcher": "DbFetchAggregates", + "children_scanner": 1 + } + ] + } +} + +METADATA_SCANNER_WITH_MISSING_CHILD = { + "scanners_package": "discover.fetchers", + "scanners": { + "ScanAggregatesRoot": [ + { + "type": "aggregate", + "fetcher": "DbFetchAggregates", + "children_scanner": "ScanAggregate" + } + ] + } +} + +METADATA_SCANNER_FETCHER_INVALID_DICT = { + "scanners_package": "discover.fetchers", + "scanners": { + "ScanEnvironment": [ + { + "type": "regions_folder", + "fetcher": { + "types_name": "regions", + "parent_type": "environment" + } + }, + ] + + } +} + +METADATA_SCANNER_WITH_FOLDER = { + "scanners_package": "discover.fetchers", + "scanners": { + "ScanEnvironment": [ + { + "type": "regions_folder", + "fetcher": { + "folder": 1, + "types_name": "regions", + "parent_type": "environment" + } + }, + { + "type": "projects_folder", + "fetcher": { + "folder": 1, + "types_name": "projects", + "parent_type": "environment" + } + } + ] + } +} + +METADATA_SCANNER_WITH_INVALID_CONDITION = { + "scanners_package": "discover.fetchers", + "scanners": { + "ScanHost": [ + { + "type": "pnics_folder", + "fetcher": "DbFetchAggregateHosts", + "environment_condition": 1 + } + ] + } +} + +METADATA_SCANNER_WITH_INVALID_MECHANISM_DRIVER_CONDITION = { + "scanners_package": "discover.fetchers", + "scanners": { + "ScanHost": [ + { + "type": "pnics_folder", + "fetcher": { + "folder": 1, + "types_name": "pnics", + "parent_type": "host", + "text": "pNICs" + }, + "environment_condition": { + "mechanism_drivers": "" + } + } + ] + } +} + +METADATA_SCANNER_WITH_INVALID_MECHANISM_DRIVER = { + "scanners_package": "discover.fetchers", + "scanners": { + "ScanHost": [ + { + "type": "pnics_folder", + "fetcher": { + "folder": 1, + "types_name": "pnics", + "parent_type": "host", + "text": "pNICs" + }, + "environment_condition": { + "mechanism_drivers": [ 1, 2] + } + } + ] + } +} + +METADATA_SCANNER_WITH_CONDITION = { + "scanners_package": "discover.fetchers", + "scanners": { + "ScanHost": [ + { + "type": "pnics_folder", + "fetcher": { + "folder": 1, + "types_name": "pnics", + "parent_type": "host", + "text": "pNICs" + }, + "environment_condition": { + "mechanism_drivers": [ + "OVS", + "LXB" + ] + } + } + ] + } +} + +CONSTANTS = { + "scan_object_types": { + "name": "scan_object_types", + "data": [ + { + "value": "regions_folder", + "label": "regions_folder" + }, + { + "value": "pnics_folder", + "label": "pnics_folder" + }, + { + "value": "projects_folder", + "label": "projects_folder" + }, + { + "value": "aggregate", + "label": "aggregate" + }, + { + "value": "host", + "label": "host" + }, + { + "value": "region", + "label": "region" + }, + { + "value": "host_ref", + "label": "host_ref" + } + ] + }, + "mechanism_drivers": { + "data": [ + { + "label": "OVS", + "value": "OVS" + }, + { + "label": "VPP", + "value": "VPP" + }, + { + "label": "LXB", + "value": "LXB" + }, + { + "label": "Arista", + "value": "Arista" + }, + { + "label": "Nexus", + "value": "Nexus" + } + ], + "name": "mechanism_drivers" + } +} diff --git a/app/test/scan/test_data/scan.py b/app/test/scan/test_data/scan.py new file mode 100644 index 0000000..fa36c3e --- /dev/null +++ b/app/test/scan/test_data/scan.py @@ -0,0 +1,435 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +UNIT_TESTS_ENV = "WebEX-Mirantis@Cisco" +UNIT_TESTS_INVENTORY = 'unit_tests' + +MONGO_CONFIG = 'mongo_config_file.txt' + +DEFAULT_ARGUMENTS = { + "MONGO_CONFIG": "", + "ENV": UNIT_TESTS_ENV, + "TYPE": "environment", + "INVENTORY": "inventory", + "SCAN_SELF": False, + "ID": UNIT_TESTS_ENV, + "PARENT_ID": "", + "PARENT_TYPE": "", + "ID_FIELD": "id", + "LOGLEVEL": "INFO", + "INVENTORY_ONLY": False, + "LINKS_ONLY": False, + "CLIQUES_ONLY": False, + "CLEAR": False +} + +SHORT_FLAGS_ARGUMENTS = { + "MONGO_CONFIG": "mongo_config_file", + "ENV": UNIT_TESTS_ENV, + "TYPE": "project", + "INVENTORY": UNIT_TESTS_INVENTORY, + "SCAN_SELF": True, + "ID": "admin", + "PARENT_ID": "RegionOne", + "PARENT_TYPE": "Region", + "ID_FIELD": "name", + "LOGLEVEL": "ERROR" +} + +ARGUMENTS_FULL = { + "MONGO_CONFIG": "mongo_config_file", + "ENV": UNIT_TESTS_ENV, + "TYPE": "project", + "INVENTORY": UNIT_TESTS_INVENTORY, + "SCAN_SELF": True, + "ID": "admin", + "PARENT_ID": "RegionOne", + "PARENT_TYPE": "Region", + "ID_FIELD": "name", + "LOGLEVEL": "ERROR", + "INVENTORY_ONLY": False, + "LINKS_ONLY": False, + "CLIQUES_ONLY": False, + "CLEAR": True, + "CLEAR_ALL": False +} + +ARGUMENTS_FULL_CLEAR_ALL = { + "MONGO_CONFIG": "mongo_config_file", + "ENV": UNIT_TESTS_ENV, + "TYPE": "project", + "INVENTORY": UNIT_TESTS_INVENTORY, + "SCAN_SELF": True, + "ID": "admin", + "PARENT_ID": "RegionOne", + "PARENT_TYPE": "Region", + "ID_FIELD": "name", + "LOGLEVEL": "ERROR", + "INVENTORY_ONLY": False, + "LINKS_ONLY": False, + "CLIQUES_ONLY": False, + "CLEAR": False, + "CLEAR_ALL": True +} + +ARGUMENTS_FULL_INVENTORY_ONLY = { + "MONGO_CONFIG": "mongo_config_file", + "ENV": UNIT_TESTS_ENV, + "TYPE": "project", + "INVENTORY": UNIT_TESTS_INVENTORY, + "SCAN_SELF": True, + "ID": "admin", + "PARENT_ID": "RegionOne", + "PARENT_TYPE": "Region", + "ID_FIELD": "name", + "LOGLEVEL": "ERROR", + "INVENTORY_ONLY": True, + "LINKS_ONLY": False, + "CLIQUES_ONLY": False, + "CLEAR": True, + "CLEAR_ALL": False +} + +ARGUMENTS_FULL_LINKS_ONLY = { + "MONGO_CONFIG": "mongo_config_file", + "ENV": UNIT_TESTS_ENV, + "TYPE": "project", + "INVENTORY": UNIT_TESTS_INVENTORY, + "SCAN_SELF": True, + "ID": "admin", + "PARENT_ID": "RegionOne", + "PARENT_TYPE": "Region", + "ID_FIELD": "name", + "LOGLEVEL": "ERROR", + "INVENTORY_ONLY": False, + "LINKS_ONLY": True, + "CLIQUES_ONLY": False, + "CLEAR": True, + "CLEAR_ALL": False +} + +ARGUMENTS_FULL_CLIQUES_ONLY = { + "MONGO_CONFIG": "mongo_config_file", + "ENV": UNIT_TESTS_ENV, + "TYPE": "project", + "INVENTORY": UNIT_TESTS_INVENTORY, + "SCAN_SELF": True, + "ID": "admin", + "PARENT_ID": "RegionOne", + "PARENT_TYPE": "Region", + "ID_FIELD": "name", + "LOGLEVEL": "ERROR", + "INVENTORY_ONLY": False, + "LINKS_ONLY": False, + "CLIQUES_ONLY": True, + "CLEAR": True, + "CLEAR_ALL": False +} + +FORM = { + "loglevel": "INFO", + "inventory_only": False, + "links_only": False, + "cliques_only": False, + "clear": True, + "type": "region", + "env": UNIT_TESTS_ENV, + "id": "RegionOne", + "parent_id": UNIT_TESTS_ENV + "-regions", + "parent_type": "regions_folder", + "id_field": "id", + "scan_self": False, + "child_type": "region", + "child_id": None +} + + +SCAN_ENV_PLAN_TO_BE_PREPARED = { + "loglevel": "INFO", + "inventory_only": False, + "links_only": False, + "cliques_only": False, + "clear": True, + "object_type": "environment", + "env": UNIT_TESTS_ENV, + "id": "", + "parent_id": "", + "type_to_scan": "", + "id_field": "id", + "scan_self": False, + "child_type": "environment", + "child_id": None +} + +SCAN_ENV_INVENTORY_ONLY_PLAN_TO_BE_PREPARED = { + "loglevel": "INFO", + "inventory_only": True, + "links_only": False, + "cliques_only": False, + "clear": True, + "object_type": "environment", + "env": UNIT_TESTS_ENV, + "id": '', + "parent_id": "", + "type_to_scan": "", + "id_field": "id", + "scan_self": False, + "child_type": "environment", + "child_id": None +} + +SCAN_ENV_LINKS_ONLY_PLAN_TO_BE_PREPARED = { + "loglevel": "INFO", + "inventory_only": False, + "links_only": True, + "cliques_only": False, + "clear": True, + "object_type": "environment", + "env": UNIT_TESTS_ENV, + "id": '', + "parent_id": "", + "type_to_scan": "", + "id_field": "id", + "scan_self": False, + "child_type": "environment", + "child_id": None +} + +SCAN_ENV_CLIQUES_ONLY_PLAN_TO_BE_PREPARED = { + "loglevel": "INFO", + "inventory_only": False, + "links_only": False, + "cliques_only": True, + "clear": True, + "object_type": "environment", + "env": UNIT_TESTS_ENV, + "id": '', + "parent_id": "", + "type_to_scan": "", + "id_field": "id", + "scan_self": False, + "child_type": "environment", + "child_id": None +} + +PREPARED_ENV_PLAN = { + 'obj': { + 'id': UNIT_TESTS_ENV + }, + 'child_id': None, + 'environment': UNIT_TESTS_ENV, + 'inventory_only': False, + 'clear': True, + 'links_only': False, + 'scanner_class': 'ScanEnvironment', + 'object_type': 'environment', + 'id': UNIT_TESTS_ENV, + 'inventory': UNIT_TESTS_INVENTORY, + 'loglevel': 'INFO', + 'child_type': None, + 'type_to_scan': '', + 'cliques_only': False, + 'id_field': 'id', + 'parent_id': '', + 'scan_self': False, + 'env': UNIT_TESTS_ENV +} + +SCANNER_CLASS = "ScanEnvironment" +SCANNER_TYPE_FOR_ENV = "ScanEnvironment" +OBJ_ID_FOR_ENV = "" +CHILD_TYPE_FOR_ENV = None +CHILD_ID_FOR_ENV = None + +PREPARED_ENV_INVENTORY_ONLY_PLAN = { + 'obj': { + 'id': UNIT_TESTS_ENV + }, + 'child_id': None, + 'clear': True, + 'inventory_only': True, + 'links_only': False, + 'scanner_class': 'ScanEnvironment', + 'object_type': 'environment', + 'id': UNIT_TESTS_ENV, + 'inventory': UNIT_TESTS_INVENTORY, + 'loglevel': 'INFO', + 'child_type': None, + 'type_to_scan': '', + 'cliques_only': False, + 'id_field': 'id', + 'parent_id': '', + 'scan_self': False, + 'env': UNIT_TESTS_ENV +} + +PREPARED_ENV_LINKS_ONLY_PLAN = { + 'obj': { + 'id': UNIT_TESTS_ENV + }, + 'child_id': None, + 'clear': True, + 'inventory_only': False, + 'links_only': True, + 'cliques_only': False, + 'scanner_class': 'ScanEnvironment', + 'object_type': 'environment', + 'id': UNIT_TESTS_ENV, + 'inventory': UNIT_TESTS_INVENTORY, + 'loglevel': 'INFO', + 'child_type': None, + 'type_to_scan': '', + 'id_field': 'id', + 'parent_id': '', + 'scan_self': False, + 'env': UNIT_TESTS_ENV +} + +PREPARED_ENV_CLIQUES_ONLY_PLAN = { + 'obj': { + 'id': UNIT_TESTS_ENV + }, + 'child_id': None, + 'clear': True, + 'inventory_only': False, + 'links_only': False, + 'cliques_only': True, + 'scanner_class': 'ScanEnvironment', + 'object_type': 'environment', + 'id': UNIT_TESTS_ENV, + 'inventory': UNIT_TESTS_INVENTORY, + 'loglevel': 'INFO', + 'child_type': None, + 'type_to_scan': '', + 'id_field': 'id', + 'parent_id': '', + 'scan_self': False, + 'env': UNIT_TESTS_ENV +} + +SCAN_REGION_FOLDER_PLAN_TO_BE_PREPARED = { + "loglevel": "INFO", + "inventory_only": False, + "links_only": False, + "cliques_only": False, + "clear": True, + "object_type": "regions_folder", + "env": UNIT_TESTS_ENV, + "id": UNIT_TESTS_ENV + "-regions", + "parent_id": UNIT_TESTS_ENV, + "parent_type": "environment", + "type_to_scan": "regions_folder", + "id_field": "id", + "scan_self": False, + "type": "regions_folder" +} + +SCAN_REGION_PLAN_TO_BE_PREPARED = { + "loglevel": "INFO", + "inventory_only": False, + "links_only": False, + "cliques_only": False, + "clear": True, + "object_type": "region", + "env": UNIT_TESTS_ENV, + "id": "RegionOne", + "parent_id": UNIT_TESTS_ENV + "-regions", + "parent_type": "regions_folder", + "type_to_scan": "region", + "id_field": "id", + "scan_self": False, + "type": "region", +} + +SCANNER_TYPE_FOR_REGION = "ScanRegionsRoot" +OBJ_ID_FOR_REGION = UNIT_TESTS_ENV + "-regions" +CHILD_TYPE_FOR_REGION = "region" +CHILD_ID_FOR_REGION = "RegionOne" + +REGIONS_FOLDER = { + "id": OBJ_ID_FOR_REGION, + "type": "regions_folder", + "parent_type": "environment", + "object_name": "Regions", + "parent_id": UNIT_TESTS_ENV, + "name": "Regions", + "create_object": True, + "text": "Regions" +} + +SCAN_PROJECT_FOLDER_PLAN_TO_BE_PREPARED = { + "loglevel": "INFO", + "inventory_only": False, + "links_only": False, + "cliques_only": False, + "clear": True, + "object_type": "projects_folder", + "env": UNIT_TESTS_ENV, + "object_id": UNIT_TESTS_ENV + "-projects", + "parent_id": UNIT_TESTS_ENV, + "type_to_scan": "project", + "id_field": "id", + "scan_self": False, + "child_type": "regions_folder", + "child_id": None +} + +SCANNER_CLASS_FOR_REGION_FOLDER = "ScanEnvironment" +OBJ_ID_FOR_REGION_FOLDER = UNIT_TESTS_ENV +CHILD_TYPE_FOR_REGION_FOLDER = "regions_folder" +CHILD_ID_FOR_REGION_FOLDER = UNIT_TESTS_ENV + "-regions" + +DEFAULT_COMMAND_ARGS = ["scanner.py"] + +SHORT_COMMAND_ARGS = ["scanner.py", "-m", "mongo_config_file", + "-e", UNIT_TESTS_ENV, "-t", "project", + "-y", UNIT_TESTS_INVENTORY, "-s", "-i", "admin", + "-p", "RegionOne", "-a", "Region", "-f", "name", + "-l", "ERROR"] + +LONG_COMMAND_ARGS = [ + "scanner.py", "--mongo_config", "mongo_config_file", + "--env", UNIT_TESTS_ENV, "--type", "project", + "--inventory", UNIT_TESTS_INVENTORY, "--scan_self", "--id", "admin", + "--parent_id", "RegionOne", "--parent_type", "Region", + "--id_field", "name", "--loglevel", "ERROR", + "--clear"] + +LONG_COMMAND_ARGS_CLEAR_ALL = [ + "scanner.py", "--mongo_config", "mongo_config_file", + "--env", UNIT_TESTS_ENV, "--type", "project", + "--inventory", UNIT_TESTS_INVENTORY, "--scan_self", "--id", "admin", + "--parent_id", "RegionOne", "--parent_type", "Region", + "--id_field", "name", "--loglevel", "ERROR", + "--clear_all"] + +LONG_COMMAND_ARGS_INVENTORY_ONLY = [ + "scanner.py", "--mongo_config", "mongo_config_file", + "--env", UNIT_TESTS_ENV, "--type", "project", + "--inventory", UNIT_TESTS_INVENTORY, "--scan_self", "--id", "admin", + "--parent_id", "RegionOne", "--parent_type", "Region", + "--id_field", "name", "--loglevel", "ERROR", "--inventory_only", + "--clear"] + +LONG_COMMAND_ARGS_LINKS_ONLY = [ + "scanner.py", "--mongo_config", "mongo_config_file", + "--env", UNIT_TESTS_ENV, "--type", "project", + "--inventory", UNIT_TESTS_INVENTORY, "--scan_self", "--id", "admin", + "--parent_id", "RegionOne", "--parent_type", "Region", + "--id_field", "name", "--loglevel", "ERROR", "--links_only", + "--clear"] + +LONG_COMMAND_ARGS_CLIQUES_ONLY = [ + "scanner.py", "--mongo_config", "mongo_config_file", + "--env", UNIT_TESTS_ENV, "--type", "project", + "--inventory", UNIT_TESTS_INVENTORY, "--scan_self", "--id", "admin", + "--parent_id", "RegionOne", "--parent_type", "Region", + "--id_field", "name", "--loglevel", "ERROR", "--cliques_only", + "--clear"] + diff --git a/app/test/scan/test_data/scanner.py b/app/test/scan/test_data/scanner.py new file mode 100644 index 0000000..cebeca2 --- /dev/null +++ b/app/test/scan/test_data/scanner.py @@ -0,0 +1,355 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import queue +from discover.fetchers.folder_fetcher import FolderFetcher + + +SCANNER_TYPE_FOR_ENV = "ScanEnvironment" + +METADATA = { + "scanners_package": "discover", + "scanners": {} +} + +TYPE_TO_FETCH = { + "type": "pnic", + "fetcher": "CliFetchHostPnicsVpp", + "environment_condition": {"mechanism_drivers": "OVS"}, + "children_scanner": "ScanOteps" +} + +TYPE_TO_FETCH_WITH_WRONG_ENVIRONMENT_CONDITION = { + "type": "pnic", + "fetcher": "CliFetchHostPnicsVpp", + "environment_condition": {"mechanism_drivers": "VPP"}, + "children_scanner": "ScanOteps" +} + +TYPE_TO_FETCH_WITH_ERROR_VALUE = { + "environment_condition": { + "distribution": "Mirantis-7.0" + } +} + +TYPE_TO_FETCH_WITHOUT_ENV_CON = { + "type": "pnic", + "fetcher": "CliFetchHostPnicsVpp", + "children_scanner": "ScanOteps" +} + +TYPES_TO_FETCH = [ + { + "type": "ports_folder", + "fetcher": FolderFetcher("ports", "network") + }, + { + "type": "network_services_folder", + "fetcher": FolderFetcher("network_services", "network", "Network vServices") + } +] + +ID_FIELD = "id" + +PROJECT1 = { + "object": { + "description": "", + "enabled": True, + "id": "75c0eb79ff4a42b0ae4973c8375ddf40", + "name": "OSDNA-project" + }, + "child_id_field": ID_FIELD, + "scanner": "ScanProject" +} + +PROJECT2 = { + "object": { + "description": "admin tenant", + "enabled": True, + "id": "8c1751e0ce714736a63fee3c776164da", + "name": "admin" + }, + "child_id_field": ID_FIELD, + "scanner": "ScanProject" +} + +SCAN_QUEUE = queue.Queue() +SCAN_QUEUE.put(PROJECT1) +SCAN_QUEUE.put(PROJECT2) +QUEUE_SIZE = 2 + +LIMIT_TO_CHILD_TYPE = "ports_folder" + +CONFIGURATIONS = { + "configuration": [ + { + "mock": "True", + "host": "10.56.20.239", + "name": "mysql", + "password": "102QreDdiD5sKcvNf9qbHrmr", + "port": 3307.0, + "user": "root", + "schema": "nova" + }, + { + "name": "OpenStack", + "host": "10.56.20.239", + "admin_token": "38MUh19YWcgQQUlk2VEFQ7Ec", + "port": "5000", + "user": "admin", + "pwd": "admin" + }, + { + "host": "10.56.20.239", + "key": "/Users/ngrandhi/.ssh/id_rsa", + "name": "CLI", + "pwd": "", + "user": "root" + }, + { + "name": "AMQP", + "host": "10.56.20.239", + "port": "5673", + "user": "nova", + "password": "NF2nSv3SisooxPkCTr8fbfOa" + } + ], + "distribution": "Mirantis-8.0", + "last_scanned:": "5/8/16", + "name": "Mirantis-Liberty-Nvn", + "mechanism_drivers": [ + "OVS" + ], + "operational": "yes", + "type": "environment" +} + +TYPES_TO_FETCHES_FOR_PNIC = { + "type": "pnic", + "fetcher": "CliFetchHostPnicsVpp", + "environment_condition": {"mechanism_drivers": "VPP"}, + "children_scanner": "ScanOteps" +} + +TYPES_TO_FETCHES_FOR_PNIC_WITHOUT_ENV_CON = { + "type": "pnic", + "fetcher": "CliFetchHostPnicsVpp", + "children_scanner": "ScanOteps" +} + +TYPES_TO_FETCHES_FOR_SCAN_AGGREGATE = [{ + "type": "host_ref", + "fetcher": "DbFetchAggregateHosts" +}] + + + + +# id = 'RegionOne-aggregates' +# obj = self.inv.get_by_id(self.env, id) +obj = {'id': 'Mirantis-Liberty-Nvn'} +id_field = 'id' +child_id = '', +child_type = '' + + +child_data = [ + { + 'id_path': '/Mirantis-Liberty-Nvn/Mirantis-Liberty-Nvn-regions', + 'object_name': 'Regions', + 'parent_id': 'Mirantis-Liberty-Nvn', + 'environment': 'Mirantis-Liberty-Nvn', + 'id': 'Mirantis-Liberty-Nvn-regions', + 'show_in_tree': True, + 'text': 'Regions', + 'type': 'regions_folder', + 'name': 'Regions', + 'create_object': True, + 'name_path': '/Mirantis-Liberty-Nvn/Regions', + 'parent_type': 'environment' + } +] + +PARENT = { + "environment" : "Mirantis-Liberty-Xiaocong", + "id" : "node-6.cisco.com-vservices-dhcps", + "name" : "node-6.cisco.com-vservices-dhcps", + "object_name" : "DHCP servers", + "parent_id" : "node-6.cisco.com-vservices", + "parent_type" : "vservices_folder", + "show_in_tree" : True, + "text" : "DHCP servers", + "type" : "vservice_dhcps_folder" +} + +PARENT_WITHOUT_ID = { + 'id': '' +} + +TYPE_TO_FETCH_FOR_ENVIRONMENT = { + "type": "regions_folder", + "fetcher": FolderFetcher("regions", "environment"), + "children_scanner": "ScanRegionsRoot" +} + +TYPE_TO_FETCH_FOR_ENV_WITHOUT_CHILDREN_FETCHER = { + "type": "regions_folder", + "fetcher": FolderFetcher("regions", "environment") +} + +DB_RESULTS_WITH_CREATE_OBJECT = [ + { + "name": "Mirantis-Liberty-Xiaocong-regions", + "parent_type": "environment", + "parent_id": "Mirantis-Liberty-Xiaocong", + "text": "Regions", + "create_object": True, + "type": "regions_folder", + "id": "Mirantis-Liberty-Xiaocong-regions" + } +] + +DB_RESULTS_WITHOUT_CREATE_OBJECT = [ + { + "name": "Mirantis-Liberty-Xiaocong-regions", + "parent_type": "environment", + "parent_id": "Mirantis-Liberty-Xiaocong", + "text": "Regions", + "create_object": False, + "type": "regions_folder", + "id": "Mirantis-Liberty-Xiaocong-regions" + } +] + +DB_RESULTS_WITH_PROJECT = [ + { + "name": "Mirantis-Liberty-Xiaocong-regions", + "parent_type": "environment", + "parent_id": "Mirantis-Liberty-Xiaocong", + "text": "Regions", "create_object": True, + "type": "regions_folder", + "id": "Mirantis-Liberty-Xiaocong-regions", + "in_project-OSDNA-project": "1", + } +] + +PROJECT_KEY = "in_project-OSDNA-project" + +DB_RESULTS_WITH_MASTER_PARENT_IN_DB = [ + { + "host": "node-6.cisco.com", + "id": "qdhcp-413de095-01ed-49dc-aa50-4479f43d390e", + "local_service_id": "qdhcp-413de095-01ed-49dc-aa50-4479f43d390e", + "master_parent_id": "node-6.cisco.com-vservices", + "master_parent_type": "vservices_folder", + "name": "dhcp-aiya", + "parent_id": "node-6.cisco.com-vservices-dhcps", + "parent_text": "DHCP servers", + "parent_type": "vservice_dhcps_folder", + "service_type": "dhcp" + } +] + +DB_RESULTS_WITHOUT_MASTER_PARENT_IN_DB = [ + { + "host": "node-6.cisco.com", + "id": "qdhcp-413de095-01ed-49dc-aa50-4479f43d390e", + "local_service_id": "qdhcp-413de095-01ed-49dc-aa50-4479f43d390e", + "master_parent_id": "node-6.cisco.com-vservices", + "master_parent_type": "vservices_folder", + "name": "dhcp-aiya", + "parent_id": "node-6.cisco.com-vservices-dhcps", + "parent_text": "DHCP servers", + "parent_type": "vservice_dhcps_folder", + "service_type": "dhcp" + } +] + + +DICTIONARY_DB_RESULTS = { + "name": "Mirantis-Liberty-Xiaocong-regions", + "parent_type": "environment", + "parent_id": "Mirantis-Liberty-Xiaocong", + "text": "Regions", "create_object": True, + "type": "regions_folder", + "id": "Mirantis-Liberty-Xiaocong-regions" +} + +MASTER_PARENT = { + "create_object" : True, + "environment" : "Mirantis-Liberty-Xiaocong", + "id" : "node-6.cisco.com-vservices", + "id_path" : "/Mirantis-Liberty/Mirantis-Liberty-regions/RegionOne/RegionOne-availability_zones/internal/node-6.cisco.com/node-6.cisco.com-vservices", + "name" : "Vservices", + "name_path" : "/Mirantis-Liberty/Regions/RegionOne/Availability Zones/internal/node-6.cisco.com/Vservices", + "object_name" : "Vservices", + "parent_id" : "node-6.cisco.com", + "parent_type" : "host", + "show_in_tree" : True, + "text" : "Vservices", + "type" : "vservices_folder" +} + +CONFIGURATIONS_WITHOUT_MECHANISM_DRIVERS = { + "configuration": [ + { + "mock": "True", + "host": "10.56.20.239", + "name": "mysql", + "password": "102QreDdiD5sKcvNf9qbHrmr", + "port": 3307.0, + "user": "root", + "schema": "nova" + }, + { + "name": "OpenStack", + "host": "10.56.20.239", + "admin_token": "38MUh19YWcgQQUlk2VEFQ7Ec", + "port": "5000", + "user": "admin", + "pwd": "admin" + }, + { + "host": "10.56.20.239", + "key": "/Users/ngrandhi/.ssh/id_rsa", + "name": "CLI", + "pwd": "", + "user": "root" + }, + { + "name": "AMQP", + "host": "10.56.20.239", + "port": "5673", + "user": "nova", + "password": "NF2nSv3SisooxPkCTr8fbfOa" + } + ], + "distribution": "Mirantis-8.0", + "last_scanned:": "5/8/16", + "name": "Mirantis-Liberty-Nvn", + "operational": "yes", + "type": "environment" +} + +SCAN_TYPE_RESULTS = [ + { + "description": "", + "enabled": True, + "id": "75c0eb79ff4a42b0ae4973c8375ddf40", + "name": "OSDNA-project" + }, + { + "description": "admin tenant", + "enabled": True, + "id": "8c1751e0ce714736a63fee3c776164da", + "name": "admin" + } +] + +LIMIT_TO_CHILD_ID = "75c0eb79ff4a42b0ae4973c8375ddf40" diff --git a/app/test/scan/test_scan.py b/app/test/scan/test_scan.py new file mode 100644 index 0000000..a01083e --- /dev/null +++ b/app/test/scan/test_scan.py @@ -0,0 +1,46 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import unittest +from unittest.mock import MagicMock + +from discover.configuration import Configuration +from monitoring.setup.monitoring_setup_manager import MonitoringSetupManager +from test.scan.config.test_config \ + import MONGODB_CONFIG, ENV_CONFIG, COLLECTION_CONFIG +from test.scan.test_data.configurations import CONFIGURATIONS +from utils.inventory_mgr import InventoryMgr +from utils.mongo_access import MongoAccess +from utils.logging.full_logger import FullLogger + + +class TestScan(unittest.TestCase): + + def configure_environment(self): + self.env = ENV_CONFIG + self.inventory_collection = COLLECTION_CONFIG + # mock the mongo access + MongoAccess.mongo_connect = MagicMock() + MongoAccess.db = MagicMock() + # mock log + FullLogger.info = MagicMock() + + self.conf = Configuration() + self.conf.use_env = MagicMock() + self.conf.environment = CONFIGURATIONS + self.conf.configuration = CONFIGURATIONS["configuration"] + + self.inv = InventoryMgr() + self.inv.clear = MagicMock() + self.inv.set_collections(self.inventory_collection) + + MonitoringSetupManager.server_setup = MagicMock() + + def setUp(self): + self.configure_environment() diff --git a/app/test/scan/test_scan_controller.py b/app/test/scan/test_scan_controller.py new file mode 100644 index 0000000..f3bcc9a --- /dev/null +++ b/app/test/scan/test_scan_controller.py @@ -0,0 +1,215 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import sys +from unittest.mock import MagicMock + +from discover.scan import ScanController, ScanPlan +from discover.scanner import Scanner +from test.scan.test_scan import TestScan +from test.scan.test_data.scan import * +from utils.inventory_mgr import InventoryMgr + + +class TestScanController(TestScan): + + def setUp(self): + super().setUp() + self.scan_controller = ScanController() + + def arg_validate(self, args, expected, key, err=None): + if key not in expected: + return + err = err if err else 'The value of {} is wrong'.format(key) + self.assertEqual(args.get(key, None), expected[key.upper()], err) + + def check_args_values(self, args, expected): + self.arg_validate(args, expected, 'env', + 'The value of environment is wrong') + keys = ['mongo_config', 'mongo_config', 'type', 'inventory', + 'scan_self', 'id', 'parent_id', 'parent_type', 'id_field', + 'loglevel', 'inventory_only', 'links_only', 'cliques_only', + 'clear'] + for key in keys: + self.arg_validate(args, expected, key) + + def test_get_args_with_default_arguments(self): + sys.argv = DEFAULT_COMMAND_ARGS + args = self.scan_controller.get_args() + # check the default value of each argument + self.check_args_values(args, DEFAULT_ARGUMENTS) + + def test_get_args_with_short_command_args(self): + sys.argv = SHORT_COMMAND_ARGS + args = self.scan_controller.get_args() + # check the value parsed by short arguments + self.check_args_values(args, SHORT_FLAGS_ARGUMENTS) + + def test_get_args_with_full_command_args(self): + sys.argv = LONG_COMMAND_ARGS + args = self.scan_controller.get_args() + # check the value parsed by long arguments + self.check_args_values(args, ARGUMENTS_FULL) + + def test_get_args_with_full_command_args_clear_all(self): + sys.argv = LONG_COMMAND_ARGS_CLEAR_ALL + args = self.scan_controller.get_args() + # check the value parsed by long arguments + self.check_args_values(args, ARGUMENTS_FULL_CLEAR_ALL) + + def test_get_args_with_full_command_args_inventory_only(self): + sys.argv = LONG_COMMAND_ARGS_INVENTORY_ONLY + args = self.scan_controller.get_args() + # check the value parsed by long arguments + self.check_args_values(args, ARGUMENTS_FULL_INVENTORY_ONLY) + + def test_get_args_with_full_command_args_links_only(self): + sys.argv = LONG_COMMAND_ARGS_LINKS_ONLY + args = self.scan_controller.get_args() + # check the value parsed by long arguments + self.check_args_values(args, ARGUMENTS_FULL_LINKS_ONLY) + + def test_get_args_with_full_command_args_cliques_only(self): + sys.argv = LONG_COMMAND_ARGS_CLIQUES_ONLY + args = self.scan_controller.get_args() + # check the value parsed by long arguments + self.check_args_values(args, ARGUMENTS_FULL_CLIQUES_ONLY) + + def side_effect(self, key, default): + if key in FORM.keys(): + return FORM[key] + else: + return default + + def check_plan_values(self, plan, scanner_type, obj_id, + child_type, child_id): + self.assertEqual(scanner_type, plan.scanner_type, + 'The scanner class is wrong') + self.assertEqual(child_type, plan.child_type, + 'The child type is wrong') + self.assertEqual(child_id, plan.child_id, + 'The child id is wrong') + self.assertEqual(obj_id, plan.object_id, 'The object is wrong') + + def test_prepare_scan_plan(self): + scan_plan = ScanPlan(SCAN_ENV_PLAN_TO_BE_PREPARED) + plan = self.scan_controller.prepare_scan_plan(scan_plan) + self.check_plan_values(plan, SCANNER_TYPE_FOR_ENV, + OBJ_ID_FOR_ENV, CHILD_TYPE_FOR_ENV, + CHILD_ID_FOR_ENV) + + def test_prepare_scan_region_plan(self): + original_get_by_id = self.inv.get_by_id + self.inv.get_by_id = MagicMock(return_value=REGIONS_FOLDER) + + self.scan_controller.inv = self.inv + scan_plan = ScanPlan(SCAN_REGION_PLAN_TO_BE_PREPARED) + plan = self.scan_controller.prepare_scan_plan(scan_plan) + + self.check_plan_values(plan, SCANNER_TYPE_FOR_REGION, + OBJ_ID_FOR_REGION, CHILD_TYPE_FOR_REGION, + CHILD_ID_FOR_REGION) + self.inv.get_by_id = original_get_by_id + + def test_prepare_scan_region_folder_plan(self): + scan_plan = ScanPlan(SCAN_REGION_FOLDER_PLAN_TO_BE_PREPARED) + plan = self.scan_controller.prepare_scan_plan(scan_plan) + self.check_plan_values(plan, SCANNER_CLASS_FOR_REGION_FOLDER, + OBJ_ID_FOR_REGION_FOLDER, + CHILD_TYPE_FOR_REGION_FOLDER, + CHILD_ID_FOR_REGION_FOLDER) + + def check_scan_method_calls(self, mock, count): + if count: + self.assertTrue(mock.called) + else: + mock.assert_not_called() + + def check_scan_counts(self, run_scan_count, scan_links_count, + scan_cliques_count, deploy_monitoring_setup_count): + self.check_scan_method_calls(Scanner.scan, run_scan_count) + self.check_scan_method_calls(Scanner.scan_links, scan_links_count) + self.check_scan_method_calls(Scanner.scan_cliques, scan_cliques_count) + self.check_scan_method_calls(Scanner.deploy_monitoring_setup, + deploy_monitoring_setup_count) + + def prepare_scan_mocks(self): + self.load_metadata = Scanner.load_metadata + self.scan = Scanner.scan + self.scan_links = Scanner.scan_links + self.scan_cliques = Scanner.scan_cliques + self.deploy_monitoring_setup = Scanner.deploy_monitoring_setup + + Scanner.load_metadata = MagicMock() + Scanner.scan = MagicMock() + Scanner.scan_links = MagicMock() + Scanner.scan_cliques = MagicMock() + Scanner.deploy_monitoring_setup = MagicMock() + + def reset_methods(self): + Scanner.load_metadata = self.load_metadata + Scanner.scan = self.scan + Scanner.scan_links = self.scan_links + Scanner.scan_cliques = self.scan_cliques + Scanner.deploy_monitoring_setup = self.deploy_monitoring_setup + + def test_scan(self): + self.scan_controller.get_args = MagicMock() + InventoryMgr.is_feature_supported = MagicMock(return_value=False) + plan = self.scan_controller.prepare_scan_plan(ScanPlan(SCAN_ENV_PLAN_TO_BE_PREPARED)) + self.scan_controller.get_scan_plan = MagicMock(return_value=plan) + self.prepare_scan_mocks() + + self.scan_controller.run() + self.check_scan_counts(1, 1, 1, 0) + self.reset_methods() + + def test_scan_with_monitoring_setup(self): + self.scan_controller.get_args = MagicMock() + InventoryMgr.is_feature_supported = MagicMock(return_value=True) + plan = self.scan_controller.prepare_scan_plan(ScanPlan(SCAN_ENV_PLAN_TO_BE_PREPARED)) + self.scan_controller.get_scan_plan = MagicMock(return_value=plan) + self.prepare_scan_mocks() + + self.scan_controller.run() + self.check_scan_counts(1, 1, 1, 1) + self.reset_methods() + + def test_scan_with_inventory_only(self): + self.scan_controller.get_args = MagicMock() + scan_plan = ScanPlan(SCAN_ENV_INVENTORY_ONLY_PLAN_TO_BE_PREPARED) + plan = self.scan_controller.prepare_scan_plan(scan_plan) + self.scan_controller.get_scan_plan = MagicMock(return_value=plan) + self.prepare_scan_mocks() + + self.scan_controller.run() + self.check_scan_counts(1, 0, 0, 0) + self.reset_methods() + + def test_scan_with_links_only(self): + self.scan_controller.get_args = MagicMock() + scan_plan = ScanPlan(SCAN_ENV_LINKS_ONLY_PLAN_TO_BE_PREPARED) + plan = self.scan_controller.prepare_scan_plan(scan_plan) + self.scan_controller.get_scan_plan = MagicMock(return_value=plan) + self.prepare_scan_mocks() + + self.scan_controller.run() + self.check_scan_counts(0, 1, 0, 0) + self.reset_methods() + + def test_scan_with_cliques_only(self): + self.scan_controller.get_args = MagicMock() + scan_plan = ScanPlan(SCAN_ENV_CLIQUES_ONLY_PLAN_TO_BE_PREPARED) + plan = self.scan_controller.prepare_scan_plan(scan_plan) + self.scan_controller.get_scan_plan = MagicMock(return_value=plan) + self.prepare_scan_mocks() + + self.scan_controller.run() + self.check_scan_counts(0, 0, 1, 0) + self.reset_methods() diff --git a/app/test/scan/test_scan_metadata_parser.py b/app/test/scan/test_scan_metadata_parser.py new file mode 100644 index 0000000..91c11ef --- /dev/null +++ b/app/test/scan/test_scan_metadata_parser.py @@ -0,0 +1,152 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.db.db_access import DbAccess +from discover.scan_metadata_parser import ScanMetadataParser +from test.scan.test_scan import TestScan +from test.scan.test_data.metadata import * +from unittest import mock +from utils.mongo_access import MongoAccess + + +SCANNERS_FILE = 'scanners.json' + +JSON_REQUIRED_FIELDS_ERROR = 'Metadata json should contain all the ' + \ + 'following fields: scanners_package, scanners' + +JSON_NO_SCANNERS = 'no scanners found in scanners list' +JSON_ERRORS_FOUND = 'Errors encountered during metadata file parsing:\n' + + +class TestScanMetadataParser(TestScan): + def setUp(self): + super().setUp() + DbAccess.conn = mock.MagicMock() + self.prepare_constants() + self.parser = ScanMetadataParser(self.inv) + + self.parser.check_metadata_file_ok = mock.MagicMock() + + def prepare_metadata(self, content): + self.parser._load_json_file = mock.MagicMock(return_value=content) + + def prepare_constants(self): + MongoAccess.db = mock.MagicMock() + MongoAccess.db["constants"].find_one = mock.MagicMock(side_effect= + lambda input: + CONSTANTS[input["name"]] + if CONSTANTS.get(input["name"]) + else [] + ) + + def handle_error_scenario(self, input_content, expected_error, + add_errors_encountered_pretext=True): + self.prepare_metadata(input_content) + found_exception = False + expected_message = expected_error + metadata = None + try: + metadata = self.parser.parse_metadata_file(SCANNERS_FILE) + except ValueError as e: + found_exception = True + expected_message = expected_error \ + if not add_errors_encountered_pretext \ + else JSON_ERRORS_FOUND + expected_error + self.assertEqual(str(e), expected_message) + self.assertTrue(found_exception, + 'failed to throw exception, expected_message: {}' + .format(expected_message)) + self.assertIsNone(metadata) + + def handle_json_missing_field(self, json_content): + self.handle_error_scenario(json_content, JSON_REQUIRED_FIELDS_ERROR, + add_errors_encountered_pretext=False) + + def test_missing_field(self): + for content in [METADATA_EMPTY, METADATA_NO_PACKAGE, + METADATA_NO_SCANNERS]: + self.handle_json_missing_field(content) + + def test_json_no_scanners(self): + self.handle_error_scenario(METADATA_ZERO_SCANNERS, JSON_NO_SCANNERS) + + def test_json_scanner_errors(self): + errors_scenarios = [ + { + 'input': METADATA_ZERO_SCANNERS, + 'msg': JSON_NO_SCANNERS + }, + { + 'input': METADATA_SCANNER_UNKNOWN_ATTRIBUTE, + 'msg': 'unknown attribute xyz in scanner ScanAggregate, type #1' + }, + { + 'input': METADATA_SCANNER_NO_TYPE, + 'msg': 'scanner ScanAggregate, type #1: ' + + 'missing attribute "type"' + }, + { + 'input': METADATA_SCANNER_NO_FETCHER, + 'msg': 'scanner ScanAggregate, type #1: ' + + 'missing attribute "fetcher"' + }, + { + 'input': METADATA_SCANNER_INCORRECT_TYPE, + 'msg': 'scanner ScanAggregate: value not in types: t1' + }, + { + 'input': METADATA_SCANNER_INCORRECT_FETCHER, + 'msg': 'failed to find fetcher class f1 ' + 'in scanner ScanAggregate type #1' + }, + { + 'input': METADATA_SCANNER_WITH_INCORRECT_CHILD, + 'msg': 'scanner ScanAggregatesRoot type #1: ' + 'children_scanner must be a string' + }, + { + 'input': METADATA_SCANNER_WITH_MISSING_CHILD, + 'msg': 'scanner ScanAggregatesRoot type #1: ' + 'children_scanner ScanAggregate not found ' + }, + { + 'input': METADATA_SCANNER_FETCHER_INVALID_DICT, + 'msg': 'scanner ScanEnvironment type #1: ' + 'only folder dict accepted in fetcher' + }, + { + 'input': METADATA_SCANNER_WITH_INVALID_CONDITION, + 'msg': 'scanner ScanHost type #1: condition must be dict' + } + ] + for scenario in errors_scenarios: + self.handle_error_scenario(scenario['input'], scenario['msg']) + + def check_json_is_ok(self, json_content): + self.prepare_metadata(json_content) + found_exception = False + metadata = None + msg = None + try: + metadata = self.parser.parse_metadata_file(SCANNERS_FILE) + except ValueError as e: + found_exception = True + msg = str(e) + self.assertFalse(found_exception, 'Exception: {}'.format(msg)) + self.assertIsNotNone(metadata) + + def test_json_valid_content(self): + valid_content = [ + METADATA_SIMPLE_SCANNER, + METADATA_SCANNER_WITH_CHILD, + METADATA_SCANNER_WITH_FOLDER, + METADATA_SCANNER_WITH_CONDITION + ] + for content in valid_content: + self.check_json_is_ok(content) diff --git a/app/test/scan/test_scanner.py b/app/test/scan/test_scanner.py new file mode 100644 index 0000000..4a7536e --- /dev/null +++ b/app/test/scan/test_scanner.py @@ -0,0 +1,355 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.scanner import Scanner +from test.scan.test_scan import TestScan +from unittest.mock import MagicMock, patch +from discover.scan_metadata_parser import ScanMetadataParser +from test.scan.test_data.scanner import * +from monitoring.setup.monitoring_setup_manager import MonitoringSetupManager + + +class TestScanner(TestScan): + + def setUp(self): + super().setUp() + ScanMetadataParser.parse_metadata_file = MagicMock(return_value=METADATA) + self.scanner = Scanner() + self.scanner.set_env(self.env) + MonitoringSetupManager.create_setup = MagicMock() + self.scanner.inv.monitoring_setup_manager = \ + MonitoringSetupManager(self.env) + + def test_check_type_env_without_environment_condition(self): + result = self.scanner.check_type_env(TYPE_TO_FETCH_WITHOUT_ENV_CON) + + self.assertEqual(result, True, + "Can't get true when the type_to_fetch " + + "doesn't contain environment condition") + + def test_check_type_with_error_value(self): + # store original method + original_get_env_config = self.scanner.config.get_env_config + + # mock get_env_config method + self.scanner.config.get_env_config =\ + MagicMock(return_value=CONFIGURATIONS) + + result = self.scanner.check_type_env(TYPE_TO_FETCH_WITH_ERROR_VALUE) + + # reset get_env_config method + self.scanner.config.get_env_config = original_get_env_config + + self.assertEqual(result, False, + "Can't get false when the type_to_fetch " + + "contain error value") + + def test_check_type_env_without_mechanism_drivers_in_env_config(self): + # store original method + original_get_env_config = self.scanner.config.get_env_config + + # mock get_env_config_method + self.scanner.config.get_env_config =\ + MagicMock(return_value=CONFIGURATIONS_WITHOUT_MECHANISM_DRIVERS) + + result = self.scanner.check_type_env(TYPE_TO_FETCH) + # reset get_env_config method + self.scanner.check_type_env = original_get_env_config + + self.assertEqual(result, False, + "Can't get false when configuration " + + "doesn't contain mechanism drivers") + + def test_check_type_env_with_wrong_mech_drivers_in_env_condition(self): + # store original method + original_get_env_config = self.scanner.config.get_env_config + + # mock get_env_config_method + self.scanner.config.get_env_config =\ + MagicMock(return_value=CONFIGURATIONS) + + result = self.scanner.\ + check_type_env(TYPE_TO_FETCH_WITH_WRONG_ENVIRONMENT_CONDITION) + # reset get_env_config method + self.scanner.check_type_env = original_get_env_config + + self.assertEqual(result, False, "Can't get false when the mechanism " + + "drivers in type_to_fetch " + + "don't exist in configurations") + + def test_check_type_env(self): + # store original method + original_get_env_config = self.scanner.config.get_env_config + + # mock method + self.scanner.config.get_env_config =\ + MagicMock(return_value=CONFIGURATIONS) + + result = self.scanner.check_type_env(TYPE_TO_FETCH) + + # reset method + self.scanner.config.get_env_config = original_get_env_config + + self.assertEqual(result, True, + "Can't get True when the type_to_fetch is correct") + + def test_scan_error_type(self): + # store original method + original_check_type_env = self.scanner.check_type_env + + # mock method + self.scanner.check_type_env = MagicMock(return_value=False) + + result = self.scanner.scan_type(TYPE_TO_FETCH_FOR_ENVIRONMENT, PARENT, + ID_FIELD) + + # reset method + self.scanner.check_type_env = original_check_type_env + + self.assertEqual(result, [], + "Can't get [], when the type_to_fetch is wrong") + + def test_scan_type_without_parent_id(self): + try: + self.scanner.scan_type(TYPE_TO_FETCH_FOR_ENVIRONMENT, + PARENT_WITHOUT_ID, ID_FIELD) + self.fail("Can't get error when the parent " + + "doesn't contain id attribute") + except: + pass + + @patch("discover.fetchers.folder_fetcher.FolderFetcher.get") + def test_scan_type_with_get_exception(self, fetcher_get): + fetcher_get.side_effect = Exception("get exception") + + try: + self.scanner.scan_type(TYPE_TO_FETCH_FOR_ENVIRONMENT, + PARENT, ID_FIELD) + self.fail("Can't get exception when fetcher.get throws an exception") + except: + pass + + @patch("discover.fetchers.folder_fetcher.FolderFetcher.get") + def test_scan_type_without_master_parent(self, fetcher_get): + fetcher_get.return_value = DB_RESULTS_WITHOUT_MASTER_PARENT_IN_DB + + # store original get_by_id + original_get_by_id = self.scanner.inv.get_by_id + original_set = self.scanner.inv.set + + # mock methods + self.scanner.inv.get_by_id = MagicMock(return_value=[]) + + result = self.scanner.scan_type(TYPE_TO_FETCH_FOR_ENVIRONMENT, PARENT, + ID_FIELD) + + # reset methods + self.scanner.inv.get_by_id = original_get_by_id + self.scanner.inv.set = original_set + self.assertEqual(result, [], "Can't get [], when the master parent " + + "doesn't exist in database") + + @patch("discover.fetchers.folder_fetcher.FolderFetcher.get") + def test_scan_type_with_master_parent(self, fetcher_get): + fetcher_get.return_value = DB_RESULTS_WITH_MASTER_PARENT_IN_DB + + # store original methods + original_get_by_id = self.scanner.inv.get_by_id + original_set = self.scanner.inv.set + + # mock methods + self.scanner.inv.get_by_id = MagicMock(return_value=MASTER_PARENT) + self.scanner.inv.set = MagicMock() + + self.scanner.scan_type(TYPE_TO_FETCH_FOR_ENVIRONMENT, PARENT, ID_FIELD) + self.assertEqual(self.scanner.inv.set.call_count, 2, "Can't create additional folder") + self.assertNotIn("master_parent_type", DB_RESULTS_WITH_MASTER_PARENT_IN_DB, "Can't delete the master_parent_type") + self.assertNotIn("master_parent_id", DB_RESULTS_WITH_MASTER_PARENT_IN_DB, "Can't delete the master_parent_id") + + # reset methods + self.scanner.inv.get_by_id = original_get_by_id + self.scanner.inv.set = original_set + + @patch("discover.fetchers.folder_fetcher.FolderFetcher.get") + def test_scan_type_with_in_project(self, fetcher_get): + fetcher_get.return_value = DB_RESULTS_WITH_PROJECT + + # store original method + original_set = self.scanner.inv.set + + # mock method + self.scanner.inv.set = MagicMock() + + self.scanner.scan_type(TYPE_TO_FETCH_FOR_ENVIRONMENT, PARENT, ID_FIELD) + self.assertIn("projects", DB_RESULTS_WITH_PROJECT[0], + "Can't get the projects from DB result") + self.assertNotIn(PROJECT_KEY, DB_RESULTS_WITH_PROJECT[0], + "Can't delete the project key in the object") + + self.scanner.inv.set = original_set + + @patch("discover.fetchers.folder_fetcher.FolderFetcher.get") + def test_scan_type_without_create_object(self, fetcher_get): + fetcher_get.return_value = DB_RESULTS_WITHOUT_CREATE_OBJECT + + original_set = self.scanner.inv.set + + self.scanner.inv.set = MagicMock() + self.scanner.scan_type(TYPE_TO_FETCH_FOR_ENVIRONMENT, PARENT, ID_FIELD) + + self.assertEqual(self.scanner.inv.set.call_count, 0, + "Set the object when the create object is false") + + self.scanner.inv.set = original_set + + @patch("discover.fetchers.folder_fetcher.FolderFetcher.get") + def test_scan_type_with_create_object(self, fetcher_get): + fetcher_get.return_value = DB_RESULTS_WITH_CREATE_OBJECT + + original_set = self.scanner.inv.set + + self.scanner.inv.set = MagicMock() + self.scanner.scan_type(TYPE_TO_FETCH_FOR_ENVIRONMENT, PARENT, ID_FIELD) + + self.assertEqual(self.scanner.inv.set.call_count, 1, + "Set the object when the create object is false") + + self.scanner.inv.set = original_set + + @patch("discover.fetchers.folder_fetcher.FolderFetcher.get") + def test_scan_type_with_children_scanner(self, fetcher_get): + fetcher_get.return_value = DB_RESULTS_WITH_CREATE_OBJECT + + original_set = self.scanner.inv.set + original_queue_for_scan = self.scanner.queue_for_scan + + self.scanner.inv.set = MagicMock() + self.scanner.queue_for_scan = MagicMock() + + self.scanner.scan_type(TYPE_TO_FETCH_FOR_ENVIRONMENT, PARENT, ID_FIELD) + + self.assertEqual(self.scanner.queue_for_scan.call_count, 1, + "Can't put children scanner in the queue") + + self.scanner.inv.set = original_set + self.scanner.queue_for_scan = original_queue_for_scan + + @patch("discover.fetchers.folder_fetcher.FolderFetcher.get") + def test_scan_type_without_children_scanner(self, fetcher_get): + fetcher_get.return_value = DB_RESULTS_WITH_CREATE_OBJECT + + original_set = self.scanner.inv.set + original_queue_for_scan = self.scanner.queue_for_scan + + self.scanner.inv.set = MagicMock() + self.scanner.queue_for_scan = MagicMock() + + self.scanner.scan_type(TYPE_TO_FETCH_FOR_ENV_WITHOUT_CHILDREN_FETCHER, + PARENT, ID_FIELD) + + self.assertEqual(self.scanner.queue_for_scan.call_count, 0, + "Can't put children scanner in the queue") + + self.scanner.inv.set = original_set + self.scanner.queue_for_scan = original_queue_for_scan + + @patch("discover.fetchers.folder_fetcher.FolderFetcher.get") + def test_scan_type(self, fetcher_get): + fetcher_get.return_value = DB_RESULTS_WITH_CREATE_OBJECT + + original_set = self.scanner.inv.set + original_queue_for_scan = self.scanner.queue_for_scan + + self.scanner.inv.set = MagicMock() + self.scanner.queue_for_scan = MagicMock() + + result = self.scanner.scan_type(TYPE_TO_FETCH_FOR_ENVIRONMENT, PARENT, + ID_FIELD) + + self.assertNotEqual(result, [], "Can't get children form scan_type") + + self.scanner.inv.set = original_set + self.scanner.queue_for_scan = original_queue_for_scan + + def test_scan_with_limit_to_child_type(self): + original_scan_type = self.scanner.scan_type + original_get_scanner = self.scanner.get_scanner + + self.scanner.scan_type = MagicMock(return_value=[]) + self.scanner.get_scanner = MagicMock(return_value=TYPES_TO_FETCH) + + limit_to_child_type = TYPES_TO_FETCH[0]['type'] + + self.scanner.scan(SCANNER_TYPE_FOR_ENV, PARENT, limit_to_child_type=limit_to_child_type) + + # only scan the limit child type + self.scanner.scan_type.assert_called_with(TYPES_TO_FETCH[0], PARENT, + ID_FIELD) + + self.scanner.scan_type = original_scan_type + self.scanner.get_scanner = original_get_scanner + + def test_scan_with_limit_to_child_id(self): + original_scan_type = self.scanner.scan_type + original_get_scanner = self.scanner.get_scanner + + self.scanner.get_scanner = MagicMock(return_value=TYPES_TO_FETCH) + limit_to_child_id = SCAN_TYPE_RESULTS[0][ID_FIELD] + + self.scanner.scan_type = MagicMock(return_value=SCAN_TYPE_RESULTS) + + children = self.scanner.scan(SCANNER_TYPE_FOR_ENV, PARENT, id_field=ID_FIELD, + limit_to_child_id=limit_to_child_id) + + # only get the limit child + self.assertEqual(children, SCAN_TYPE_RESULTS[0]) + + self.scanner.scan_type = original_scan_type + self.scanner.get_scanner = original_get_scanner + + def test_scan(self): + original_scan_type = self.scanner.scan_type + original_get_scanner = self.scanner.get_scanner + + self.scanner.get_scanner = MagicMock(return_values=TYPES_TO_FETCH) + result = self.scanner.scan(SCANNER_TYPE_FOR_ENV, PARENT) + + self.assertEqual(PARENT, result, + "Can't get the original parent after the scan") + + self.scanner.get_scanner = original_get_scanner + self.scanner.scan_type = original_scan_type + + def test_run_scan(self): + original_scan = self.scanner.scan + original_scan_from_queue = self.scanner.scan_from_queue + + self.scanner.scan = MagicMock() + self.scanner.scan_from_queue = MagicMock() + + self.scanner.run_scan(SCANNER_TYPE_FOR_ENV, PARENT, ID_FIELD, LIMIT_TO_CHILD_ID, + LIMIT_TO_CHILD_TYPE) + + self.scanner.scan.assert_called_with(SCANNER_TYPE_FOR_ENV, PARENT, ID_FIELD, + LIMIT_TO_CHILD_ID, + LIMIT_TO_CHILD_TYPE) + self.scanner.scan_from_queue.assert_any_call() + + self.scanner.scan = original_scan + self.scanner.scan_from_queue = original_scan_from_queue + + @patch("discover.scanner.Scanner.scan") + def test_scan_from_queue(self, scan): + scan.return_value = [] + Scanner.scan_queue = SCAN_QUEUE + + self.scanner.scan_from_queue() + + self.assertEqual(self.scanner.scan.call_count, QUEUE_SIZE, + "Can't scan all the objects in the queue") -- cgit 1.2.3-korg