Calipso initial release for OPNFV

Change-Id: I7210c244b0c10fa80bfa8c77cb86c9d6ddf8bc88
Signed-off-by: Yaron Yogev <yaronyogev@gmail.com>

5 files changed, 1186 insertions, 0 deletions

diff --git a/app/test/scan/test_data/__init__.py b/app/test/scan/test_data/__init__.py
new file mode 100644
index 0000000..b0637e9
--- /dev/null
+++ b/app/test/scan/test_data/__init__.py
@@ -0,0 +1,9 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
diff --git a/app/test/scan/test_data/configurations.py b/app/test/scan/test_data/configurations.py
new file mode 100644
index 0000000..da68dd1
--- /dev/null
+++ b/app/test/scan/test_data/configurations.py
@@ -0,0 +1,69 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+CONFIGURATIONS = { 
+    "app_path": "/home/scan/calipso_prod/app/",
+    "scanners_file": "/home/yarony/osdna_dev/app/discover/scanners.json",
+    "configuration": [
+        {
+            "mock": "True",
+            "host": "10.56.20.239",
+            "name": "mysql",
+            "password": "102QreDdiD5sKcvNf9qbHrmr",
+            "port": 3307.0,
+            "user": "root",
+            "schema": "nova"
+        },
+        {
+            "name": "OpenStack",
+            "host": "10.56.20.239",
+            "admin_token": "38MUh19YWcgQQUlk2VEFQ7Ec",
+            "port": "5000",
+            "user": "admin",
+            "pwd": "admin"
+        },
+        {
+            "host": "10.56.20.239",
+            "key": "/Users/ngrandhi/.ssh/id_rsa",
+            "name": "CLI",
+            "pwd": "",
+            "user": "root"
+        },
+        {
+            "name": "AMQP",
+            "host": "10.56.20.239",
+            "port": "5673",
+            "user": "nova",
+            "password": "NF2nSv3SisooxPkCTr8fbfOa"
+        },
+        {
+            "config_folder": "/tmp/sensu_config",
+            "provision": "Deploy",
+            "env_type": "development",
+            "name": "Monitoring",
+            "rabbitmq_port": "5672",
+            "rabbitmq_pass": "osdna",
+            "rabbitmq_user": "sensu",
+            "ssh_port": "20022",
+            "ssh_user": "scan",
+            "ssh_password": "scan",
+            "server_ip": "korlev-osdna-staging1.cisco.com",
+            "server_name": "osdna-sensu",
+            "type": "Sensu"
+        }
+    ],
+    "distribution": "Mirantis-8.0",
+    "last_scanned:": "5/8/16",
+    "name": "Mirantis-Liberty-Nvn",
+    "mechanism_drivers": [
+        "OVS"
+    ],
+    "operational": "yes",
+    "type": "environment"
+}
diff --git a/app/test/scan/test_data/metadata.py b/app/test/scan/test_data/metadata.py
new file mode 100644
index 0000000..ed47c80
--- /dev/null
+++ b/app/test/scan/test_data/metadata.py
@@ -0,0 +1,318 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+METADATA_EMPTY = {}
+
+METADATA_SCANNERS_MISSING = {"scanners_package": "discover.fetchers"}
+
+METADATA_NO_PACKAGE = {
+  "scanners": {}
+}
+
+METADATA_NO_SCANNERS = {
+  "scanners_package": "discover.fetchers"
+}
+
+METADATA_ZERO_SCANNERS = {
+  "scanners_package": "discover.fetchers",
+  "scanners": {}
+}
+
+METADATA_SIMPLE_SCANNER = {
+  "scanners_package": "discover.fetchers",
+  "scanners": {
+    "ScanAggregate": [
+      {
+        "type": "host_ref",
+        "fetcher": "DbFetchAggregateHosts"
+      }
+    ]
+  }
+}
+
+METADATA_SCANNER_UNKNOWN_ATTRIBUTE = {
+  "scanners_package": "discover.fetchers",
+  "scanners": {
+    "ScanAggregate": [
+      {
+        "xyz": "123",
+        "type": "host_ref",
+        "fetcher": "DbFetchAggregateHosts"
+      }
+    ]
+  }
+}
+
+METADATA_SCANNER_NO_TYPE = {
+  "scanners_package": "discover.fetchers",
+  "scanners": {
+    "ScanAggregate": [
+      {
+        "fetcher": "DbFetchAggregateHosts"
+      }
+    ]
+  }
+}
+
+METADATA_SCANNER_NO_FETCHER = {
+  "scanners_package": "discover.fetchers",
+  "scanners": {
+    "ScanAggregate": [
+      {
+        "type": "host_ref"
+      }
+    ]
+  }
+}
+
+METADATA_SCANNER_INCORRECT_TYPE = {
+  "scanners_package": "discover.fetchers",
+  "scanners": {
+    "ScanAggregate": [
+      {
+        "type": "t1",
+        "fetcher": "DbFetchAggregateHosts"
+      }
+    ]
+  }
+}
+
+METADATA_SCANNER_INCORRECT_FETCHER = {
+  "scanners_package": "discover.fetchers",
+  "scanners": {
+    "ScanAggregate": [
+      {
+        "type": "host_ref",
+        "fetcher": "f1"
+      }
+    ]
+  }
+}
+
+METADATA_SCANNER_WITH_CHILD = {
+  "scanners_package": "discover.fetchers",
+  "scanners": {
+    "ScanAggregatesRoot": [
+      {
+        "type": "aggregate",
+        "fetcher": "DbFetchAggregates",
+        "children_scanner": "ScanAggregate"
+      }
+    ],
+    "ScanAggregate": [
+      {
+        "type": "host_ref",
+        "fetcher": "DbFetchAggregateHosts"
+      }
+    ]
+  }
+}
+
+METADATA_SCANNER_WITH_INCORRECT_CHILD = {
+  "scanners_package": "discover.fetchers",
+  "scanners": {
+    "ScanAggregatesRoot": [
+      {
+        "type": "aggregate",
+        "fetcher": "DbFetchAggregates",
+        "children_scanner": 1
+      }
+    ]
+  }
+}
+
+METADATA_SCANNER_WITH_MISSING_CHILD = {
+  "scanners_package": "discover.fetchers",
+  "scanners": {
+    "ScanAggregatesRoot": [
+      {
+        "type": "aggregate",
+        "fetcher": "DbFetchAggregates",
+        "children_scanner": "ScanAggregate"
+      }
+    ]
+  }
+}
+
+METADATA_SCANNER_FETCHER_INVALID_DICT = {
+  "scanners_package": "discover.fetchers",
+  "scanners": {
+    "ScanEnvironment": [
+      {
+        "type": "regions_folder",
+        "fetcher": {
+          "types_name": "regions",
+          "parent_type": "environment"
+        }
+      },
+    ]
+
+  }
+}
+
+METADATA_SCANNER_WITH_FOLDER = {
+  "scanners_package": "discover.fetchers",
+  "scanners": {
+    "ScanEnvironment": [
+      {
+        "type": "regions_folder",
+        "fetcher": {
+          "folder": 1,
+          "types_name": "regions",
+          "parent_type": "environment"
+        }
+      },
+      {
+        "type": "projects_folder",
+        "fetcher": {
+          "folder": 1,
+          "types_name": "projects",
+          "parent_type": "environment"
+        }
+      }
+    ]
+  }
+}
+
+METADATA_SCANNER_WITH_INVALID_CONDITION = {
+  "scanners_package": "discover.fetchers",
+  "scanners": {
+    "ScanHost": [
+      {
+        "type": "pnics_folder",
+        "fetcher": "DbFetchAggregateHosts",
+        "environment_condition": 1
+      }
+    ]
+  }
+}
+
+METADATA_SCANNER_WITH_INVALID_MECHANISM_DRIVER_CONDITION = {
+  "scanners_package": "discover.fetchers",
+  "scanners": {
+    "ScanHost": [
+      {
+        "type": "pnics_folder",
+        "fetcher": {
+          "folder": 1,
+          "types_name": "pnics",
+          "parent_type": "host",
+          "text": "pNICs"
+        },
+        "environment_condition": {
+          "mechanism_drivers": ""
+        }
+      }
+    ]
+  }
+}
+
+METADATA_SCANNER_WITH_INVALID_MECHANISM_DRIVER = {
+  "scanners_package": "discover.fetchers",
+  "scanners": {
+    "ScanHost": [
+      {
+        "type": "pnics_folder",
+        "fetcher": {
+          "folder": 1,
+          "types_name": "pnics",
+          "parent_type": "host",
+          "text": "pNICs"
+        },
+        "environment_condition": {
+          "mechanism_drivers": [ 1, 2]
+        }
+      }
+    ]
+  }
+}
+
+METADATA_SCANNER_WITH_CONDITION = {
+  "scanners_package": "discover.fetchers",
+  "scanners": {
+    "ScanHost": [
+      {
+        "type": "pnics_folder",
+        "fetcher": {
+          "folder": 1,
+          "types_name": "pnics",
+          "parent_type": "host",
+          "text": "pNICs"
+        },
+        "environment_condition": {
+          "mechanism_drivers": [
+            "OVS",
+            "LXB"
+          ]
+        }
+      }
+    ]
+  }
+}
+
+CONSTANTS = {
+  "scan_object_types": {
+      "name": "scan_object_types", 
+      "data": [
+          {
+              "value": "regions_folder", 
+              "label": "regions_folder"
+          },
+          {
+              "value": "pnics_folder", 
+              "label": "pnics_folder"
+          },
+          {
+              "value": "projects_folder",
+              "label": "projects_folder"
+          },
+          {
+              "value": "aggregate", 
+              "label": "aggregate"
+          },
+          {
+              "value": "host", 
+              "label": "host"
+          },
+          {
+              "value": "region", 
+              "label": "region"
+          }, 
+          {
+              "value": "host_ref", 
+              "label": "host_ref"
+          }
+      ]
+  },
+  "mechanism_drivers": { 
+      "data": [
+          {
+              "label": "OVS", 
+              "value": "OVS"
+          }, 
+          {
+              "label": "VPP", 
+              "value": "VPP"
+          }, 
+          {
+              "label": "LXB", 
+              "value": "LXB"
+          }, 
+          {
+              "label": "Arista", 
+              "value": "Arista"
+          }, 
+          {
+              "label": "Nexus", 
+              "value": "Nexus"
+          }
+      ], 
+      "name": "mechanism_drivers"
+  }
+}
diff --git a/app/test/scan/test_data/scan.py b/app/test/scan/test_data/scan.py
new file mode 100644
index 0000000..fa36c3e
--- /dev/null
+++ b/app/test/scan/test_data/scan.py
@@ -0,0 +1,435 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+UNIT_TESTS_ENV = "WebEX-Mirantis@Cisco"
+UNIT_TESTS_INVENTORY = 'unit_tests'
+
+MONGO_CONFIG = 'mongo_config_file.txt'
+
+DEFAULT_ARGUMENTS = {
+    "MONGO_CONFIG": "",
+    "ENV": UNIT_TESTS_ENV,
+    "TYPE": "environment",
+    "INVENTORY": "inventory",
+    "SCAN_SELF": False,
+    "ID": UNIT_TESTS_ENV,
+    "PARENT_ID": "",
+    "PARENT_TYPE": "",
+    "ID_FIELD": "id",
+    "LOGLEVEL": "INFO",
+    "INVENTORY_ONLY": False,
+    "LINKS_ONLY": False,
+    "CLIQUES_ONLY": False,
+    "CLEAR": False
+}
+
+SHORT_FLAGS_ARGUMENTS = {
+    "MONGO_CONFIG": "mongo_config_file",
+    "ENV": UNIT_TESTS_ENV,
+    "TYPE": "project",
+    "INVENTORY": UNIT_TESTS_INVENTORY,
+    "SCAN_SELF": True,
+    "ID": "admin",
+    "PARENT_ID": "RegionOne",
+    "PARENT_TYPE": "Region",
+    "ID_FIELD": "name",
+    "LOGLEVEL": "ERROR"
+}
+
+ARGUMENTS_FULL = {
+    "MONGO_CONFIG": "mongo_config_file",
+    "ENV": UNIT_TESTS_ENV,
+    "TYPE": "project",
+    "INVENTORY": UNIT_TESTS_INVENTORY,
+    "SCAN_SELF": True,
+    "ID": "admin",
+    "PARENT_ID": "RegionOne",
+    "PARENT_TYPE": "Region",
+    "ID_FIELD": "name",
+    "LOGLEVEL": "ERROR",
+    "INVENTORY_ONLY": False,
+    "LINKS_ONLY": False,
+    "CLIQUES_ONLY": False,
+    "CLEAR": True,
+    "CLEAR_ALL": False
+}
+
+ARGUMENTS_FULL_CLEAR_ALL = {
+    "MONGO_CONFIG": "mongo_config_file",
+    "ENV": UNIT_TESTS_ENV,
+    "TYPE": "project",
+    "INVENTORY": UNIT_TESTS_INVENTORY,
+    "SCAN_SELF": True,
+    "ID": "admin",
+    "PARENT_ID": "RegionOne",
+    "PARENT_TYPE": "Region",
+    "ID_FIELD": "name",
+    "LOGLEVEL": "ERROR",
+    "INVENTORY_ONLY": False,
+    "LINKS_ONLY": False,
+    "CLIQUES_ONLY": False,
+    "CLEAR": False,
+    "CLEAR_ALL": True
+}
+
+ARGUMENTS_FULL_INVENTORY_ONLY = {
+    "MONGO_CONFIG": "mongo_config_file",
+    "ENV": UNIT_TESTS_ENV,
+    "TYPE": "project",
+    "INVENTORY": UNIT_TESTS_INVENTORY,
+    "SCAN_SELF": True,
+    "ID": "admin",
+    "PARENT_ID": "RegionOne",
+    "PARENT_TYPE": "Region",
+    "ID_FIELD": "name",
+    "LOGLEVEL": "ERROR",
+    "INVENTORY_ONLY": True,
+    "LINKS_ONLY": False,
+    "CLIQUES_ONLY": False,
+    "CLEAR": True,
+    "CLEAR_ALL": False
+}
+
+ARGUMENTS_FULL_LINKS_ONLY = {
+    "MONGO_CONFIG": "mongo_config_file",
+    "ENV": UNIT_TESTS_ENV,
+    "TYPE": "project",
+    "INVENTORY": UNIT_TESTS_INVENTORY,
+    "SCAN_SELF": True,
+    "ID": "admin",
+    "PARENT_ID": "RegionOne",
+    "PARENT_TYPE": "Region",
+    "ID_FIELD": "name",
+    "LOGLEVEL": "ERROR",
+    "INVENTORY_ONLY": False,
+    "LINKS_ONLY": True,
+    "CLIQUES_ONLY": False,
+    "CLEAR": True,
+    "CLEAR_ALL": False
+}
+
+ARGUMENTS_FULL_CLIQUES_ONLY = {
+    "MONGO_CONFIG": "mongo_config_file",
+    "ENV": UNIT_TESTS_ENV,
+    "TYPE": "project",
+    "INVENTORY": UNIT_TESTS_INVENTORY,
+    "SCAN_SELF": True,
+    "ID": "admin",
+    "PARENT_ID": "RegionOne",
+    "PARENT_TYPE": "Region",
+    "ID_FIELD": "name",
+    "LOGLEVEL": "ERROR",
+    "INVENTORY_ONLY": False,
+    "LINKS_ONLY": False,
+    "CLIQUES_ONLY": True,
+    "CLEAR": True,
+    "CLEAR_ALL": False
+}
+
+FORM = {
+    "loglevel": "INFO",
+    "inventory_only": False,
+    "links_only": False,
+    "cliques_only": False,
+    "clear": True,
+    "type": "region",
+    "env": UNIT_TESTS_ENV,
+    "id": "RegionOne",
+    "parent_id": UNIT_TESTS_ENV + "-regions",
+    "parent_type": "regions_folder",
+    "id_field": "id",
+    "scan_self": False,
+    "child_type": "region",
+    "child_id": None
+}
+
+
+SCAN_ENV_PLAN_TO_BE_PREPARED = {
+    "loglevel": "INFO",
+    "inventory_only": False,
+    "links_only": False,
+    "cliques_only": False,
+    "clear": True,
+    "object_type": "environment",
+    "env": UNIT_TESTS_ENV,
+    "id": "",
+    "parent_id": "",
+    "type_to_scan": "",
+    "id_field": "id",
+    "scan_self": False,
+    "child_type": "environment",
+    "child_id": None
+}
+
+SCAN_ENV_INVENTORY_ONLY_PLAN_TO_BE_PREPARED = {
+    "loglevel": "INFO",
+    "inventory_only": True,
+    "links_only": False,
+    "cliques_only": False,
+    "clear": True,
+    "object_type": "environment",
+    "env": UNIT_TESTS_ENV,
+    "id": '',
+    "parent_id": "",
+    "type_to_scan": "",
+    "id_field": "id",
+    "scan_self": False,
+    "child_type": "environment",
+    "child_id": None
+}
+
+SCAN_ENV_LINKS_ONLY_PLAN_TO_BE_PREPARED = {
+    "loglevel": "INFO",
+    "inventory_only": False,
+    "links_only": True,
+    "cliques_only": False,
+    "clear": True,
+    "object_type": "environment",
+    "env": UNIT_TESTS_ENV,
+    "id": '',
+    "parent_id": "",
+    "type_to_scan": "",
+    "id_field": "id",
+    "scan_self": False,
+    "child_type": "environment",
+    "child_id": None
+}
+
+SCAN_ENV_CLIQUES_ONLY_PLAN_TO_BE_PREPARED = {
+    "loglevel": "INFO",
+    "inventory_only": False,
+    "links_only": False,
+    "cliques_only": True,
+    "clear": True,
+    "object_type": "environment",
+    "env": UNIT_TESTS_ENV,
+    "id": '',
+    "parent_id": "",
+    "type_to_scan": "",
+    "id_field": "id",
+    "scan_self": False,
+    "child_type": "environment",
+    "child_id": None
+}
+
+PREPARED_ENV_PLAN = {
+    'obj': {
+        'id': UNIT_TESTS_ENV
+    },
+    'child_id': None,
+    'environment': UNIT_TESTS_ENV,
+    'inventory_only': False,
+    'clear': True,
+    'links_only': False,
+    'scanner_class': 'ScanEnvironment',
+    'object_type': 'environment',
+    'id': UNIT_TESTS_ENV,
+    'inventory': UNIT_TESTS_INVENTORY,
+    'loglevel': 'INFO',
+    'child_type': None,
+    'type_to_scan': '',
+    'cliques_only': False,
+    'id_field': 'id',
+    'parent_id': '',
+    'scan_self': False,
+    'env': UNIT_TESTS_ENV
+}
+
+SCANNER_CLASS = "ScanEnvironment"
+SCANNER_TYPE_FOR_ENV = "ScanEnvironment"
+OBJ_ID_FOR_ENV = ""
+CHILD_TYPE_FOR_ENV = None
+CHILD_ID_FOR_ENV = None
+
+PREPARED_ENV_INVENTORY_ONLY_PLAN = {
+    'obj': {
+        'id': UNIT_TESTS_ENV
+    },
+    'child_id': None,
+    'clear': True,
+    'inventory_only': True,
+    'links_only': False,
+    'scanner_class': 'ScanEnvironment',
+    'object_type': 'environment',
+    'id': UNIT_TESTS_ENV,
+    'inventory': UNIT_TESTS_INVENTORY,
+    'loglevel': 'INFO',
+    'child_type': None,
+    'type_to_scan': '',
+    'cliques_only': False,
+    'id_field': 'id',
+    'parent_id': '',
+    'scan_self': False,
+    'env': UNIT_TESTS_ENV
+}
+
+PREPARED_ENV_LINKS_ONLY_PLAN = {
+    'obj': {
+        'id': UNIT_TESTS_ENV
+    },
+    'child_id': None,
+    'clear': True,
+    'inventory_only': False,
+    'links_only': True,
+    'cliques_only': False,
+    'scanner_class': 'ScanEnvironment',
+    'object_type': 'environment',
+    'id': UNIT_TESTS_ENV,
+    'inventory': UNIT_TESTS_INVENTORY,
+    'loglevel': 'INFO',
+    'child_type': None,
+    'type_to_scan': '',
+    'id_field': 'id',
+    'parent_id': '',
+    'scan_self': False,
+    'env': UNIT_TESTS_ENV
+}
+
+PREPARED_ENV_CLIQUES_ONLY_PLAN = {
+    'obj': {
+        'id': UNIT_TESTS_ENV
+    },
+    'child_id': None,
+    'clear': True,
+    'inventory_only': False,
+    'links_only': False,
+    'cliques_only': True,
+    'scanner_class': 'ScanEnvironment',
+    'object_type': 'environment',
+    'id': UNIT_TESTS_ENV,
+    'inventory': UNIT_TESTS_INVENTORY,
+    'loglevel': 'INFO',
+    'child_type': None,
+    'type_to_scan': '',
+    'id_field': 'id',
+    'parent_id': '',
+    'scan_self': False,
+    'env': UNIT_TESTS_ENV
+}
+
+SCAN_REGION_FOLDER_PLAN_TO_BE_PREPARED = {
+    "loglevel": "INFO",
+    "inventory_only": False,
+    "links_only": False,
+    "cliques_only": False,
+    "clear": True,
+    "object_type": "regions_folder",
+    "env": UNIT_TESTS_ENV,
+    "id": UNIT_TESTS_ENV + "-regions",
+    "parent_id": UNIT_TESTS_ENV,
+    "parent_type": "environment",
+    "type_to_scan": "regions_folder",
+    "id_field": "id",
+    "scan_self": False,
+    "type": "regions_folder"
+}
+
+SCAN_REGION_PLAN_TO_BE_PREPARED = {
+    "loglevel": "INFO",
+    "inventory_only": False,
+    "links_only": False,
+    "cliques_only": False,
+    "clear": True,
+    "object_type": "region",
+    "env": UNIT_TESTS_ENV,
+    "id": "RegionOne",
+    "parent_id": UNIT_TESTS_ENV + "-regions",
+    "parent_type": "regions_folder",
+    "type_to_scan": "region",
+    "id_field": "id",
+    "scan_self": False,
+    "type": "region",
+}
+
+SCANNER_TYPE_FOR_REGION = "ScanRegionsRoot"
+OBJ_ID_FOR_REGION = UNIT_TESTS_ENV + "-regions"
+CHILD_TYPE_FOR_REGION = "region"
+CHILD_ID_FOR_REGION = "RegionOne"
+
+REGIONS_FOLDER = {
+    "id": OBJ_ID_FOR_REGION,
+    "type": "regions_folder",
+    "parent_type": "environment",
+    "object_name": "Regions",
+    "parent_id": UNIT_TESTS_ENV,
+    "name": "Regions",
+    "create_object": True,
+    "text": "Regions"
+}
+
+SCAN_PROJECT_FOLDER_PLAN_TO_BE_PREPARED = {
+    "loglevel": "INFO",
+    "inventory_only": False,
+    "links_only": False,
+    "cliques_only": False,
+    "clear": True,
+    "object_type": "projects_folder",
+    "env": UNIT_TESTS_ENV,
+    "object_id": UNIT_TESTS_ENV + "-projects",
+    "parent_id": UNIT_TESTS_ENV,
+    "type_to_scan": "project",
+    "id_field": "id",
+    "scan_self": False,
+    "child_type": "regions_folder",
+    "child_id": None
+}
+
+SCANNER_CLASS_FOR_REGION_FOLDER = "ScanEnvironment"
+OBJ_ID_FOR_REGION_FOLDER = UNIT_TESTS_ENV
+CHILD_TYPE_FOR_REGION_FOLDER = "regions_folder"
+CHILD_ID_FOR_REGION_FOLDER = UNIT_TESTS_ENV + "-regions"
+
+DEFAULT_COMMAND_ARGS = ["scanner.py"]
+
+SHORT_COMMAND_ARGS = ["scanner.py", "-m", "mongo_config_file",
+                      "-e", UNIT_TESTS_ENV, "-t", "project",
+                      "-y", UNIT_TESTS_INVENTORY, "-s", "-i", "admin",
+                      "-p", "RegionOne", "-a", "Region", "-f", "name",
+                      "-l", "ERROR"]
+
+LONG_COMMAND_ARGS = [
+    "scanner.py", "--mongo_config", "mongo_config_file",
+    "--env", UNIT_TESTS_ENV, "--type", "project",
+    "--inventory", UNIT_TESTS_INVENTORY, "--scan_self", "--id", "admin",
+    "--parent_id", "RegionOne", "--parent_type", "Region",
+    "--id_field", "name", "--loglevel", "ERROR",
+    "--clear"]
+
+LONG_COMMAND_ARGS_CLEAR_ALL = [
+    "scanner.py", "--mongo_config", "mongo_config_file",
+    "--env", UNIT_TESTS_ENV, "--type", "project",
+    "--inventory", UNIT_TESTS_INVENTORY, "--scan_self", "--id", "admin",
+    "--parent_id", "RegionOne", "--parent_type", "Region",
+    "--id_field", "name", "--loglevel", "ERROR",
+    "--clear_all"]
+
+LONG_COMMAND_ARGS_INVENTORY_ONLY = [
+    "scanner.py", "--mongo_config", "mongo_config_file",
+    "--env", UNIT_TESTS_ENV, "--type", "project",
+    "--inventory", UNIT_TESTS_INVENTORY, "--scan_self", "--id", "admin",
+    "--parent_id", "RegionOne", "--parent_type", "Region",
+    "--id_field", "name", "--loglevel", "ERROR", "--inventory_only",
+    "--clear"]
+
+LONG_COMMAND_ARGS_LINKS_ONLY = [
+    "scanner.py", "--mongo_config", "mongo_config_file",
+    "--env", UNIT_TESTS_ENV, "--type", "project",
+    "--inventory", UNIT_TESTS_INVENTORY, "--scan_self", "--id", "admin",
+    "--parent_id", "RegionOne", "--parent_type", "Region",
+    "--id_field", "name", "--loglevel", "ERROR", "--links_only",
+    "--clear"]
+
+LONG_COMMAND_ARGS_CLIQUES_ONLY = [
+    "scanner.py", "--mongo_config", "mongo_config_file",
+    "--env", UNIT_TESTS_ENV, "--type", "project",
+    "--inventory", UNIT_TESTS_INVENTORY, "--scan_self", "--id", "admin",
+    "--parent_id", "RegionOne", "--parent_type", "Region",
+    "--id_field", "name", "--loglevel", "ERROR", "--cliques_only",
+    "--clear"]
+
diff --git a/app/test/scan/test_data/scanner.py b/app/test/scan/test_data/scanner.py
new file mode 100644
index 0000000..cebeca2
--- /dev/null
+++ b/app/test/scan/test_data/scanner.py
@@ -0,0 +1,355 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+import queue
+from discover.fetchers.folder_fetcher import FolderFetcher
+
+
+SCANNER_TYPE_FOR_ENV = "ScanEnvironment"
+
+METADATA = {
+    "scanners_package": "discover",
+    "scanners": {}
+}
+
+TYPE_TO_FETCH = {
+    "type": "pnic",
+    "fetcher": "CliFetchHostPnicsVpp",
+    "environment_condition": {"mechanism_drivers": "OVS"},
+    "children_scanner": "ScanOteps"
+}
+
+TYPE_TO_FETCH_WITH_WRONG_ENVIRONMENT_CONDITION = {
+    "type": "pnic",
+    "fetcher": "CliFetchHostPnicsVpp",
+    "environment_condition": {"mechanism_drivers": "VPP"},
+    "children_scanner": "ScanOteps"
+}
+
+TYPE_TO_FETCH_WITH_ERROR_VALUE = {
+    "environment_condition": {
+        "distribution": "Mirantis-7.0"
+    }
+}
+
+TYPE_TO_FETCH_WITHOUT_ENV_CON = {
+    "type": "pnic",
+    "fetcher": "CliFetchHostPnicsVpp",
+    "children_scanner": "ScanOteps"
+}
+
+TYPES_TO_FETCH = [
+    {
+        "type": "ports_folder",
+        "fetcher": FolderFetcher("ports", "network")
+    },
+    {
+        "type": "network_services_folder",
+        "fetcher": FolderFetcher("network_services", "network", "Network vServices")
+    }
+]
+
+ID_FIELD = "id"
+
+PROJECT1 = {
+    "object": {
+        "description": "",
+        "enabled": True,
+        "id": "75c0eb79ff4a42b0ae4973c8375ddf40",
+        "name": "OSDNA-project"
+    },
+    "child_id_field": ID_FIELD,
+    "scanner": "ScanProject"
+}
+
+PROJECT2 = {
+    "object": {
+        "description": "admin tenant",
+        "enabled": True,
+        "id": "8c1751e0ce714736a63fee3c776164da",
+        "name": "admin"
+    },
+    "child_id_field": ID_FIELD,
+    "scanner": "ScanProject"
+}
+
+SCAN_QUEUE = queue.Queue()
+SCAN_QUEUE.put(PROJECT1)
+SCAN_QUEUE.put(PROJECT2)
+QUEUE_SIZE = 2
+
+LIMIT_TO_CHILD_TYPE = "ports_folder"
+
+CONFIGURATIONS = {
+    "configuration": [
+        {
+            "mock": "True",
+            "host": "10.56.20.239",
+            "name": "mysql",
+            "password": "102QreDdiD5sKcvNf9qbHrmr",
+            "port": 3307.0,
+            "user": "root",
+            "schema": "nova"
+        },
+        {
+            "name": "OpenStack",
+            "host": "10.56.20.239",
+            "admin_token": "38MUh19YWcgQQUlk2VEFQ7Ec",
+            "port": "5000",
+            "user": "admin",
+            "pwd": "admin"
+        },
+        {
+            "host": "10.56.20.239",
+            "key": "/Users/ngrandhi/.ssh/id_rsa",
+            "name": "CLI",
+            "pwd": "",
+            "user": "root"
+        },
+        {
+            "name": "AMQP",
+            "host": "10.56.20.239",
+            "port": "5673",
+            "user": "nova",
+            "password": "NF2nSv3SisooxPkCTr8fbfOa"
+        }
+    ],
+    "distribution": "Mirantis-8.0",
+    "last_scanned:": "5/8/16",
+    "name": "Mirantis-Liberty-Nvn",
+    "mechanism_drivers": [
+        "OVS"
+    ],
+    "operational": "yes",
+    "type": "environment"
+}
+
+TYPES_TO_FETCHES_FOR_PNIC = {
+    "type": "pnic",
+    "fetcher": "CliFetchHostPnicsVpp",
+    "environment_condition": {"mechanism_drivers": "VPP"},
+    "children_scanner": "ScanOteps"
+}
+
+TYPES_TO_FETCHES_FOR_PNIC_WITHOUT_ENV_CON = {
+    "type": "pnic",
+    "fetcher": "CliFetchHostPnicsVpp",
+    "children_scanner": "ScanOteps"
+}
+
+TYPES_TO_FETCHES_FOR_SCAN_AGGREGATE = [{
+    "type": "host_ref",
+    "fetcher": "DbFetchAggregateHosts"
+}]
+
+
+
+
+# id = 'RegionOne-aggregates'
+# obj = self.inv.get_by_id(self.env, id)
+obj = {'id': 'Mirantis-Liberty-Nvn'}
+id_field = 'id'
+child_id = '',
+child_type = ''
+
+
+child_data = [
+    {
+        'id_path': '/Mirantis-Liberty-Nvn/Mirantis-Liberty-Nvn-regions',
+        'object_name': 'Regions',
+        'parent_id': 'Mirantis-Liberty-Nvn',
+        'environment': 'Mirantis-Liberty-Nvn',
+        'id': 'Mirantis-Liberty-Nvn-regions',
+        'show_in_tree': True,
+        'text': 'Regions',
+        'type': 'regions_folder',
+        'name': 'Regions',
+        'create_object': True,
+        'name_path': '/Mirantis-Liberty-Nvn/Regions',
+        'parent_type': 'environment'
+    }
+]
+
+PARENT = {
+    "environment" : "Mirantis-Liberty-Xiaocong",
+    "id" : "node-6.cisco.com-vservices-dhcps",
+    "name" : "node-6.cisco.com-vservices-dhcps",
+    "object_name" : "DHCP servers",
+    "parent_id" : "node-6.cisco.com-vservices",
+    "parent_type" : "vservices_folder",
+    "show_in_tree" : True,
+    "text" : "DHCP servers",
+    "type" : "vservice_dhcps_folder"
+}
+
+PARENT_WITHOUT_ID = {
+    'id': ''
+}
+
+TYPE_TO_FETCH_FOR_ENVIRONMENT = {
+    "type": "regions_folder",
+    "fetcher": FolderFetcher("regions", "environment"),
+    "children_scanner": "ScanRegionsRoot"
+}
+
+TYPE_TO_FETCH_FOR_ENV_WITHOUT_CHILDREN_FETCHER = {
+    "type": "regions_folder",
+    "fetcher": FolderFetcher("regions", "environment")
+}
+
+DB_RESULTS_WITH_CREATE_OBJECT = [
+    {
+        "name": "Mirantis-Liberty-Xiaocong-regions",
+        "parent_type": "environment",
+        "parent_id": "Mirantis-Liberty-Xiaocong",
+        "text": "Regions",
+        "create_object": True,
+        "type": "regions_folder",
+        "id": "Mirantis-Liberty-Xiaocong-regions"
+    }
+]
+
+DB_RESULTS_WITHOUT_CREATE_OBJECT = [
+    {
+        "name": "Mirantis-Liberty-Xiaocong-regions",
+        "parent_type": "environment",
+        "parent_id": "Mirantis-Liberty-Xiaocong",
+        "text": "Regions",
+        "create_object": False,
+        "type": "regions_folder",
+        "id": "Mirantis-Liberty-Xiaocong-regions"
+    }
+]
+
+DB_RESULTS_WITH_PROJECT = [
+    {
+        "name": "Mirantis-Liberty-Xiaocong-regions",
+        "parent_type": "environment",
+        "parent_id": "Mirantis-Liberty-Xiaocong",
+        "text": "Regions", "create_object": True,
+        "type": "regions_folder",
+        "id": "Mirantis-Liberty-Xiaocong-regions",
+        "in_project-OSDNA-project": "1",
+    }
+]
+
+PROJECT_KEY = "in_project-OSDNA-project"
+
+DB_RESULTS_WITH_MASTER_PARENT_IN_DB = [
+    {
+        "host": "node-6.cisco.com",
+        "id": "qdhcp-413de095-01ed-49dc-aa50-4479f43d390e",
+        "local_service_id": "qdhcp-413de095-01ed-49dc-aa50-4479f43d390e",
+        "master_parent_id": "node-6.cisco.com-vservices",
+        "master_parent_type": "vservices_folder",
+        "name": "dhcp-aiya",
+        "parent_id": "node-6.cisco.com-vservices-dhcps",
+        "parent_text": "DHCP servers",
+        "parent_type": "vservice_dhcps_folder",
+        "service_type": "dhcp"
+    }
+]
+
+DB_RESULTS_WITHOUT_MASTER_PARENT_IN_DB = [
+    {
+        "host": "node-6.cisco.com",
+        "id": "qdhcp-413de095-01ed-49dc-aa50-4479f43d390e",
+        "local_service_id": "qdhcp-413de095-01ed-49dc-aa50-4479f43d390e",
+        "master_parent_id": "node-6.cisco.com-vservices",
+        "master_parent_type": "vservices_folder",
+        "name": "dhcp-aiya",
+        "parent_id": "node-6.cisco.com-vservices-dhcps",
+        "parent_text": "DHCP servers",
+        "parent_type": "vservice_dhcps_folder",
+        "service_type": "dhcp"
+    }
+]
+
+
+DICTIONARY_DB_RESULTS = {
+    "name": "Mirantis-Liberty-Xiaocong-regions",
+    "parent_type": "environment",
+    "parent_id": "Mirantis-Liberty-Xiaocong",
+    "text": "Regions", "create_object": True,
+    "type": "regions_folder",
+    "id": "Mirantis-Liberty-Xiaocong-regions"
+}
+
+MASTER_PARENT = {
+    "create_object" : True,
+    "environment" : "Mirantis-Liberty-Xiaocong",
+    "id" : "node-6.cisco.com-vservices",
+    "id_path" : "/Mirantis-Liberty/Mirantis-Liberty-regions/RegionOne/RegionOne-availability_zones/internal/node-6.cisco.com/node-6.cisco.com-vservices",
+    "name" : "Vservices",
+    "name_path" : "/Mirantis-Liberty/Regions/RegionOne/Availability Zones/internal/node-6.cisco.com/Vservices",
+    "object_name" : "Vservices",
+    "parent_id" : "node-6.cisco.com",
+    "parent_type" : "host",
+    "show_in_tree" : True,
+    "text" : "Vservices",
+    "type" : "vservices_folder"
+}
+
+CONFIGURATIONS_WITHOUT_MECHANISM_DRIVERS = {
+    "configuration": [
+        {
+            "mock": "True",
+            "host": "10.56.20.239",
+            "name": "mysql",
+            "password": "102QreDdiD5sKcvNf9qbHrmr",
+            "port": 3307.0,
+            "user": "root",
+            "schema": "nova"
+        },
+        {
+            "name": "OpenStack",
+            "host": "10.56.20.239",
+            "admin_token": "38MUh19YWcgQQUlk2VEFQ7Ec",
+            "port": "5000",
+            "user": "admin",
+            "pwd": "admin"
+        },
+        {
+            "host": "10.56.20.239",
+            "key": "/Users/ngrandhi/.ssh/id_rsa",
+            "name": "CLI",
+            "pwd": "",
+            "user": "root"
+        },
+        {
+            "name": "AMQP",
+            "host": "10.56.20.239",
+            "port": "5673",
+            "user": "nova",
+            "password": "NF2nSv3SisooxPkCTr8fbfOa"
+        }
+    ],
+    "distribution": "Mirantis-8.0",
+    "last_scanned:": "5/8/16",
+    "name": "Mirantis-Liberty-Nvn",
+    "operational": "yes",
+    "type": "environment"
+}
+
+SCAN_TYPE_RESULTS = [
+    {
+        "description": "",
+        "enabled": True,
+        "id": "75c0eb79ff4a42b0ae4973c8375ddf40",
+        "name": "OSDNA-project"
+    },
+    {
+        "description": "admin tenant",
+        "enabled": True,
+        "id": "8c1751e0ce714736a63fee3c776164da",
+        "name": "admin"
+    }
+]
+
+LIMIT_TO_CHILD_ID = "75c0eb79ff4a42b0ae4973c8375ddf40"