summaryrefslogtreecommitdiffstats
path: root/app/discover/scan.py
blob: 6c40a7f5b812fc7f6de1623c3d8c596b6c984a62 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
#!/usr/bin/env python3
###############################################################################
# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
# and others                                                                  #
#                                                                             #
# All rights reserved. This program and the accompanying materials            #
# are made available under the terms of the Apache License, Version 2.0       #
# which accompanies this distribution, and is available at                    #
# http://www.apache.org/licenses/LICENSE-2.0                                  #
###############################################################################

# Scan an object and insert/update in the inventory

# phase 2: either scan default environment, or scan specific object

import argparse
import sys

from discover.configuration import Configuration
from discover.fetcher import Fetcher
from discover.scan_error import ScanError
from discover.scanner import Scanner
from monitoring.setup.monitoring_setup_manager import MonitoringSetupManager
from utils.constants import EnvironmentFeatures
from utils.mongo_access import MongoAccess
from utils.exceptions import ScanArgumentsError
from utils.inventory_mgr import InventoryMgr
from utils.ssh_connection import SshConnection
from utils.util import setup_args


class ScanPlan:
    """
    @DynamicAttrs
    """

    # Each tuple of COMMON_ATTRIBUTES consists of:
    # attr_name, arg_name and def_key
    #
    # attr_name - name of class attribute to be set
    # arg_name - corresponding name of argument (equal to attr_name if not set)
    # def_key - corresponding key in DEFAULTS (equal to attr_name if not set)
    COMMON_ATTRIBUTES = (("loglevel",),
                         ("inventory_only",),
                         ("links_only",),
                         ("cliques_only",),
                         ("monitoring_setup_only",),
                         ("clear",),
                         ("clear_all",),
                         ("object_type", "type", "type"),
                         ("env",),
                         ("object_id", "id", "env"),
                         ("parent_id",),
                         ("type_to_scan", "parent_type", "parent_type"),
                         ("id_field",),
                         ("scan_self",),
                         ("child_type", "type", "type"))

    def __init__(self, args=None):
        self.obj = None
        self.scanner_type = None
        self.args = args
        for attribute in self.COMMON_ATTRIBUTES:
            setattr(self, attribute[0], None)

        if isinstance(args, dict):
            self._init_from_dict()
        else:
            self._init_from_args()
        self._validate_args()

    def _validate_args(self):
        errors = []
        if (self.inventory_only and self.links_only) \
                or (self.inventory_only and self.cliques_only) \
                or (self.links_only and self.cliques_only):
            errors.append("Only one of (inventory_only, links_only, "
                          "cliques_only) can be True.")
        if errors:
            raise ScanArgumentsError("\n".join(errors))

    def _set_arg_from_dict(self, attribute_name, arg_name=None,
                           default_key=None):
        default_attr = default_key if default_key else attribute_name
        setattr(self, attribute_name,
                self.args.get(arg_name if arg_name else attribute_name,
                              ScanController.DEFAULTS[default_attr]))

    def _set_arg_from_cmd(self, attribute_name, arg_name=None):
        setattr(self,
                attribute_name,
                getattr(self.args, arg_name if arg_name else attribute_name))

    def _set_arg_from_form(self, attribute_name, arg_name=None,
                           default_key=None):
        default_attr = default_key if default_key else attribute_name
        setattr(self,
                attribute_name,
                self.args.getvalue(arg_name if arg_name else attribute_name,
                                   ScanController.DEFAULTS[default_attr]))

    def _init_from_dict(self):
        for arg in self.COMMON_ATTRIBUTES:
            self._set_arg_from_dict(*arg)
        self.child_id = None

    def _init_from_args(self):
        for arg in self.COMMON_ATTRIBUTES:
            self._set_arg_from_cmd(*arg[:2])
        self.child_id = None


class ScanController(Fetcher):
    DEFAULTS = {
        "env": "",
        "mongo_config": "",
        "type": "",
        "inventory": "inventory",
        "scan_self": False,
        "parent_id": "",
        "parent_type": "",
        "id_field": "id",
        "loglevel": "INFO",
        "inventory_only": False,
        "links_only": False,
        "cliques_only": False,
        "monitoring_setup_only": False,
        "clear": False,
        "clear_all": False
    }

    def __init__(self):
        super().__init__()
        self.conf = None
        self.inv = None

    def get_args(self):
        # try to read scan plan from command line parameters
        parser = argparse.ArgumentParser()
        parser.add_argument("-m", "--mongo_config", nargs="?", type=str,
                            default=self.DEFAULTS["mongo_config"],
                            help="name of config file " +
                                 "with MongoDB server access details")
        parser.add_argument("-e", "--env", nargs="?", type=str,
                            default=self.DEFAULTS["env"],
                            help="name of environment to scan \n"
                                 "(default: " + self.DEFAULTS["env"] + ")")
        parser.add_argument("-t", "--type", nargs="?", type=str,
                            default=self.DEFAULTS["type"],
                            help="type of object to scan \n"
                                 "(default: environment)")
        parser.add_argument("-y", "--inventory", nargs="?", type=str,
                            default=self.DEFAULTS["inventory"],
                            help="name of inventory collection \n"
                                 "(default: 'inventory')")
        parser.add_argument("-s", "--scan_self", action="store_true",
                            help="scan changes to a specific object \n"
                                 "(default: False)")
        parser.add_argument("-i", "--id", nargs="?", type=str,
                            default=self.DEFAULTS["env"],
                            help="ID of object to scan (when scan_self=true)")
        parser.add_argument("-p", "--parent_id", nargs="?", type=str,
                            default=self.DEFAULTS["parent_id"],
                            help="ID of parent object (when scan_self=true)")
        parser.add_argument("-a", "--parent_type", nargs="?", type=str,
                            default=self.DEFAULTS["parent_type"],
                            help="type of parent object (when scan_self=true)")
        parser.add_argument("-f", "--id_field", nargs="?", type=str,
                            default=self.DEFAULTS["id_field"],
                            help="name of ID field (when scan_self=true) \n"
                                 "(default: 'id', use 'name' for projects)")
        parser.add_argument("-l", "--loglevel", nargs="?", type=str,
                            default=self.DEFAULTS["loglevel"],
                            help="logging level \n(default: '{}')"
                                 .format(self.DEFAULTS["loglevel"]))
        parser.add_argument("--clear", action="store_true",
                            help="clear all data related to "
                                 "the specified environment prior to scanning\n"
                                 "(default: False)")
        parser.add_argument("--clear_all", action="store_true",
                            help="clear all data prior to scanning\n"
                                 "(default: False)")
        parser.add_argument("--monitoring_setup_only", action="store_true",
                            help="do only monitoring setup deployment \n"
                                 "(default: False)")

        # At most one of these arguments may be present
        scan_only_group = parser.add_mutually_exclusive_group()
        scan_only_group.add_argument("--inventory_only", action="store_true",
                                     help="do only scan to inventory\n" +
                                          "(default: False)")
        scan_only_group.add_argument("--links_only", action="store_true",
                                     help="do only links creation \n" +
                                          "(default: False)")
        scan_only_group.add_argument("--cliques_only", action="store_true",
                                     help="do only cliques creation \n" +
                                          "(default: False)")

        return parser.parse_args()

    def get_scan_plan(self, args):
        # PyCharm type checker can't reliably check types of document
        # noinspection PyTypeChecker
        return self.prepare_scan_plan(ScanPlan(args))

    def prepare_scan_plan(self, plan):
        # Find out object type if not specified in arguments
        if not plan.object_type:
            if not plan.object_id:
                plan.object_type = "environment"
            else:
                # If we scan a specific object, it has to exist in db
                scanned_object = self.inv.get_by_id(plan.env, plan.object_id)
                if not scanned_object:
                    exc_msg = "No object found with specified id: '{}'" \
                        .format(plan.object_id)
                    raise ScanArgumentsError(exc_msg)
                plan.object_type = scanned_object["type"]
                plan.parent_id = scanned_object["parent_id"]
                plan.type_to_scan = scanned_object["parent_type"]

        class_module = plan.object_type
        if not plan.scan_self:
            plan.scan_self = plan.object_type != "environment"

        plan.object_type = plan.object_type.title().replace("_", "")

        if not plan.scan_self:
            plan.child_type = None
        else:
            plan.child_id = plan.object_id
            plan.object_id = plan.parent_id
            if plan.type_to_scan.endswith("_folder"):
                class_module = plan.child_type + "s_root"
            else:
                class_module = plan.type_to_scan
            plan.object_type = class_module.title().replace("_", "")

        if class_module == "environment":
            plan.obj = {"id": plan.env}
        else:
            # fetch object from inventory
            obj = self.inv.get_by_id(plan.env, plan.object_id)
            if not obj:
                raise ValueError("No match for object ID: {}"
                                 .format(plan.object_id))
            plan.obj = obj

        plan.scanner_type = "Scan" + plan.object_type
        return plan

    def run(self, args: dict = None):
        args = setup_args(args, self.DEFAULTS, self.get_args)
        # After this setup we assume args dictionary has all keys
        # defined in self.DEFAULTS
        self.log.set_loglevel(args['loglevel'])

        try:
            MongoAccess.set_config_file(args['mongo_config'])
            self.inv = InventoryMgr()
            self.inv.log.set_loglevel(args['loglevel'])
            self.inv.set_collections(args['inventory'])
            self.conf = Configuration()
        except FileNotFoundError as e:
            return False, 'Mongo configuration file not found: {}'\
                .format(str(e))

        scan_plan = self.get_scan_plan(args)
        if scan_plan.clear or scan_plan.clear_all:
            self.inv.clear(scan_plan)
        self.conf.log.set_loglevel(scan_plan.loglevel)

        env_name = scan_plan.env
        self.conf.use_env(env_name)

        # generate ScanObject Class and instance.
        scanner = Scanner()
        scanner.log.set_loglevel(args['loglevel'])
        scanner.set_env(env_name)
        scanner.found_errors[env_name] = False

        # decide what scanning operations to do
        inventory_only = scan_plan.inventory_only
        links_only = scan_plan.links_only
        cliques_only = scan_plan.cliques_only
        monitoring_setup_only = scan_plan.monitoring_setup_only
        run_all = False if inventory_only or links_only or cliques_only \
            or monitoring_setup_only else True

        # setup monitoring server
        monitoring = \
            self.inv.is_feature_supported(env_name,
                                          EnvironmentFeatures.MONITORING)
        if monitoring:
            self.inv.monitoring_setup_manager = \
                MonitoringSetupManager(env_name)
            self.inv.monitoring_setup_manager.server_setup()

        # do the actual scanning
        try:
            if inventory_only or run_all:
                scanner.run_scan(
                    scan_plan.scanner_type,
                    scan_plan.obj,
                    scan_plan.id_field,
                    scan_plan.child_id,
                    scan_plan.child_type)
            if links_only or run_all:
                scanner.scan_links()
            if cliques_only or run_all:
                scanner.scan_cliques()
            if monitoring:
                if monitoring_setup_only:
                    self.inv.monitoring_setup_manager.simulate_track_changes()
                if not (inventory_only or links_only or cliques_only):
                    scanner.deploy_monitoring_setup()
        except ScanError as e:
            return False, "scan error: " + str(e)
        SshConnection.disconnect_all()
        status = 'ok' if not scanner.found_errors.get(env_name, False) \
            else 'errors detected'
        self.log.info('Scan completed, status: {}'.format(status))
        return True, status


if __name__ == '__main__':
    scan_manager = ScanController()
    ret, msg = scan_manager.run()
    if not ret:
        scan_manager.log.error(msg)
    sys.exit(0 if ret else 1)