diff options
| author |   Yaron Yogev <yaronyogev@gmail.com> | 2017-07-27 09:02:54 +0300 |
|---|---|---|
| committer | Yaron Yogev <yaronyogev@gmail.com> | 2017-07-27 14:56:25 +0300 |
| commit | 7e83d0876ddb84a45e130eeba28bc40ef53c074b (patch) | |
| tree | 47d76239ae7658d87c66abd142df92709427e7dd /app/api | |
| parent | 378ecbd8947589b9cbb39013a0c2e2aa201e03bd (diff) | |
Calipso initial release for OPNFV
Change-Id: I7210c244b0c10fa80bfa8c77cb86c9d6ddf8bc88
Signed-off-by: Yaron Yogev <yaronyogev@gmail.com>
Diffstat (limited to 'app/api')
32 files changed, 2424 insertions, 0 deletions
diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 0000000..1e85a2a --- /dev/null +++ b/app/api/__init__.py @@ -0,0 +1,10 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + diff --git a/app/api/app.py b/app/api/app.py new file mode 100644 index 0000000..5fa3da9 --- /dev/null +++ b/app/api/app.py @@ -0,0 +1,71 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import importlib + +import falcon + +from api.auth.token import Token +from api.backends.ldap_access import LDAPAccess +from api.exceptions.exceptions import CalipsoApiException +from api.middleware.authentication import AuthenticationMiddleware +from utils.inventory_mgr import InventoryMgr +from utils.logging.full_logger import FullLogger +from utils.mongo_access import MongoAccess + + +class App: + + ROUTE_DECLARATIONS = { + "/inventory": "resource.inventory.Inventory", + "/links": "resource.links.Links", + "/messages": "resource.messages.Messages", + "/cliques": "resource.cliques.Cliques", + "/clique_types": "resource.clique_types.CliqueTypes", + "/clique_constraints": "resource.clique_constraints.CliqueConstraints", + "/scans": "resource.scans.Scans", + "/scheduled_scans": "resource.scheduled_scans.ScheduledScans", + "/constants": "resource.constants.Constants", + "/monitoring_config_templates": + "resource.monitoring_config_templates.MonitoringConfigTemplates", + "/aggregates": "resource.aggregates.Aggregates", + "/environment_configs": + "resource.environment_configs.EnvironmentConfigs", + "/auth/tokens": "auth.tokens.Tokens" + } + + responders_path = "api.responders" + + def __init__(self, mongo_config="", ldap_config="", + log_level="", inventory="", token_lifetime=86400): + MongoAccess.set_config_file(mongo_config) + self.inv = InventoryMgr() + self.inv.set_collections(inventory) + self.log = FullLogger() + self.log.set_loglevel(log_level) + self.ldap_access = LDAPAccess(ldap_config) + Token.set_token_lifetime(token_lifetime) + self.middleware = AuthenticationMiddleware() + self.app = falcon.API(middleware=[self.middleware]) + self.app.add_error_handler(CalipsoApiException) + self.set_routes(self.app) + + def get_app(self): + return self.app + + def set_routes(self, app): + for url in self.ROUTE_DECLARATIONS.keys(): + class_path = self.ROUTE_DECLARATIONS.get(url) + module = self.responders_path + "." + \ + class_path[:class_path.rindex(".")] + class_name = class_path.split('.')[-1] + module = importlib.import_module(module) + class_ = getattr(module, class_name) + resource = class_() + app.add_route(url, resource) diff --git a/app/api/auth/__init__.py b/app/api/auth/__init__.py new file mode 100644 index 0000000..1e85a2a --- /dev/null +++ b/app/api/auth/__init__.py @@ -0,0 +1,10 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + diff --git a/app/api/auth/auth.py b/app/api/auth/auth.py new file mode 100644 index 0000000..04fc4b9 --- /dev/null +++ b/app/api/auth/auth.py @@ -0,0 +1,71 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from api.auth.token import Token +from api.backends.ldap_access import LDAPAccess +from utils.inventory_mgr import InventoryMgr +from utils.logging.full_logger import FullLogger + + +class Auth: + + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + self.log = FullLogger() + self.tokens_coll = self.inv.client['tokens']['api_tokens'] + self.ldap_access = LDAPAccess() + + def get_token(self, token): + tokens = None + try: + tokens = list(self.tokens_coll.find({'token': token})) + except Exception as e: + self.log.error('Failed to get token for ', str(e)) + + return tokens + + def write_token(self, token): + error = None + try: + self.tokens_coll.insert_one(token) + except Exception as e: + self.log.error("Failed to write new token {0} to database for {1}" + .format(token[token], str(e))) + error = 'Failed to create new token' + + return error + + def delete_token(self, token): + error = None + try: + self.tokens_coll.delete_one({'token': token}) + except Exception as e: + self.log.error('Failed to delete token {0} for {1}'. + format(token, str(e))) + error = 'Failed to delete token {0}'.format(token) + + return error + + def validate_credentials(self, username, pwd): + return self.ldap_access.authenticate_user(username, pwd) + + def validate_token(self, token): + error = None + tokens = self.get_token(token) + if not tokens: + error = "Token {0} doesn't exist".format(token) + elif len(tokens) > 1: + self.log.error('Multiple tokens found for {0}'.format(token)) + error = "Multiple tokens found" + else: + t = tokens[0] + error = Token.validate_token(t) + + return error diff --git a/app/api/auth/token.py b/app/api/auth/token.py new file mode 100644 index 0000000..d057d22 --- /dev/null +++ b/app/api/auth/token.py @@ -0,0 +1,39 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import datetime +import uuid + + +class Token: + token_lifetime = 86400 + FIELD = 'X-AUTH-TOKEN' + + @classmethod + def set_token_lifetime(cls, lifetime): + Token.token_lifetime = lifetime + + @classmethod + def new_uuid_token(cls, method): + token = {} + token['issued_at'] = datetime.datetime.now() + token['expires_at'] = token['issued_at'] +\ + datetime.timedelta(seconds=Token.token_lifetime) + token['token'] = uuid.uuid4().hex + token['method'] = method + return token + + @classmethod + def validate_token(cls, token): + error = None + now = datetime.datetime.now() + if now > token['expires_at']: + error = 'Token {0} has expired'.format(token['token']) + + return error diff --git a/app/api/backends/__init__.py b/app/api/backends/__init__.py new file mode 100644 index 0000000..1e85a2a --- /dev/null +++ b/app/api/backends/__init__.py @@ -0,0 +1,10 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + diff --git a/app/api/backends/ldap_access.py b/app/api/backends/ldap_access.py new file mode 100644 index 0000000..a998656 --- /dev/null +++ b/app/api/backends/ldap_access.py @@ -0,0 +1,89 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import ssl + +from ldap3 import Server, Connection, Tls + +from utils.config_file import ConfigFile +from utils.logging.full_logger import FullLogger +from utils.singleton import Singleton + + +class LDAPAccess(metaclass=Singleton): + + default_config_file = "ldap.conf" + TLS_REQUEST_CERTS = { + "demand": ssl.CERT_REQUIRED, + "allow": ssl.CERT_OPTIONAL, + "never": ssl.CERT_NONE, + "default": ssl.CERT_NONE + } + user_ssl = True + + def __init__(self, config_file_path=""): + super().__init__() + self.log = FullLogger() + self.ldap_params = self.get_ldap_params(config_file_path) + self.server = self.connect_ldap_server() + + def get_ldap_params(self, config_file_path): + ldap_params = { + "url": "ldap://localhost:389" + } + if not config_file_path: + config_file_path = ConfigFile.get(self.default_config_file) + if config_file_path: + try: + config_file = ConfigFile(config_file_path) + params = config_file.read_config() + ldap_params.update(params) + except Exception as e: + self.log.error(str(e)) + raise + if "user_tree_dn" not in ldap_params: + raise ValueError("user_tree_dn must be specified in " + + config_file_path) + if "user_id_attribute" not in ldap_params: + raise ValueError("user_id_attribute must be specified in " + + config_file_path) + return ldap_params + + def connect_ldap_server(self): + ca_certificate_file = self.ldap_params.get('tls_cacertfile') + req_cert = self.ldap_params.get('tls_req_cert') + ldap_url = self.ldap_params.get('url') + + if ca_certificate_file: + if not req_cert or req_cert not in self.TLS_REQUEST_CERTS.keys(): + req_cert = 'default' + tls_req_cert = self.TLS_REQUEST_CERTS[req_cert] + tls = Tls(local_certificate_file=ca_certificate_file, + validate=tls_req_cert) + return Server(ldap_url, use_ssl=self.user_ssl, tls=tls) + + return Server(ldap_url, use_ssl=self.user_ssl) + + def authenticate_user(self, username, pwd): + if not self.server: + self.server = self.connect_ldap_server() + + user_dn = self.ldap_params['user_id_attribute'] + "=" + + username + "," + self.ldap_params['user_tree_dn'] + connection = Connection(self.server, user=user_dn, password=pwd) + # validate the user by binding + # bound is true if binding succeed, otherwise false + bound = False + try: + bound = connection.bind() + connection.unbind() + except Exception as e: + self.log.error('Failed to bind the server for {0}'.format(str(e))) + + return bound diff --git a/app/api/exceptions/__init__.py b/app/api/exceptions/__init__.py new file mode 100644 index 0000000..1e85a2a --- /dev/null +++ b/app/api/exceptions/__init__.py @@ -0,0 +1,10 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + diff --git a/app/api/exceptions/exceptions.py b/app/api/exceptions/exceptions.py new file mode 100644 index 0000000..f0a1d9f --- /dev/null +++ b/app/api/exceptions/exceptions.py @@ -0,0 +1,26 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from utils.logging.console_logger import ConsoleLogger + + +class CalipsoApiException(Exception): + log = ConsoleLogger() + + def __init__(self, status, body="", message=""): + super().__init__(message) + self.message = message + self.status = status + self.body = body + + @staticmethod + def handle(ex, req, resp, params): + CalipsoApiException.log.error(ex.message) + resp.status = ex.status + resp.body = ex.body diff --git a/app/api/middleware/__init__.py b/app/api/middleware/__init__.py new file mode 100644 index 0000000..1e85a2a --- /dev/null +++ b/app/api/middleware/__init__.py @@ -0,0 +1,10 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + diff --git a/app/api/middleware/authentication.py b/app/api/middleware/authentication.py new file mode 100644 index 0000000..bc62fa8 --- /dev/null +++ b/app/api/middleware/authentication.py @@ -0,0 +1,63 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import base64 + +from api.responders.responder_base import ResponderBase +from api.auth.auth import Auth +from api.auth.token import Token + + +class AuthenticationMiddleware(ResponderBase): + def __init__(self): + super().__init__() + self.auth = Auth() + self.BASIC_AUTH = "AUTHORIZATION" + self.EXCEPTION_ROUTES = ['/auth/tokens'] + + def process_request(self, req, resp): + if req.path in self.EXCEPTION_ROUTES: + return + + self.log.debug("Authentication middleware is processing the request") + headers = self.change_dict_naming_convention(req.headers, + lambda s: s.upper()) + auth_error = None + if self.BASIC_AUTH in headers: + # basic authentication + self.log.debug("Authenticating the basic credentials") + basic = headers[self.BASIC_AUTH] + auth_error = self.authenticate_with_basic_auth(basic) + elif Token.FIELD in headers: + # token authentication + self.log.debug("Authenticating token") + token = headers[Token.FIELD] + auth_error = self.auth.validate_token(token) + else: + auth_error = "Authentication required" + + if auth_error: + self.unauthorized(auth_error) + + def authenticate_with_basic_auth(self, basic): + error = None + if not basic or not basic.startswith("Basic"): + error = "Credentials not provided" + else: + # get username and password + credential = basic.lstrip("Basic").lstrip() + username_password = base64.b64decode(credential).decode("utf-8") + credentials = username_password.split(":") + if not self.auth.validate_credentials(credentials[0], credentials[1]): + self.log.info("Authentication for {0} failed".format(credentials[0])) + error = "Authentication failed" + else: + self.log.info("Authentication for {0} succeeded".format(credentials[0])) + + return error diff --git a/app/api/responders/__init__.py b/app/api/responders/__init__.py new file mode 100644 index 0000000..1e85a2a --- /dev/null +++ b/app/api/responders/__init__.py @@ -0,0 +1,10 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + diff --git a/app/api/responders/auth/__init__.py b/app/api/responders/auth/__init__.py new file mode 100644 index 0000000..1e85a2a --- /dev/null +++ b/app/api/responders/auth/__init__.py @@ -0,0 +1,10 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + diff --git a/app/api/responders/auth/tokens.py b/app/api/responders/auth/tokens.py new file mode 100644 index 0000000..0b3a22f --- /dev/null +++ b/app/api/responders/auth/tokens.py @@ -0,0 +1,117 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from datetime import datetime + +from bson.objectid import ObjectId + +from api.auth.auth import Auth +from api.auth.token import Token +from api.responders.responder_base import ResponderBase +from api.validation.data_validate import DataValidate +from utils.string_utils import stringify_object_values_by_types + + +class Tokens(ResponderBase): + + def __init__(self): + super().__init__() + self.auth_requirements = { + 'methods': self.require(list, False, + DataValidate.LIST, + ['credentials', 'token'], + True), + 'credentials': self.require(dict, True), + 'token': self.require(str) + } + + self.credential_requirements = { + 'username': self.require(str, mandatory=True), + 'password': self.require(str, mandatory=True) + } + self.auth = Auth() + + def on_post(self, req, resp): + self.log.debug('creating new token') + error, data = self.get_content_from_request(req) + if error: + self.bad_request(error) + + if 'auth' not in data: + self.bad_request('Request must contain auth object') + + auth = data['auth'] + + self.validate_query_data(auth, self.auth_requirements) + + if 'credentials' in auth: + self.validate_query_data(auth['credentials'], + self.credential_requirements) + + auth_error = self.authenticate(auth) + if auth_error: + self.unauthorized(auth_error) + + new_token = Token.new_uuid_token(auth['method']) + write_error = self.auth.write_token(new_token) + + if write_error: + # TODO if writing token to the database failed, what kind of error should be return? + self.bad_request(write_error) + + stringify_object_values_by_types(new_token, [datetime, ObjectId]) + self.set_successful_response(resp, new_token, '201') + + def authenticate(self, auth): + error = None + methods = auth['methods'] + credentials = auth.get('credentials') + token = auth.get('token') + + if not token and not credentials: + return 'must provide credentials or token' + + if 'credentials' in methods: + if not credentials: + return'credentials must be provided for credentials method' + else: + if not self.auth.validate_credentials(credentials['username'], + credentials['password']): + error = 'authentication failed' + else: + auth['method'] = "credentials" + return None + + if 'token' in methods: + if not token: + return 'token must be provided for token method' + else: + error = self.auth.validate_token(token) + if not error: + auth['method'] = 'token' + + return error + + def on_delete(self, req, resp): + headers = self.change_dict_naming_convention(req.headers, + lambda s: s.upper()) + if Token.FIELD not in headers: + self.unauthorized('Authentication failed') + + token = headers[Token.FIELD] + error = self.auth.validate_token(token) + if error: + self.unauthorized(error) + + delete_error = self.auth.delete_token(token) + + if delete_error: + self.bad_request(delete_error) + + self.set_successful_response(resp) diff --git a/app/api/responders/resource/__init__.py b/app/api/responders/resource/__init__.py new file mode 100644 index 0000000..1e85a2a --- /dev/null +++ b/app/api/responders/resource/__init__.py @@ -0,0 +1,10 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + diff --git a/app/api/responders/resource/aggregates.py b/app/api/responders/resource/aggregates.py new file mode 100644 index 0000000..36fcfa4 --- /dev/null +++ b/app/api/responders/resource/aggregates.py @@ -0,0 +1,157 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from api.responders.responder_base import ResponderBase +from api.validation.data_validate import DataValidate + + +class Aggregates(ResponderBase): + def __init__(self): + super().__init__() + self.AGGREGATE_TYPES = ["environment", "message", "constant"] + self.AGGREGATES_MAP = { + "environment": self.get_environments_aggregates, + "message": self.get_messages_aggregates, + "constant": self.get_constants_aggregates + } + + def on_get(self, req, resp): + self.log.debug("Getting aggregates information") + + filters = self.parse_query_params(req) + filters_requirements = { + "env_name": self.require(str), + "type": self.require(str, validate=DataValidate.LIST, + requirement=self.AGGREGATE_TYPES, + mandatory=True, + error_messages={"mandatory": + "type must be specified: " + + "environment/" + + " message/" + + "constant"}) + } + self.validate_query_data(filters, filters_requirements) + query = self.build_query(filters) + query_type = query["type"] + if query_type == "environment": + env_name = query.get("env_name") + if not env_name: + self.bad_request("env_name must be specified") + if not self.check_environment_name(env_name): + self.bad_request("unknown environment: " + env_name) + + aggregates = self.AGGREGATES_MAP[query_type](query) + self.set_successful_response(resp, aggregates) + + def build_query(self, filters): + query = {} + env_name = filters.get("env_name") + query_type = filters["type"] + query["type"] = filters["type"] + if query_type == "environment": + if env_name: + query['env_name'] = env_name + return query + return query + + def get_environments_aggregates(self, query): + env_name = query['env_name'] + aggregates = { + "type": query["type"], + "env_name": env_name, + "aggregates": { + "object_types": { + + } + } + } + pipeline = [ + { + '$match': { + 'environment': env_name + } + }, + { + '$group': { + '_id': '$type', + 'total': { + '$sum': 1 + } + } + } + ] + groups = self.aggregate(pipeline, "inventory") + for group in groups: + aggregates['aggregates']['object_types'][group['_id']] = \ + group['total'] + return aggregates + + def get_messages_aggregates(self, query): + aggregates = { + "type": query['type'], + "aggregates": { + "levels": {}, + "environments": {} + } + } + env_pipeline = [ + { + '$group': { + '_id': '$environment', + 'total': { + '$sum': 1 + } + } + } + ] + environments = self.aggregate(env_pipeline, "messages") + for environment in environments: + aggregates['aggregates']['environments'][environment['_id']] = \ + environment['total'] + level_pipeline = [ + { + '$group': { + '_id': '$level', + 'total': { + '$sum': 1 + } + } + } + ] + levels = self.aggregate(level_pipeline, "messages") + for level in levels: + aggregates['aggregates']['levels'][level['_id']] = \ + level['total'] + + return aggregates + + def get_constants_aggregates(self, query): + aggregates = { + "type": query['type'], + "aggregates": { + "names": {} + } + } + pipeline = [ + { + '$project': { + '_id': 0, + 'name': 1, + 'total': { + '$size': '$data' + } + } + } + ] + constants = self.aggregate(pipeline, "constants") + for constant in constants: + aggregates['aggregates']['names'][constant['name']] = \ + constant['total'] + + return aggregates diff --git a/app/api/responders/resource/clique_constraints.py b/app/api/responders/resource/clique_constraints.py new file mode 100644 index 0000000..eddead9 --- /dev/null +++ b/app/api/responders/resource/clique_constraints.py @@ -0,0 +1,67 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from api.responders.responder_base import ResponderBase +from api.validation.data_validate import DataValidate +from bson.objectid import ObjectId + + +class CliqueConstraints(ResponderBase): + def __init__(self): + super().__init__() + self.ID = '_id' + self.PROJECTION = { + self.ID: True + } + self.COLLECTION = 'clique_constraints' + + def on_get(self, req, resp): + self.log.debug("Getting clique_constraints") + filters = self.parse_query_params(req) + focal_point_types = self.get_constants_by_name("object_types") + filters_requirements = { + 'id': self.require(ObjectId, True), + 'focal_point_type': self.require(str, False, DataValidate.LIST, + focal_point_types), + 'constraint': self.require([list, str]), + 'page': self.require(int, True), + 'page_size': self.require(int, True) + } + self.validate_query_data(filters, filters_requirements) + page, page_size = self.get_pagination(filters) + query = self.build_query(filters) + if self.ID in query: + clique_constraint = self.get_object_by_id(self.COLLECTION, + query, + [ObjectId], self.ID) + self.set_successful_response(resp, clique_constraint) + else: + clique_constraints_ids = self.get_objects_list(self.COLLECTION, + query, + page, page_size, self.PROJECTION) + self.set_successful_response( + resp, {"clique_constraints": clique_constraints_ids} + ) + + def build_query(self, filters): + query = {} + filters_keys = ['focal_point_type'] + self.update_query_with_filters(filters, filters_keys, query) + constraints = filters.get('constraint') + if constraints: + if type(constraints) != list: + constraints = [constraints] + + query['constraints'] = { + '$all': constraints + } + _id = filters.get('id') + if _id: + query[self.ID] = _id + return query diff --git a/app/api/responders/resource/clique_types.py b/app/api/responders/resource/clique_types.py new file mode 100644 index 0000000..9a39dc8 --- /dev/null +++ b/app/api/responders/resource/clique_types.py @@ -0,0 +1,103 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from api.responders.responder_base import ResponderBase +from api.validation.data_validate import DataValidate +from bson.objectid import ObjectId + + +class CliqueTypes(ResponderBase): + def __init__(self): + super().__init__() + self.COLLECTION = "clique_types" + self.ID = "_id" + self.PROJECTION = { + self.ID: True, + "focal_point_type": True, + "link_types": True, + "environment": True + } + + def on_get(self, req, resp): + self.log.debug("Getting clique types") + + filters = self.parse_query_params(req) + focal_point_types = self.get_constants_by_name("object_types") + link_types = self.get_constants_by_name("link_types") + filters_requirements = { + 'env_name': self.require(str, mandatory=True), + 'id': self.require(ObjectId, True), + 'focal_point_type': self.require(str, + validate=DataValidate.LIST, + requirement=focal_point_types), + 'link_type': self.require([list, str], + validate=DataValidate.LIST, + requirement=link_types), + 'page': self.require(int, True), + 'page_size': self.require(int, True) + } + + self.validate_query_data(filters, filters_requirements) + page, page_size = self.get_pagination(filters) + query = self.build_query(filters) + if self.ID in query: + clique_type = self.get_object_by_id(self.COLLECTION, query, + [ObjectId], self.ID) + self.set_successful_response(resp, clique_type) + else: + clique_types_ids = self.get_objects_list(self.COLLECTION, + query, + page, page_size, self.PROJECTION) + self.set_successful_response(resp, + {"clique_types": clique_types_ids}) + + def on_post(self, req, resp): + self.log.debug("Posting new clique_type") + error, clique_type = self.get_content_from_request(req) + if error: + self.bad_request(error) + focal_point_types = self.get_constants_by_name("object_types") + link_types = self.get_constants_by_name("link_types") + clique_type_requirements = { + 'environment': self.require(str, mandatory=True), + 'focal_point_type': self.require(str, False, DataValidate.LIST, + focal_point_types, True), + 'link_types': self.require(list, False, DataValidate.LIST, + link_types, True), + 'name': self.require(str, mandatory=True) + } + + self.validate_query_data(clique_type, clique_type_requirements) + + env_name = clique_type['environment'] + if not self.check_environment_name(env_name): + self.bad_request("unkown environment: " + env_name) + + self.write(clique_type, self.COLLECTION) + self.set_successful_response(resp, + {"message": "created a new clique_type " + "for environment {0}" + .format(env_name)}, + "201") + + def build_query(self, filters): + query = {} + filters_keys = ['focal_point_type'] + self.update_query_with_filters(filters, filters_keys, query) + link_types = filters.get('link_type') + if link_types: + if type(link_types) != list: + link_types = [link_types] + query['link_types'] = {'$all': link_types} + _id = filters.get('id') + if _id: + query[self.ID] = _id + + query['environment'] = filters['env_name'] + return query diff --git a/app/api/responders/resource/cliques.py b/app/api/responders/resource/cliques.py new file mode 100644 index 0000000..ece347a --- /dev/null +++ b/app/api/responders/resource/cliques.py @@ -0,0 +1,73 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from api.validation.data_validate import DataValidate +from api.responders.responder_base import ResponderBase +from bson.objectid import ObjectId + +from utils.util import generate_object_ids + + +class Cliques(ResponderBase): + def __init__(self): + super().__init__() + self.COLLECTION = "cliques" + self.ID = '_id' + self.PROJECTION = { + self.ID: True, + "focal_point_type": True, + "environment": True + } + + def on_get(self, req, resp): + self.log.debug("Getting cliques") + + filters = self.parse_query_params(req) + focal_point_types = self.get_constants_by_name("object_types") + link_types = self.get_constants_by_name("link_types") + filters_requirements = { + 'env_name': self.require(str, mandatory=True), + 'id': self.require(ObjectId, True), + 'focal_point': self.require(ObjectId, True), + 'focal_point_type': self.require(str, validate=DataValidate.LIST, + requirement=focal_point_types), + 'link_type': self.require(str, validate=DataValidate.LIST, + requirement=link_types), + 'link_id': self.require(ObjectId, True), + 'page': self.require(int, True), + 'page_size': self.require(int, True) + } + self.validate_query_data(filters, filters_requirements) + page, page_size = self.get_pagination(filters) + query = self.build_query(filters) + + if self.ID in query: + clique = self.get_object_by_id(self.COLLECTION, query, + [ObjectId], self.ID) + self.set_successful_response(resp, clique) + else: + cliques_ids = self.get_objects_list(self.COLLECTION, query, + page, page_size, self.PROJECTION) + self.set_successful_response(resp, {"cliques": cliques_ids}) + + def build_query(self, filters): + query = {} + filters_keys = ['focal_point', 'focal_point_type'] + self.update_query_with_filters(filters, filters_keys, query) + link_type = filters.get('link_type') + if link_type: + query['links_detailed.link_type'] = link_type + link_id = filters.get('link_id') + if link_id: + query['links_detailed._id'] = link_id + _id = filters.get('id') + if _id: + query[self.ID] = _id + query['environment'] = filters['env_name'] + return query diff --git a/app/api/responders/resource/constants.py b/app/api/responders/resource/constants.py new file mode 100644 index 0000000..be71b5d --- /dev/null +++ b/app/api/responders/resource/constants.py @@ -0,0 +1,30 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from api.responders.responder_base import ResponderBase +from bson.objectid import ObjectId + + +class Constants(ResponderBase): + def __init__(self): + super().__init__() + self.ID = '_id' + self.COLLECTION = 'constants' + + def on_get(self, req, resp): + self.log.debug("Getting constants with name") + filters = self.parse_query_params(req) + filters_requirements = { + "name": self.require(str, mandatory=True), + } + self.validate_query_data(filters, filters_requirements) + query = {"name": filters['name']} + constant = self.get_object_by_id(self.COLLECTION, query, + [ObjectId], self.ID) + self.set_successful_response(resp, constant) diff --git a/app/api/responders/resource/environment_configs.py b/app/api/responders/resource/environment_configs.py new file mode 100644 index 0000000..bee6a4d --- /dev/null +++ b/app/api/responders/resource/environment_configs.py @@ -0,0 +1,381 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from api.validation import regex +from api.validation.data_validate import DataValidate +from api.responders.responder_base import ResponderBase +from bson.objectid import ObjectId +from datetime import datetime +from utils.constants import EnvironmentFeatures +from utils.inventory_mgr import InventoryMgr + + +class EnvironmentConfigs(ResponderBase): + def __init__(self): + super(EnvironmentConfigs, self).__init__() + self.inv = InventoryMgr() + self.ID = "name" + self.PROJECTION = { + self.ID: True, + "_id": False, + "name": True, + "distribution": True + } + self.COLLECTION = "environments_config" + self.CONFIGURATIONS_NAMES = ["mysql", "OpenStack", + "CLI", "AMQP", "Monitoring", + "NFV_provider", "ACI"] + self.OPTIONAL_CONFIGURATIONS_NAMES = ["AMQP", "Monitoring", + "NFV_provider", "ACI"] + + self.provision_types = self.\ + get_constants_by_name("environment_provision_types") + self.env_types = self.get_constants_by_name("env_types") + self.monitoring_types = self.\ + get_constants_by_name("environment_monitoring_types") + self.distributions = self.\ + get_constants_by_name("distributions") + self.mechanism_drivers = self.\ + get_constants_by_name("mechanism_drivers") + self.operational_values = self.\ + get_constants_by_name("environment_operational_status") + self.type_drivers = self.\ + get_constants_by_name("type_drivers") + + self.CONFIGURATIONS_REQUIREMENTS = { + "mysql": { + "name": self.require(str, mandatory=True), + "host": self.require(str, + validate=DataValidate.REGEX, + requirement=[regex.IP, regex.HOSTNAME], + mandatory=True), + "password": self.require(str, mandatory=True), + "port": self.require(int, + True, + DataValidate.REGEX, + regex.PORT, + mandatory=True), + "user": self.require(str, mandatory=True) + }, + "OpenStack": { + "name": self.require(str, mandatory=True), + "admin_token": self.require(str, mandatory=True), + "host": self.require(str, + validate=DataValidate.REGEX, + requirement=[regex.IP, regex.HOSTNAME], + mandatory=True), + "port": self.require(int, + True, + validate=DataValidate.REGEX, + requirement=regex.PORT, + mandatory=True), + "pwd": self.require(str, mandatory=True), + "user": self.require(str, mandatory=True) + }, + "CLI": { + "name": self.require(str, mandatory=True), + "host": self.require(str, + validate=DataValidate.REGEX, + requirement=[regex.IP, regex.HOSTNAME], + mandatory=True), + "user": self.require(str, mandatory=True), + "pwd": self.require(str), + "key": self.require(str, + validate=DataValidate.REGEX, + requirement=regex.PATH) + }, + "AMQP": { + "name": self.require(str, mandatory=True), + "host": self.require(str, + validate=DataValidate.REGEX, + requirement=[regex.IP, regex.HOSTNAME], + mandatory=True), + "password": self.require(str, mandatory=True), + "port": self.require(int, + True, + validate=DataValidate.REGEX, + requirement=regex.PORT, + mandatory=True), + "user": self.require(str, mandatory=True) + }, + "Monitoring": { + "name": self.require(str, mandatory=True), + "config_folder": self.require(str, + validate=DataValidate.REGEX, + requirement=regex.PATH, + mandatory=True), + "provision": self.require(str, + validate=DataValidate.LIST, + requirement=self.provision_types, + mandatory=True), + "env_type": self.require(str, + validate=DataValidate.LIST, + requirement=self.env_types, + mandatory=True), + "api_port": self.require(int, True, mandatory=True), + "rabbitmq_pass": self.require(str, mandatory=True), + "rabbitmq_user": self.require(str, mandatory=True), + "rabbitmq_port": self.require(int, + True, + validate=DataValidate.REGEX, + requirement=regex.PORT, + mandatory=True), + "ssh_port": self.require(int, + True, + validate=DataValidate.REGEX, + requirement=regex.PORT), + "ssh_user": self.require(str), + "ssh_password": self.require(str), + "server_ip": self.require(str, + validate=DataValidate.REGEX, + requirement=[regex.IP, regex.HOSTNAME], + mandatory=True), + "server_name": self.require(str, mandatory=True), + "type": self.require(str, + validate=DataValidate.LIST, + requirement=self.monitoring_types, + mandatory=True) + }, + "NFV_provider": { + "name": self.require(str, mandatory=True), + "host": self.require(str, + validate=DataValidate.REGEX, + requirement=[regex.IP, regex.HOSTNAME], + mandatory=True), + "nfv_token": self.require(str, mandatory=True), + "port": self.require(int, + True, + DataValidate.REGEX, + regex.PORT, + True), + "user": self.require(str, mandatory=True), + "pwd": self.require(str, mandatory=True) + }, + "ACI": { + "name": self.require(str, mandatory=True), + "host": self.require(str, + validate=DataValidate.REGEX, + requirement=[regex.IP, regex.HOSTNAME], + mandatory=True), + "user": self.require(str, mandatory=True), + "pwd": self.require(str, mandatory=True) + } + } + self.AUTH_REQUIREMENTS = { + "view-env": self.require(list, mandatory=True), + "edit-env": self.require(list, mandatory=True) + } + + def on_get(self, req, resp): + self.log.debug("Getting environment config") + filters = self.parse_query_params(req) + + filters_requirements = { + "name": self.require(str), + "distribution": self.require(str, False, + DataValidate.LIST, + self.distributions), + "mechanism_drivers": self.require([str, list], + False, + DataValidate.LIST, + self.mechanism_drivers), + "type_drivers": self.require(str, False, + DataValidate.LIST, + self.type_drivers), + "user": self.require(str), + "listen": self.require(bool, True), + "scanned": self.require(bool, True), + "monitoring_setup_done": self.require(bool, True), + "operational": self.require(str, False, + DataValidate.LIST, + self.operational_values), + "page": self.require(int, True), + "page_size": self.require(int, True) + } + + self.validate_query_data(filters, filters_requirements) + page, page_size = self.get_pagination(filters) + + query = self.build_query(filters) + + if self.ID in query: + environment_config = self.get_object_by_id(self.COLLECTION, query, + [ObjectId, datetime], self.ID) + self.set_successful_response(resp, environment_config) + else: + objects_ids = self.get_objects_list(self.COLLECTION, query, + page, page_size, self.PROJECTION) + self.set_successful_response(resp, {'environment_configs': objects_ids}) + + def build_query(self, filters): + query = {} + filters_keys = ["name", "distribution", "type_drivers", "user", + "listen", "monitoring_setup_done", "scanned", + "operational"] + self.update_query_with_filters(filters, filters_keys, query) + mechanism_drivers = filters.get("mechanism_drivers") + if mechanism_drivers: + if type(mechanism_drivers) != list: + mechanism_drivers = [mechanism_drivers] + query['mechanism_drivers'] = {'$all': mechanism_drivers} + + return query + + def on_post(self, req, resp): + self.log.debug("Creating a new environment config") + + error, env_config = self.get_content_from_request(req) + if error: + self.bad_request(error) + + environment_config_requirement = { + "app_path": self.require(str, mandatory=True), + "configuration": self.require(list, mandatory=True), + "distribution": self.require(str, False, DataValidate.LIST, + self.distributions, True), + "listen": self.require(bool, True, mandatory=True), + "user": self.require(str), + "mechanism_drivers": self.require(list, False, DataValidate.LIST, + self.mechanism_drivers, True), + "name": self.require(str, mandatory=True), + "operational": self.require(str, True, DataValidate.LIST, + self.operational_values, mandatory=True), + "scanned": self.require(bool, True), + "last_scanned": self.require(str), + "type": self.require(str, mandatory=True), + "type_drivers": self.require(str, False, DataValidate.LIST, + self.type_drivers, True), + "enable_monitoring": self.require(bool, True), + "monitoring_setup_done": self.require(bool, True), + "auth": self.require(dict) + } + self.validate_query_data(env_config, + environment_config_requirement, + can_be_empty_keys=["last_scanned"] + ) + self.check_and_convert_datetime("last_scanned", env_config) + # validate the configurations + configurations = env_config['configuration'] + config_validation = self.validate_environment_config(configurations) + + if not config_validation['passed']: + self.bad_request(config_validation['error_message']) + + err_msg = self.validate_env_config_with_supported_envs(env_config) + if err_msg: + self.bad_request(err_msg) + + err_msg = self.validate_env_config_with_constraints(env_config) + if err_msg: + self.bad_request(err_msg) + + if "auth" in env_config: + err_msg = self.validate_data(env_config.get("auth"), + self.AUTH_REQUIREMENTS) + if err_msg: + self.bad_request("auth error: " + err_msg) + + if "scanned" not in env_config: + env_config["scanned"] = False + + self.write(env_config, self.COLLECTION) + self.set_successful_response(resp, + {"message": "created environment_config " + "for {0}" + .format(env_config["name"])}, + "201") + + def validate_environment_config(self, configurations): + configurations_of_names = {} + validation = {"passed": True} + if [config for config in configurations + if 'name' not in config]: + validation['passed'] = False + validation['error_message'] = "configuration must have name" + return validation + + unknown_configs = [config['name'] for config in configurations + if config['name'] not in self.CONFIGURATIONS_NAMES] + if unknown_configs: + validation['passed'] = False + validation['error_message'] = 'Unknown configurations: {0}'. \ + format(' and '.join(unknown_configs)) + return validation + + for name in self.CONFIGURATIONS_NAMES: + configs = self.get_configuration_by_name(name, configurations) + if configs: + if len(configs) > 1: + validation["passed"] = False + validation["error_message"] = "environment configurations can " \ + "only contain one " \ + "configuration for {0}".format(name) + return validation + configurations_of_names[name] = configs[0] + else: + if name not in self.OPTIONAL_CONFIGURATIONS_NAMES: + validation["passed"] = False + validation['error_message'] = "configuration for {0} " \ + "is mandatory".format(name) + return validation + + for name, config in configurations_of_names.items(): + error_message = self.validate_configuration(name, config) + if error_message: + validation['passed'] = False + validation['error_message'] = "{0} error: {1}".\ + format(name, error_message) + break + if name is 'CLI': + if 'key' not in config and 'pwd' not in config: + validation['passed'] = False + validation['error_message'] = 'CLI error: either key ' \ + 'or pwd must be provided' + return validation + + def validate_env_config_with_supported_envs(self, env_config): + # validate the environment config with supported environments + matches = { + 'environment.distribution': env_config['distribution'], + 'environment.type_drivers': env_config['type_drivers'], + 'environment.mechanism_drivers': {'$in': env_config['mechanism_drivers']} + } + + err_prefix = 'configuration not accepted: ' + if not self.inv.is_feature_supported_in_env(matches, + EnvironmentFeatures.SCANNING): + return err_prefix + 'scanning is not supported in this environment' + + configs = env_config['configuration'] + if not self.inv.is_feature_supported_in_env(matches, + EnvironmentFeatures.MONITORING) \ + and self.get_configuration_by_name('Monitoring', configs): + return err_prefix + 'monitoring is not supported in this environment, ' \ + 'please remove the Monitoring configuration' + + if not self.inv.is_feature_supported_in_env(matches, + EnvironmentFeatures.LISTENING) \ + and self.get_configuration_by_name('AMQP', configs): + return err_prefix + 'listening is not supported in this environment, ' \ + 'please remove the AMQP configuration' + + return None + + def validate_env_config_with_constraints(self, env_config): + if env_config['listen'] and \ + not self.get_configuration_by_name('AMQP', env_config['configuration']): + return 'configuration not accepted: ' \ + 'must provide AMQP configuration to listen the environment' + + def get_configuration_by_name(self, name, configurations): + return [config for config in configurations if config['name'] == name] + + def validate_configuration(self, name, configuration): + return self.validate_data(configuration, + self.CONFIGURATIONS_REQUIREMENTS[name]) diff --git a/app/api/responders/resource/inventory.py b/app/api/responders/resource/inventory.py new file mode 100644 index 0000000..02bc486 --- /dev/null +++ b/app/api/responders/resource/inventory.py @@ -0,0 +1,65 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from api.responders.responder_base import ResponderBase +from bson.objectid import ObjectId +from datetime import datetime + + +class Inventory(ResponderBase): + def __init__(self): + super().__init__() + self.COLLECTION = 'inventory' + self.ID = 'id' + self.PROJECTION = { + self.ID: True, + "name": True, + "name_path": True + } + + def on_get(self, req, resp): + self.log.debug("Getting objects from inventory") + + filters = self.parse_query_params(req) + filters_requirements = { + 'env_name': self.require(str, mandatory=True), + 'id': self.require(str), + 'id_path': self.require(str), + 'parent_id': self.require(str), + 'parent_path': self.require(str), + 'sub_tree': self.require(bool, True), + 'page': self.require(int, True), + 'page_size': self.require(int, True) + } + self.validate_query_data(filters, filters_requirements) + page, page_size = self.get_pagination(filters) + query = self.build_query(filters) + if self.ID in query: + obj = self.get_object_by_id(self.COLLECTION, query, + [ObjectId, datetime], self.ID) + self.set_successful_response(resp, obj) + else: + objects_ids = self.get_objects_list(self.COLLECTION, query, + page, page_size, self.PROJECTION) + self.set_successful_response(resp, {"objects": objects_ids}) + + def build_query(self, filters): + query = {} + filters_keys = ['parent_id', 'id_path', 'id'] + self.update_query_with_filters(filters, filters_keys, query) + parent_path = filters.get('parent_path') + if parent_path: + regular_expression = parent_path + if filters.get('sub_tree', False): + regular_expression += "[/]?" + else: + regular_expression += "/[^/]+$" + query['id_path'] = {"$regex": regular_expression} + query['environment'] = filters['env_name'] + return query diff --git a/app/api/responders/resource/links.py b/app/api/responders/resource/links.py new file mode 100644 index 0000000..33fd432 --- /dev/null +++ b/app/api/responders/resource/links.py @@ -0,0 +1,76 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from api.responders.responder_base import ResponderBase +from api.validation.data_validate import DataValidate +from bson.objectid import ObjectId + + +class Links(ResponderBase): + def __init__(self): + super().__init__() + self.COLLECTION = 'links' + self.ID = '_id' + self.PROJECTION = { + self.ID: True, + "link_name": True, + "link_type": True, + "environment": True, + "host": True + } + + def on_get(self, req, resp): + self.log.debug("Getting links from links") + + filters = self.parse_query_params(req) + + link_types = self.get_constants_by_name("link_types") + link_states = self.get_constants_by_name("link_states") + filters_requirements = { + 'env_name': self.require(str, mandatory=True), + 'id': self.require(ObjectId, True), + 'host': self.require(str), + 'link_type': self.require(str, validate=DataValidate.LIST, + requirement=link_types), + 'link_name': self.require(str), + 'source_id': self.require(str), + 'target_id': self.require(str), + 'state': self.require(str, validate=DataValidate.LIST, + requirement=link_states), + 'page': self.require(int, True), + 'page_size': self.require(int, True) + } + + self.validate_query_data(filters, filters_requirements, r'^attributes\:\w+$') + filters = self.change_dict_naming_convention(filters, self.replace_colon_with_dot) + page, page_size = self.get_pagination(filters) + query = self.build_query(filters) + if self.ID in query: + link = self.get_object_by_id(self.COLLECTION, query, + [ObjectId], self.ID) + self.set_successful_response(resp, link) + else: + links_ids = self.get_objects_list(self.COLLECTION, query, + page, page_size, self.PROJECTION) + self.set_successful_response(resp, {"links": links_ids}) + + def build_query(self, filters): + query = {} + filters_keys = ['host', 'link_type', 'link_name', + 'source_id', 'target_id', 'state'] + self.update_query_with_filters(filters, filters_keys, query) + # add attributes to the query + for key in filters.keys(): + if key.startswith("attributes."): + query[key] = filters[key] + _id = filters.get('id') + if _id: + query[self.ID] = _id + query['environment'] = filters['env_name'] + return query diff --git a/app/api/responders/resource/messages.py b/app/api/responders/resource/messages.py new file mode 100644 index 0000000..0dda31b --- /dev/null +++ b/app/api/responders/resource/messages.py @@ -0,0 +1,78 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from datetime import datetime + +from api.responders.responder_base import ResponderBase +from api.validation.data_validate import DataValidate +from bson.objectid import ObjectId + + +class Messages(ResponderBase): + def __init__(self): + super().__init__() + self.ID = "id" + self.COLLECTION = 'messages' + self.PROJECTION = { + self.ID: True, + "environment": True, + "source_system": True, + "level": True + } + + def on_get(self, req, resp): + self.log.debug("Getting messages from messages") + filters = self.parse_query_params(req) + messages_severity = self.get_constants_by_name("messages_severity") + object_types = self.get_constants_by_name("object_types") + filters_requirements = { + 'env_name': self.require(str, mandatory=True), + 'source_system': self.require(str), + 'id': self.require(str), + 'level': self.require(str, validate=DataValidate.LIST, + requirement=messages_severity), + 'related_object': self.require(str), + 'related_object_type': self.require(str, validate=DataValidate.LIST, + requirement=object_types), + 'start_time': self.require(str), + 'end_time': self.require(str), + 'page': self.require(int, True), + 'page_size': self.require(int, True) + } + self.validate_query_data(filters, filters_requirements) + page, page_size = self.get_pagination(filters) + self.check_and_convert_datetime('start_time', filters) + self.check_and_convert_datetime('end_time', filters) + + query = self.build_query(filters) + if self.ID in query: + message = self.get_object_by_id(self.COLLECTION, query, + [ObjectId, datetime], self.ID) + self.set_successful_response(resp, message) + else: + objects_ids = self.get_objects_list(self.COLLECTION, query, + page, page_size, self.PROJECTION) + self.set_successful_response(resp, {'messages': objects_ids}) + + def build_query(self, filters): + query = {} + filters_keys = ['source_system', 'id', 'level', 'related_object', + 'related_object_type'] + self.update_query_with_filters(filters, filters_keys, query) + start_time = filters.get('start_time') + if start_time: + query['timestamp'] = {"$gte": start_time} + end_time = filters.get('end_time') + if end_time: + if 'timestamp' in query: + query['timestamp'].update({"$lte": end_time}) + else: + query['timestamp'] = {"$lte": end_time} + query['environment'] = filters['env_name'] + return query diff --git a/app/api/responders/resource/monitoring_config_templates.py b/app/api/responders/resource/monitoring_config_templates.py new file mode 100644 index 0000000..42d3973 --- /dev/null +++ b/app/api/responders/resource/monitoring_config_templates.py @@ -0,0 +1,65 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from api.validation.data_validate import DataValidate +from api.responders.responder_base import ResponderBase +from bson.objectid import ObjectId + + +class MonitoringConfigTemplates(ResponderBase): + def __init__(self): + super().__init__() + self.ID = "_id" + self.COLLECTION = "monitoring_config_templates" + self.PROJECTION = { + self.ID: True, + "side": True, + "type": True + } + + def on_get(self, req, resp): + self.log.debug("Getting monitoring config template") + + filters = self.parse_query_params(req) + + sides = self.get_constants_by_name("monitoring_sides") + filters_requirements = { + "id": self.require(ObjectId, True), + "order": self.require(int, True), + "side": self.require(str, validate=DataValidate.LIST, + requirement=sides), + "type": self.require(str), + "page": self.require(int, True), + "page_size": self.require(int, True) + } + + self.validate_query_data(filters, filters_requirements) + + page, page_size = self.get_pagination(filters) + query = self.build_query(filters) + if self.ID in query: + template = self.get_object_by_id(self.COLLECTION, query, + [ObjectId], self.ID) + self.set_successful_response(resp, template) + else: + templates = self.get_objects_list(self.COLLECTION, query, + page, page_size, self.PROJECTION) + self.set_successful_response( + resp, + {"monitoring_config_templates": templates} + ) + + def build_query(self, filters): + query = {} + filters_keys = ["order", "side", "type"] + self.update_query_with_filters(filters, filters_keys, query) + _id = filters.get('id') + if _id: + query[self.ID] = _id + return query diff --git a/app/api/responders/resource/scans.py b/app/api/responders/resource/scans.py new file mode 100644 index 0000000..c9ad2e2 --- /dev/null +++ b/app/api/responders/resource/scans.py @@ -0,0 +1,111 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from api.validation.data_validate import DataValidate +from api.responders.responder_base import ResponderBase +from bson.objectid import ObjectId +from datetime import datetime + + +class Scans(ResponderBase): + def __init__(self): + super().__init__() + self.COLLECTION = "scans" + self.ID = "_id" + self.PROJECTION = { + self.ID: True, + "environment": True, + "status": True, + "scan_completed": True + } + + def on_get(self, req, resp): + self.log.debug("Getting scans") + filters = self.parse_query_params(req) + + scan_statuses = self.get_constants_by_name("scan_statuses") + filters_requirements = { + "env_name": self.require(str, mandatory=True), + "id": self.require(ObjectId, True), + "base_object": self.require(str), + "status": self.require(str, False, DataValidate.LIST, scan_statuses), + "page": self.require(int, True), + "page_size": self.require(int, True) + } + + self.validate_query_data(filters, filters_requirements) + page, page_size = self.get_pagination(filters) + + query = self.build_query(filters) + if "_id" in query: + scan = self.get_object_by_id(self.COLLECTION, query, + [ObjectId, datetime], self.ID) + self.set_successful_response(resp, scan) + else: + scans_ids = self.get_objects_list(self.COLLECTION, query, + page, page_size, self.PROJECTION) + self.set_successful_response(resp, {"scans": scans_ids}) + + def on_post(self, req, resp): + self.log.debug("Posting new scan") + error, scan = self.get_content_from_request(req) + if error: + self.bad_request(error) + + scan_statuses = self.get_constants_by_name("scan_statuses") + log_levels = self.get_constants_by_name("log_levels") + + scan_requirements = { + "status": self.require(str, + validate=DataValidate.LIST, + requirement=scan_statuses, + mandatory=True), + "log_level": self.require(str, + validate=DataValidate.LIST, + requirement=log_levels), + "clear": self.require(bool, True), + "scan_only_inventory": self.require(bool, True), + "scan_only_links": self.require(bool, True), + "scan_only_cliques": self.require(bool, True), + "environment": self.require(str, mandatory=True), + "inventory": self.require(str), + "object_id": self.require(str) + } + self.validate_query_data(scan, scan_requirements) + scan_only_keys = [k for k in scan if k.startswith("scan_only_")] + if len(scan_only_keys) > 1: + self.bad_request("multiple scan_only_* flags found: {0}. " + "only one of them can be set." + .format(", ".join(scan_only_keys))) + + env_name = scan["environment"] + if not self.check_environment_name(env_name): + self.bad_request("unkown environment: " + env_name) + + scan["scan_completed"] = False + scan["submit_timestamp"] = datetime.now() + self.write(scan, self.COLLECTION) + self.set_successful_response(resp, + {"message": "created a new scan for " + "environment {0}" + .format(env_name)}, + "201") + + def build_query(self, filters): + query = {} + filters_keys = ["status"] + self.update_query_with_filters(filters, filters_keys, query) + base_object = filters.get("base_object") + if base_object: + query['object_id'] = base_object + _id = filters.get("id") + if _id: + query['_id'] = _id + query['environment'] = filters['env_name'] + return query diff --git a/app/api/responders/resource/scheduled_scans.py b/app/api/responders/resource/scheduled_scans.py new file mode 100644 index 0000000..0588cd0 --- /dev/null +++ b/app/api/responders/resource/scheduled_scans.py @@ -0,0 +1,113 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from api.validation.data_validate import DataValidate +from api.responders.responder_base import ResponderBase +from bson.objectid import ObjectId +from datetime import datetime + + +class ScheduledScans(ResponderBase): + def __init__(self): + super().__init__() + self.COLLECTION = "scheduled_scans" + self.ID = "_id" + self.PROJECTION = { + self.ID: True, + "environment": True, + "scheduled_timestamp": True, + "freq": True + } + self.SCAN_FREQ = [ + "YEARLY", + "MONTHLY", + "WEEKLY", + "DAILY", + "HOURLY" + ] + + def on_get(self, req, resp): + self.log.debug("Getting scheduled scans") + filters = self.parse_query_params(req) + + filters_requirements = { + "environment": self.require(str, mandatory=True), + "id": self.require(ObjectId, True), + "freq": self.require(str, False, + DataValidate.LIST, self.SCAN_FREQ), + "page": self.require(int, True), + "page_size": self.require(int, True) + } + + self.validate_query_data(filters, filters_requirements) + page, page_size = self.get_pagination(filters) + + query = self.build_query(filters) + if self.ID in query: + scheduled_scan = self.get_object_by_id(self.COLLECTION, query, + [ObjectId, datetime], + self.ID) + self.set_successful_response(resp, scheduled_scan) + else: + scheduled_scan_ids = self.get_objects_list(self.COLLECTION, query, + page, page_size, + self.PROJECTION, + [datetime]) + self.set_successful_response(resp, + {"scheduled_scans": scheduled_scan_ids}) + + def on_post(self, req, resp): + self.log.debug("Posting new scheduled scan") + error, scheduled_scan = self.get_content_from_request(req) + if error: + self.bad_request(error) + + log_levels = self.get_constants_by_name("log_levels") + scheduled_scan_requirements = { + "environment": self.require(str, mandatory=True), + "scan_only_links": self.require(bool, True), + "scan_only_cliques": self.require(bool, True), + "scan_only_inventory": self.require(bool, True), + "freq": self.require(str, validate=DataValidate.LIST, + requirement=self.SCAN_FREQ, + mandatory=True), + "log_level": self.require(str, + validate=DataValidate.LIST, + requirement=log_levels), + "clear": self.require(bool, True), + "submit_timestamp": self.require(str, mandatory=True) + } + self.validate_query_data(scheduled_scan, scheduled_scan_requirements) + self.check_and_convert_datetime("submit_timestamp", scheduled_scan) + scan_only_keys = [k for k in scheduled_scan if k.startswith("scan_only_")] + if len(scan_only_keys) > 1: + self.bad_request("multiple scan_only_* flags found: {0}. " + "only one of them can be set." + .format(", ".join(scan_only_keys))) + + env_name = scheduled_scan["environment"] + if not self.check_environment_name(env_name): + self.bad_request("unkown environment: " + env_name) + + self.write(scheduled_scan, self.COLLECTION) + self.set_successful_response(resp, + {"message": "created a new scheduled scan for " + "environment {0}" + .format(env_name)}, + "201") + + def build_query(self, filters): + query = {} + filters_keys = ["freq", "environment"] + self.update_query_with_filters(filters, filters_keys, query) + + _id = filters.get("id") + if _id: + query["_id"] = _id + return query diff --git a/app/api/responders/responder_base.py b/app/api/responders/responder_base.py new file mode 100644 index 0000000..479a897 --- /dev/null +++ b/app/api/responders/responder_base.py @@ -0,0 +1,223 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import json +import re +from urllib import parse + +from dateutil import parser +from pymongo import errors + +from api.exceptions import exceptions +from api.validation.data_validate import DataValidate +from utils.dict_naming_converter import DictNamingConverter +from utils.inventory_mgr import InventoryMgr +from utils.logging.full_logger import FullLogger +from utils.string_utils import jsonify, stringify_object_values_by_types + + +class ResponderBase(DataValidate, DictNamingConverter): + UNCHANGED_COLLECTIONS = ["monitoring_config_templates", + "environments_config", + "messages", + "scheduled_scans"] + + def __init__(self): + super().__init__() + self.log = FullLogger() + self.inv = InventoryMgr() + + def set_successful_response(self, resp, body="", status="200"): + if not isinstance(body, str): + try: + body = jsonify(body) + except Exception as e: + self.log.exception(e) + raise ValueError("The response body should be a string") + resp.status = status + resp.body = body + + def set_error_response(self, title="", code="", message="", body=""): + if body: + raise exceptions.CalipsoApiException(code, body, message) + body = { + "error": { + "message": message, + "code": code, + "title": title + } + } + body = jsonify(body) + raise exceptions.CalipsoApiException(code, body, message) + + def not_found(self, message="Requested resource not found"): + self.set_error_response("Not Found", "404", message) + + def conflict(self, + message="The posted data conflicts with the existing data"): + self.set_error_response("Conflict", "409", message) + + def bad_request(self, message="Invalid request content"): + self.set_error_response("Bad Request", "400", message) + + def unauthorized(self, message="Request requires authorization"): + self.set_error_response("Unauthorized", "401", message) + + def validate_query_data(self, data, data_requirements, + additional_key_reg=None, + can_be_empty_keys=[]): + error_message = self.validate_data(data, data_requirements, + additional_key_reg, + can_be_empty_keys) + if error_message: + self.bad_request(error_message) + + def check_and_convert_datetime(self, time_key, data): + time = data.get(time_key) + + if time: + time = time.replace(' ', '+') + try: + data[time_key] = parser.parse(time) + except Exception: + self.bad_request("{0} must follow ISO 8610 date and time format," + "YYYY-MM-DDThh:mm:ss.sss+hhmm".format(time_key)) + + def check_environment_name(self, env_name): + query = {"name": env_name} + objects = self.read("environments_config", query) + if not objects: + return False + return True + + def get_object_by_id(self, collection, query, stringify_types, id): + objs = self.read(collection, query) + if not objs: + env_name = query.get("environment") + if env_name and \ + not self.check_environment_name(env_name): + self.bad_request("unkown environment: " + env_name) + self.not_found() + obj = objs[0] + stringify_object_values_by_types(obj, stringify_types) + if id is "_id": + obj['id'] = obj.get('_id') + return obj + + def get_objects_list(self, collection, query, page, page_size, + projection, stringify_types=None): + objects = self.read(collection, query, projection, page, page_size) + if not objects: + env_name = query.get("environment") + if env_name and \ + not self.check_environment_name(env_name): + self.bad_request("unkown environment: " + env_name) + self.not_found() + for obj in objects: + if "id" not in obj and "_id" in obj: + obj["id"] = str(obj["_id"]) + if "_id" in obj: + del obj["_id"] + if stringify_types: + stringify_object_values_by_types(objects, stringify_types) + + return objects + + def parse_query_params(self, req): + query_string = req.query_string + if not query_string: + return {} + try: + query_params = dict((k, v if len(v) > 1 else v[0]) + for k, v in + parse.parse_qs(query_string, + keep_blank_values=True, + strict_parsing=True).items()) + return query_params + except ValueError as e: + self.bad_request(str("Invalid query string: {0}".format(str(e)))) + + def replace_colon_with_dot(self, s): + return s.replace(':', '.') + + def get_pagination(self, filters): + page_size = filters.get('page_size', 1000) + page = filters.get('page', 0) + return page, page_size + + def update_query_with_filters(self, filters, filters_keys, query): + for filter_key in filters_keys: + filter = filters.get(filter_key) + if filter is not None: + query.update({filter_key: filter}) + + def get_content_from_request(self, req): + error = "" + content = "" + if not req.content_length: + error = "No data found in the request body" + return error, content + + data = req.stream.read() + content_string = data.decode() + try: + content = json.loads(content_string) + if not isinstance(content, dict): + error = "The data in the request body must be an object" + except Exception: + error = "The request can not be fulfilled due to bad syntax" + + return error, content + + def get_collection_by_name(self, name): + if name in self.UNCHANGED_COLLECTIONS: + return self.inv.db[name] + return self.inv.collections[name] + + def get_constants_by_name(self, name): + constants = self.get_collection_by_name("constants").\ + find_one({"name": name}) + # consts = [d['value'] for d in constants['data']] + consts = [] + if not constants: + self.log.error('constant type: ' + name + + 'no constants exists') + return consts + for d in constants['data']: + try: + consts.append(d['value']) + except KeyError: + self.log.error('constant type: ' + name + + ': no "value" key for data: ' + str(d)) + return consts + + def read(self, collection, matches={}, projection=None, skip=0, limit=1000): + collection = self.get_collection_by_name(collection) + skip *= limit + query = collection.find(matches, projection).skip(skip).limit(limit) + return list(query) + + def write(self, document, collection="inventory"): + try: + self.get_collection_by_name(collection).\ + insert_one(document) + except errors.DuplicateKeyError as e: + self.conflict("The key value ({0}) already exists". + format(', '. + join(self.get_duplicate_key_values(e.details['errmsg'])))) + except errors.WriteError as e: + self.bad_request('Failed to create resource for {0}'.format(str(e))) + + def get_duplicate_key_values(self, err_msg): + return ["'{0}'".format(key) for key in re.findall(r'"([^",]+)"', err_msg)] + + def aggregate(self, pipeline, collection): + collection = self.get_collection_by_name(collection) + data = collection.aggregate(pipeline) + return list(data) diff --git a/app/api/server.py b/app/api/server.py new file mode 100755 index 0000000..3fef46e --- /dev/null +++ b/app/api/server.py @@ -0,0 +1,74 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import argparse + +from gunicorn.app.base import BaseApplication +from gunicorn.six import iteritems + +from api.app import App + + +# This class is used to integrate Gunicorn with falcon application +class StandaloneApplication(BaseApplication): + def __init__(self, app, options=None): + self.options = options + self.application = app + super().__init__() + + def load_config(self): + config = dict([(key, value) for key, value in iteritems(self.options) + if key in self.cfg.settings and value is not None]) + for key, value in iteritems(config): + self.cfg.set(key.lower(), value) + + def load(self): + return self.application + + +def get_args(): + parser = argparse.ArgumentParser(description="Parameters for Calipso API") + parser.add_argument("-m", "--mongo_config", nargs="?", type=str, + default="", + help="name of config file with mongo access " + "details") + parser.add_argument("--ldap_config", nargs="?", type=str, + default="", + help="name of the config file with ldap server " + "config details") + parser.add_argument("-l", "--loglevel", nargs="?", type=str, + default="INFO", + help="logging level \n(default: 'INFO')") + parser.add_argument("-b", "--bind", nargs="?", type=str, + default="127.0.0.1:8000", + help="binding address of the API server\n" + "(default 127.0.0.1:8000)") + parser.add_argument("-y", "--inventory", nargs="?", type=str, + default="inventory", + help="name of inventory collection \n" + + "(default: 'inventory')") + parser.add_argument("-t", "--token-lifetime", nargs="?", type=int, + default=86400, + help="lifetime of the token") + args = parser.parse_args() + return args + + +if __name__ == "__main__": + args = get_args() + # Gunicorn configuration + options = { + "bind": args.bind + } + app = App(args.mongo_config, + args.ldap_config, + args.loglevel, + args.inventory, + args.token_lifetime).get_app() + StandaloneApplication(app, options).run() diff --git a/app/api/validation/__init__.py b/app/api/validation/__init__.py new file mode 100644 index 0000000..1e85a2a --- /dev/null +++ b/app/api/validation/__init__.py @@ -0,0 +1,10 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + diff --git a/app/api/validation/data_validate.py b/app/api/validation/data_validate.py new file mode 100644 index 0000000..6928c4b --- /dev/null +++ b/app/api/validation/data_validate.py @@ -0,0 +1,185 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + + +from api.validation import regex + + +class DataValidate: + LIST = "list" + REGEX = "regex" + + def __init__(self): + super().__init__() + self.BOOL_CONVERSION = { + "true": True, + "1": True, + 1: True, + "false": False, + "0": False, + 0: False + } + self.TYPES_CUSTOMIZED_NAMES = { + 'str': 'string', + 'bool': 'boolean', + 'int': 'integer', + 'ObjectId': 'MongoDB ObjectId' + } + self.VALIDATE_SWITCHER = { + self.LIST: self.validate_value_in_list, + self.REGEX: regex.validate + } + + def validate_type(self, obj, t, convert_to_type): + if convert_to_type: + # user may input different values for the + # boolean True or False, convert the number or + # the string to corresponding python bool values + if t == bool: + if isinstance(obj, str): + obj = obj.lower() + if obj in self.BOOL_CONVERSION: + return self.BOOL_CONVERSION[obj] + return None + try: + obj = t(obj) + except Exception: + return None + return obj + else: + return obj if isinstance(obj, t) else None + + # get the requirement for validation + # this requirement object will be used in validate_data method + @staticmethod + def require(types, convert_to_type=False, validate=None, + requirement=None, mandatory=False, error_messages=None): + if error_messages is None: + error_messages = {} + return { + "types": types, + "convert_to_type": convert_to_type, + "validate": validate, + "requirement": requirement, + "mandatory": mandatory, + "error_messages": error_messages + } + + def validate_data(self, data, requirements, + additional_key_re=None, + can_be_empty_keys=[]): + + illegal_keys = [key for key in data.keys() + if key not in requirements.keys()] + + if additional_key_re: + illegal_keys = [key for key in illegal_keys + if not re.match(additional_key_re, key)] + + if illegal_keys: + return 'Invalid key(s): {0}'.format(' and '.join(illegal_keys)) + + for key, requirement in requirements.items(): + value = data.get(key) + error_messages = requirement['error_messages'] + + if not value and value is not False and value is not 0: + if key in data and key not in can_be_empty_keys: + return "Invalid data: value of {0} key doesn't exist ".format(key) + # check if the key is mandatory + mandatory_error = error_messages.get('mandatory') + error_message = self.mandatory_check(key, + requirement['mandatory'], + mandatory_error) + if error_message: + return error_message + continue + + # check the required types + error_message = self.types_check(requirement["types"], + requirement["convert_to_type"], + key, + value, data, + error_messages.get('types')) + if error_message: + return error_message + + # after the types check, the value of the key may be changed + # get the value again + value = data[key] + validate = requirement.get('validate') + if not validate: + continue + requirement_value = requirement.get('requirement') + # validate the data against the requirement + req_error = error_messages.get("requirement") + error_message = self.requirement_check(key, value, validate, + requirement_value, + req_error) + if error_message: + return error_message + return None + + @staticmethod + def mandatory_check(key, mandatory, error_message): + if mandatory: + return error_message if error_message \ + else "{} must be specified".format(key) + return None + + def types_check(self, requirement_types, convert_to_type, key, + value, data, error_message): + if not isinstance(requirement_types, list): + requirement_types = [requirement_types] + for requirement_type in requirement_types: + converted_val = self.validate_type( + value, requirement_type, convert_to_type + ) + if converted_val is not None: + if convert_to_type: + # value has been converted, update the data + data[key] = converted_val + return None + required_types = self.get_type_names(requirement_types) + return error_message if error_message else \ + "{0} must be {1}".format(key, " or ".join(required_types)) + + def requirement_check(self, key, value, validate, + requirement, error_message): + return self.VALIDATE_SWITCHER[validate](key, value, requirement, + error_message) + + @staticmethod + def validate_value_in_list(key, value, + required_list, error_message): + if not isinstance(value, list): + value = [value] + + if [v for v in value if v not in required_list]: + return error_message if error_message else\ + "The possible value of {0} is {1}".\ + format(key, " or ".join(required_list)) + return None + + # get customized type names from type names array + def get_type_names(self, types): + return [self.get_type_name(t) for t in types] + + # get customized type name from string <class 'type'> + def get_type_name(self, t): + t = str(t) + a = t.split(" ")[1] + type_name = a.rstrip(">").strip("'") + # strip the former module names + type_name = type_name.split('.')[-1] + if type_name in self.TYPES_CUSTOMIZED_NAMES.keys(): + type_name = self.TYPES_CUSTOMIZED_NAMES[type_name] + return type_name diff --git a/app/api/validation/regex.py b/app/api/validation/regex.py new file mode 100644 index 0000000..2684636 --- /dev/null +++ b/app/api/validation/regex.py @@ -0,0 +1,57 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + +PORT = "port number" +IP = "ipv4/ipv6 address" +HOSTNAME = "host name" +PATH = "path" + +_PORT_REGEX = re.compile('^0*(?:6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|' + '6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{1,3}|[0-9])$') + +_HOSTNAME_REGEX = re.compile('^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])' + '(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$') + +_PATH_REGEX = re.compile('^(\/){1}([^\/\0]+(\/)?)+$') + +_IPV4_REGEX = re.compile('^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}$') +_IPV6_REGEX = re.compile('^(((?=.*(::))(?!.*\3.+\3))\3?|[\dA-F]{1,4}:)([\dA-F]{1,4}(\3|:\b)|\2){5}(([\dA-F]{1,4}' + '(\3|:\b|$)|\2){2}|(((2[0-4]|1\d|[1-9])?\d|25[0-5])\.?\b){4})$') + +_REGEX_MAP = { + PORT: _PORT_REGEX, + HOSTNAME: _HOSTNAME_REGEX, + PATH: _PATH_REGEX, + IP: [_IPV4_REGEX, _IPV6_REGEX] +} + + +def validate(key, value, regex_names, error_message=None): + if not isinstance(regex_names, list): + regex_names = [regex_names] + + for regex_name in regex_names: + regexes = _REGEX_MAP[regex_name] + + if not isinstance(regexes, list): + regexes = [regexes] + + try: + value = str(value) + match_regexes = [regex for regex in regexes + if regex.match(value)] + if match_regexes: + return None + except: + pass + + return error_message if error_message else \ + '{0} must be a valid {1}'.format(key, " or ".join(regex_names)) |