From 98c3ac7c859e34fe60d061b9ca591aba429e4118 Mon Sep 17 00:00:00 2001 From: Koren Lev Date: Mon, 18 Dec 2017 19:16:16 +0200 Subject: release 1.2 + new tagging Change-Id: I1e876451ec4a330f458dd57adadb15e39969b225 Signed-off-by: Koren Lev --- app/discover/fetchers/api/api_access.py | 61 ++++++---------------- .../fetchers/api/api_fetch_host_instances.py | 2 +- .../fetchers/api/api_fetch_project_hosts.py | 44 +++++++++++++--- app/discover/fetchers/api/api_fetch_regions.py | 2 +- app/discover/fetchers/db/db_access.py | 29 ++++++---- app/discover/fetchers/kube/__init__.py | 9 ++++ app/discover/fetchers/kube/kube_access.py | 28 ++++++++++ .../fetchers/kube/kube_fetch_namespaces.py | 32 ++++++++++++ 8 files changed, 143 insertions(+), 64 deletions(-) create mode 100644 app/discover/fetchers/kube/__init__.py create mode 100644 app/discover/fetchers/kube/kube_access.py create mode 100644 app/discover/fetchers/kube/kube_fetch_namespaces.py (limited to 'app/discover/fetchers') diff --git a/app/discover/fetchers/api/api_access.py b/app/discover/fetchers/api/api_access.py index f685faf..1fca202 100644 --- a/app/discover/fetchers/api/api_access.py +++ b/app/discover/fetchers/api/api_access.py @@ -12,21 +12,18 @@ import re import requests import time -from discover.configuration import Configuration -from discover.fetcher import Fetcher +from utils.api_access_base import ApiAccessBase from utils.string_utils import jsonify -class ApiAccess(Fetcher): +class ApiAccess(ApiAccessBase): + + ADMIN_PORT = "35357" + subject_token = None initialized = False regions = {} - config = None - api_config = None - host = "" - base_url = "" - admin_token = "" tokens = {} admin_endpoint = "" admin_project = None @@ -38,28 +35,19 @@ class ApiAccess(Fetcher): # identity API v2 version with admin token def __init__(self, config=None): - super(ApiAccess, self).__init__() - if ApiAccess.initialized: + super().__init__('OpenStack', config) + self.base_url = "http://" + self.host + ":" + self.port + if self.initialized: return - ApiAccess.config = {'OpenStack': config} if config else Configuration() - ApiAccess.api_config = ApiAccess.config.get("OpenStack") - host = ApiAccess.api_config.get("host", "") - ApiAccess.host = host - port = ApiAccess.api_config.get("port", "") - if not (host and port): - raise ValueError('Missing definition of host or port ' + - 'for OpenStack API access') - ApiAccess.base_url = "http://" + host + ":" + port - ApiAccess.admin_token = ApiAccess.api_config.get("admin_token", "") - ApiAccess.admin_project = ApiAccess.api_config.get("admin_project", - "admin") - ApiAccess.admin_endpoint = "http://" + host + ":" + "35357" + ApiAccess.admin_project = self.api_config.get("admin_project", "admin") + ApiAccess.admin_endpoint = "http://" + self.host + ":" + self.ADMIN_PORT token = self.v2_auth_pwd(ApiAccess.admin_project) if not token: raise ValueError("Authentication failed. Failed to obtain token") else: self.subject_token = token + self.initialized = True @staticmethod def parse_time(time_str): @@ -95,9 +83,9 @@ class ApiAccess(Fetcher): subject_token = self.get_existing_token(project_id) if subject_token: return subject_token - req_url = ApiAccess.base_url + "/v2.0/tokens" + req_url = self.base_url + "/v2.0/tokens" response = requests.post(req_url, json=post_body, headers=headers, - timeout=5) + timeout=self.CONNECT_TIMEOUT) response = response.json() ApiAccess.auth_response[project_id] = response if 'error' in response: @@ -120,8 +108,8 @@ class ApiAccess(Fetcher): return token_details def v2_auth_pwd(self, project): - user = ApiAccess.api_config["user"] - pwd = ApiAccess.api_config["pwd"] + user = self.api_config["user"] + pwd = self.api_config["pwd"] post_body = { "auth": { "passwordCredentials": { @@ -148,23 +136,6 @@ class ApiAccess(Fetcher): auth_response = ApiAccess.auth_response.get('admin', {}) return auth_response - def get_rel_url(self, relative_url, headers): - req_url = ApiAccess.base_url + relative_url - return self.get_url(req_url, headers) - - def get_url(self, req_url, headers): - response = requests.get(req_url, headers=headers) - if response.status_code != requests.codes.ok: - # some error happened - if "reason" in response: - msg = ", reason: {}".format(response.reason) - else: - msg = ", response: {}".format(response.text) - self.log.error("req_url: {} {}".format(req_url, msg)) - return None - ret = response.json() - return ret - def get_region_url(self, region_name, service): if region_name not in self.regions: return None @@ -174,7 +145,7 @@ class ApiAccess(Fetcher): return None orig_url = s["adminURL"] # replace host name with the host found in config - url = re.sub(r"^([^/]+)//[^:]+", r"\1//" + ApiAccess.host, orig_url) + url = re.sub(r"^([^/]+)//[^:]+", r"\1//" + self.host, orig_url) return url # like get_region_url(), but remove everything starting from the "/v2" diff --git a/app/discover/fetchers/api/api_fetch_host_instances.py b/app/discover/fetchers/api/api_fetch_host_instances.py index 56cffda..bf8513a 100644 --- a/app/discover/fetchers/api/api_fetch_host_instances.py +++ b/app/discover/fetchers/api/api_fetch_host_instances.py @@ -18,7 +18,7 @@ class ApiFetchHostInstances(ApiAccess, DbAccess, metaclass=Singleton): def __init__(self): super(ApiFetchHostInstances, self).__init__() self.inv = InventoryMgr() - self.endpoint = ApiAccess.base_url.replace(":5000", ":8774") + self.endpoint = self.base_url.replace(":5000", ":8774") self.projects = None self.db_fetcher = DbFetchInstances() diff --git a/app/discover/fetchers/api/api_fetch_project_hosts.py b/app/discover/fetchers/api/api_fetch_project_hosts.py index 5b911f5..2aeb24f 100644 --- a/app/discover/fetchers/api/api_fetch_project_hosts.py +++ b/app/discover/fetchers/api/api_fetch_project_hosts.py @@ -11,9 +11,11 @@ import json from discover.fetchers.api.api_access import ApiAccess from discover.fetchers.db.db_access import DbAccess +from discover.fetchers.cli.cli_access import CliAccess +from utils.ssh_connection import SshError -class ApiFetchProjectHosts(ApiAccess, DbAccess): +class ApiFetchProjectHosts(ApiAccess, DbAccess, CliAccess): def __init__(self): super(ApiFetchProjectHosts, self).__init__() @@ -107,6 +109,7 @@ class ApiFetchProjectHosts(ApiAccess, DbAccess): s = services["nova-compute"] if s["available"] and s["active"]: self.add_host_type(doc, "Compute", az['zoneName']) + self.fetch_host_os_details(doc) return doc # fetch more details of network nodes from neutron DB agents table @@ -121,7 +124,12 @@ class ApiFetchProjectHosts(ApiAccess, DbAccess): """.format(self.neutron_db) results = self.get_objects_list(query, "") for r in results: - host = hosts[r["host"]] + host = r["host"] + if host not in hosts: + self.log.error("host from agents table not in hosts list: {}" + .format(host)) + continue + host = hosts[host] host["config"] = json.loads(r["configurations"]) self.add_host_type(host, "Network", '') @@ -136,9 +144,33 @@ class ApiFetchProjectHosts(ApiAccess, DbAccess): for db_row in results: doc.update(db_row) - def add_host_type(self, doc, type, zone): - if not type in doc["host_type"]: - doc["host_type"].append(type) - if type == 'Compute': + @staticmethod + def add_host_type(doc, host_type, zone): + if host_type not in doc["host_type"]: + doc["host_type"].append(host_type) + if host_type == 'Compute': doc['zone'] = zone doc['parent_id'] = zone + + def fetch_host_os_details(self, doc): + cmd = 'cat /etc/os-release && echo "ARCHITECURE=`arch`"' + try: + lines = self.run_fetch_lines(cmd, ssh_to_host=doc['host']) + except SshError as e: + self.log.error('{}: {}', cmd, str(e)) + os_attributes = {} + attributes_to_fetch = { + 'NAME': 'name', + 'VERSION': 'version', + 'ID': 'ID', + 'ID_LIKE': 'ID_LIKE', + 'ARCHITECURE': 'architecure' + } + for attr in attributes_to_fetch: + matches = [l for l in lines if l.startswith(attr + '=')] + if matches: + line = matches[0] + attr_name = attributes_to_fetch[attr] + os_attributes[attr_name] = line[line.index('=')+1:].strip('"') + if os_attributes: + doc['OS'] = os_attributes diff --git a/app/discover/fetchers/api/api_fetch_regions.py b/app/discover/fetchers/api/api_fetch_regions.py index 23a3736..4e83b01 100644 --- a/app/discover/fetchers/api/api_fetch_regions.py +++ b/app/discover/fetchers/api/api_fetch_regions.py @@ -13,7 +13,7 @@ from discover.fetchers.api.api_access import ApiAccess class ApiFetchRegions(ApiAccess): def __init__(self): super(ApiFetchRegions, self).__init__() - self.endpoint = ApiAccess.base_url + self.endpoint = self.base_url def get(self, regions_folder_id): token = self.v2_auth_pwd(self.admin_project) diff --git a/app/discover/fetchers/db/db_access.py b/app/discover/fetchers/db/db_access.py index 090ab84..5ff49d5 100644 --- a/app/discover/fetchers/db/db_access.py +++ b/app/discover/fetchers/db/db_access.py @@ -38,8 +38,7 @@ class DbAccess(Fetcher): conn = None query_count_per_con = 0 - # connection timeout set to 30 seconds, - # due to problems over long connections + # connection timeout set to 5 seconds TIMEOUT = 5 def __init__(self, mysql_config=None): @@ -47,6 +46,9 @@ class DbAccess(Fetcher): self.config = {'mysql': mysql_config} if mysql_config \ else Configuration() self.conf = self.config.get("mysql") + self.connect_timeout = int(self.conf['connect_timeout']) \ + if 'connect_timeout' in self.conf \ + else self.TIMEOUT self.connect_to_db() self.neutron_db = self.get_neutron_db_name() @@ -55,16 +57,18 @@ class DbAccess(Fetcher): return try: connector = mysql.connector - DbAccess.conn = connector.connect(host=_host, port=_port, - connection_timeout=self.TIMEOUT, - user=_user, - password=_pwd, - database=_database, - raise_on_warnings=True) + conn = connector.connect(host=_host, port=_port, + connection_timeout=self.connect_timeout, + user=_user, + password=_pwd, + database=_database, + raise_on_warnings=True) + DbAccess.conn = conn DbAccess.conn.ping(True) # auto-reconnect if necessary except Exception as e: - self.log.critical("failed to connect to MySQL DB: {}" - .format(str(e))) + msg = "failed to connect to MySQL DB: {}".format(str(e)) + self.log.critical(msg) + raise ScanError(msg) return DbAccess.query_count_per_con = 0 @@ -93,8 +97,11 @@ class DbAccess(Fetcher): DbAccess.conn = None self.conf = self.config.get("mysql") cnf = self.conf + pwd = cnf.get('pwd', '') + if not pwd: + raise ScanError('db_access: attribute pwd is missing') self.db_connect(cnf.get('host', ''), cnf.get('port', ''), - cnf.get('user', ''), cnf.get('pwd', ''), + cnf.get('user', ''), pwd, cnf.get('schema', 'nova')) @with_cursor diff --git a/app/discover/fetchers/kube/__init__.py b/app/discover/fetchers/kube/__init__.py new file mode 100644 index 0000000..b0637e9 --- /dev/null +++ b/app/discover/fetchers/kube/__init__.py @@ -0,0 +1,9 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### diff --git a/app/discover/fetchers/kube/kube_access.py b/app/discover/fetchers/kube/kube_access.py new file mode 100644 index 0000000..38bb978 --- /dev/null +++ b/app/discover/fetchers/kube/kube_access.py @@ -0,0 +1,28 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from kubernetes.client import Configuration as KubConf, CoreV1Api + +from utils.api_access_base import ApiAccessBase + + +class KubeAccess(ApiAccessBase): + + def __init__(self, config=None): + super().__init__('Kubernetes', config) + self.base_url = 'https://{}:{}'.format(self.host, self.port) + self.bearer_token = self.api_config.get('token', '') + conf = KubConf() + conf.host = self.base_url + conf.user = self.api_config.get('user') + conf.api_key_prefix['authorization'] = 'Bearer' + conf.api_key['authorization'] = self.bearer_token + conf.verify_ssl = False + self.api = CoreV1Api() + diff --git a/app/discover/fetchers/kube/kube_fetch_namespaces.py b/app/discover/fetchers/kube/kube_fetch_namespaces.py new file mode 100644 index 0000000..951ddb8 --- /dev/null +++ b/app/discover/fetchers/kube/kube_fetch_namespaces.py @@ -0,0 +1,32 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.kube.kube_access import KubeAccess + + +class KubeFetchNamespaces(KubeAccess): + + def __init__(self, config=None): + super().__init__(config) + + def get(self, object_id): + namespaces = self.api.list_namespace() + return [self.get_namespace(i) for i in namespaces.items] + + @staticmethod + def get_namespace(namespace): + attrs = ['creation_timestamp', 'self_link', 'uid'] + namespace_details = { + 'name': namespace.metadata.name, + 'status': namespace.status.phase + } + namespace_details.update({x: getattr(namespace.metadata, x, '') + for x in attrs}) + namespace_details['id'] = namespace_details['uid'] + return namespace_details -- cgit 1.2.3-korg