aboutsummaryrefslogtreecommitdiffstats
path: root/app/discover/fetchers
diff options
context:
space:
mode:
Diffstat (limited to 'app/discover/fetchers')
-rw-r--r--app/discover/fetchers/api/api_access.py61
-rw-r--r--app/discover/fetchers/api/api_fetch_host_instances.py2
-rw-r--r--app/discover/fetchers/api/api_fetch_project_hosts.py44
-rw-r--r--app/discover/fetchers/api/api_fetch_regions.py2
-rw-r--r--app/discover/fetchers/db/db_access.py29
-rw-r--r--app/discover/fetchers/kube/__init__.py9
-rw-r--r--app/discover/fetchers/kube/kube_access.py28
-rw-r--r--app/discover/fetchers/kube/kube_fetch_namespaces.py32
8 files changed, 143 insertions, 64 deletions
diff --git a/app/discover/fetchers/api/api_access.py b/app/discover/fetchers/api/api_access.py
index f685faf..1fca202 100644
--- a/app/discover/fetchers/api/api_access.py
+++ b/app/discover/fetchers/api/api_access.py
@@ -12,21 +12,18 @@ import re
import requests
import time
-from discover.configuration import Configuration
-from discover.fetcher import Fetcher
+from utils.api_access_base import ApiAccessBase
from utils.string_utils import jsonify
-class ApiAccess(Fetcher):
+class ApiAccess(ApiAccessBase):
+
+ ADMIN_PORT = "35357"
+
subject_token = None
initialized = False
regions = {}
- config = None
- api_config = None
- host = ""
- base_url = ""
- admin_token = ""
tokens = {}
admin_endpoint = ""
admin_project = None
@@ -38,28 +35,19 @@ class ApiAccess(Fetcher):
# identity API v2 version with admin token
def __init__(self, config=None):
- super(ApiAccess, self).__init__()
- if ApiAccess.initialized:
+ super().__init__('OpenStack', config)
+ self.base_url = "http://" + self.host + ":" + self.port
+ if self.initialized:
return
- ApiAccess.config = {'OpenStack': config} if config else Configuration()
- ApiAccess.api_config = ApiAccess.config.get("OpenStack")
- host = ApiAccess.api_config.get("host", "")
- ApiAccess.host = host
- port = ApiAccess.api_config.get("port", "")
- if not (host and port):
- raise ValueError('Missing definition of host or port ' +
- 'for OpenStack API access')
- ApiAccess.base_url = "http://" + host + ":" + port
- ApiAccess.admin_token = ApiAccess.api_config.get("admin_token", "")
- ApiAccess.admin_project = ApiAccess.api_config.get("admin_project",
- "admin")
- ApiAccess.admin_endpoint = "http://" + host + ":" + "35357"
+ ApiAccess.admin_project = self.api_config.get("admin_project", "admin")
+ ApiAccess.admin_endpoint = "http://" + self.host + ":" + self.ADMIN_PORT
token = self.v2_auth_pwd(ApiAccess.admin_project)
if not token:
raise ValueError("Authentication failed. Failed to obtain token")
else:
self.subject_token = token
+ self.initialized = True
@staticmethod
def parse_time(time_str):
@@ -95,9 +83,9 @@ class ApiAccess(Fetcher):
subject_token = self.get_existing_token(project_id)
if subject_token:
return subject_token
- req_url = ApiAccess.base_url + "/v2.0/tokens"
+ req_url = self.base_url + "/v2.0/tokens"
response = requests.post(req_url, json=post_body, headers=headers,
- timeout=5)
+ timeout=self.CONNECT_TIMEOUT)
response = response.json()
ApiAccess.auth_response[project_id] = response
if 'error' in response:
@@ -120,8 +108,8 @@ class ApiAccess(Fetcher):
return token_details
def v2_auth_pwd(self, project):
- user = ApiAccess.api_config["user"]
- pwd = ApiAccess.api_config["pwd"]
+ user = self.api_config["user"]
+ pwd = self.api_config["pwd"]
post_body = {
"auth": {
"passwordCredentials": {
@@ -148,23 +136,6 @@ class ApiAccess(Fetcher):
auth_response = ApiAccess.auth_response.get('admin', {})
return auth_response
- def get_rel_url(self, relative_url, headers):
- req_url = ApiAccess.base_url + relative_url
- return self.get_url(req_url, headers)
-
- def get_url(self, req_url, headers):
- response = requests.get(req_url, headers=headers)
- if response.status_code != requests.codes.ok:
- # some error happened
- if "reason" in response:
- msg = ", reason: {}".format(response.reason)
- else:
- msg = ", response: {}".format(response.text)
- self.log.error("req_url: {} {}".format(req_url, msg))
- return None
- ret = response.json()
- return ret
-
def get_region_url(self, region_name, service):
if region_name not in self.regions:
return None
@@ -174,7 +145,7 @@ class ApiAccess(Fetcher):
return None
orig_url = s["adminURL"]
# replace host name with the host found in config
- url = re.sub(r"^([^/]+)//[^:]+", r"\1//" + ApiAccess.host, orig_url)
+ url = re.sub(r"^([^/]+)//[^:]+", r"\1//" + self.host, orig_url)
return url
# like get_region_url(), but remove everything starting from the "/v2"
diff --git a/app/discover/fetchers/api/api_fetch_host_instances.py b/app/discover/fetchers/api/api_fetch_host_instances.py
index 56cffda..bf8513a 100644
--- a/app/discover/fetchers/api/api_fetch_host_instances.py
+++ b/app/discover/fetchers/api/api_fetch_host_instances.py
@@ -18,7 +18,7 @@ class ApiFetchHostInstances(ApiAccess, DbAccess, metaclass=Singleton):
def __init__(self):
super(ApiFetchHostInstances, self).__init__()
self.inv = InventoryMgr()
- self.endpoint = ApiAccess.base_url.replace(":5000", ":8774")
+ self.endpoint = self.base_url.replace(":5000", ":8774")
self.projects = None
self.db_fetcher = DbFetchInstances()
diff --git a/app/discover/fetchers/api/api_fetch_project_hosts.py b/app/discover/fetchers/api/api_fetch_project_hosts.py
index 5b911f5..2aeb24f 100644
--- a/app/discover/fetchers/api/api_fetch_project_hosts.py
+++ b/app/discover/fetchers/api/api_fetch_project_hosts.py
@@ -11,9 +11,11 @@ import json
from discover.fetchers.api.api_access import ApiAccess
from discover.fetchers.db.db_access import DbAccess
+from discover.fetchers.cli.cli_access import CliAccess
+from utils.ssh_connection import SshError
-class ApiFetchProjectHosts(ApiAccess, DbAccess):
+class ApiFetchProjectHosts(ApiAccess, DbAccess, CliAccess):
def __init__(self):
super(ApiFetchProjectHosts, self).__init__()
@@ -107,6 +109,7 @@ class ApiFetchProjectHosts(ApiAccess, DbAccess):
s = services["nova-compute"]
if s["available"] and s["active"]:
self.add_host_type(doc, "Compute", az['zoneName'])
+ self.fetch_host_os_details(doc)
return doc
# fetch more details of network nodes from neutron DB agents table
@@ -121,7 +124,12 @@ class ApiFetchProjectHosts(ApiAccess, DbAccess):
""".format(self.neutron_db)
results = self.get_objects_list(query, "")
for r in results:
- host = hosts[r["host"]]
+ host = r["host"]
+ if host not in hosts:
+ self.log.error("host from agents table not in hosts list: {}"
+ .format(host))
+ continue
+ host = hosts[host]
host["config"] = json.loads(r["configurations"])
self.add_host_type(host, "Network", '')
@@ -136,9 +144,33 @@ class ApiFetchProjectHosts(ApiAccess, DbAccess):
for db_row in results:
doc.update(db_row)
- def add_host_type(self, doc, type, zone):
- if not type in doc["host_type"]:
- doc["host_type"].append(type)
- if type == 'Compute':
+ @staticmethod
+ def add_host_type(doc, host_type, zone):
+ if host_type not in doc["host_type"]:
+ doc["host_type"].append(host_type)
+ if host_type == 'Compute':
doc['zone'] = zone
doc['parent_id'] = zone
+
+ def fetch_host_os_details(self, doc):
+ cmd = 'cat /etc/os-release && echo "ARCHITECURE=`arch`"'
+ try:
+ lines = self.run_fetch_lines(cmd, ssh_to_host=doc['host'])
+ except SshError as e:
+ self.log.error('{}: {}', cmd, str(e))
+ os_attributes = {}
+ attributes_to_fetch = {
+ 'NAME': 'name',
+ 'VERSION': 'version',
+ 'ID': 'ID',
+ 'ID_LIKE': 'ID_LIKE',
+ 'ARCHITECURE': 'architecure'
+ }
+ for attr in attributes_to_fetch:
+ matches = [l for l in lines if l.startswith(attr + '=')]
+ if matches:
+ line = matches[0]
+ attr_name = attributes_to_fetch[attr]
+ os_attributes[attr_name] = line[line.index('=')+1:].strip('"')
+ if os_attributes:
+ doc['OS'] = os_attributes
diff --git a/app/discover/fetchers/api/api_fetch_regions.py b/app/discover/fetchers/api/api_fetch_regions.py
index 23a3736..4e83b01 100644
--- a/app/discover/fetchers/api/api_fetch_regions.py
+++ b/app/discover/fetchers/api/api_fetch_regions.py
@@ -13,7 +13,7 @@ from discover.fetchers.api.api_access import ApiAccess
class ApiFetchRegions(ApiAccess):
def __init__(self):
super(ApiFetchRegions, self).__init__()
- self.endpoint = ApiAccess.base_url
+ self.endpoint = self.base_url
def get(self, regions_folder_id):
token = self.v2_auth_pwd(self.admin_project)
diff --git a/app/discover/fetchers/db/db_access.py b/app/discover/fetchers/db/db_access.py
index 090ab84..5ff49d5 100644
--- a/app/discover/fetchers/db/db_access.py
+++ b/app/discover/fetchers/db/db_access.py
@@ -38,8 +38,7 @@ class DbAccess(Fetcher):
conn = None
query_count_per_con = 0
- # connection timeout set to 30 seconds,
- # due to problems over long connections
+ # connection timeout set to 5 seconds
TIMEOUT = 5
def __init__(self, mysql_config=None):
@@ -47,6 +46,9 @@ class DbAccess(Fetcher):
self.config = {'mysql': mysql_config} if mysql_config \
else Configuration()
self.conf = self.config.get("mysql")
+ self.connect_timeout = int(self.conf['connect_timeout']) \
+ if 'connect_timeout' in self.conf \
+ else self.TIMEOUT
self.connect_to_db()
self.neutron_db = self.get_neutron_db_name()
@@ -55,16 +57,18 @@ class DbAccess(Fetcher):
return
try:
connector = mysql.connector
- DbAccess.conn = connector.connect(host=_host, port=_port,
- connection_timeout=self.TIMEOUT,
- user=_user,
- password=_pwd,
- database=_database,
- raise_on_warnings=True)
+ conn = connector.connect(host=_host, port=_port,
+ connection_timeout=self.connect_timeout,
+ user=_user,
+ password=_pwd,
+ database=_database,
+ raise_on_warnings=True)
+ DbAccess.conn = conn
DbAccess.conn.ping(True) # auto-reconnect if necessary
except Exception as e:
- self.log.critical("failed to connect to MySQL DB: {}"
- .format(str(e)))
+ msg = "failed to connect to MySQL DB: {}".format(str(e))
+ self.log.critical(msg)
+ raise ScanError(msg)
return
DbAccess.query_count_per_con = 0
@@ -93,8 +97,11 @@ class DbAccess(Fetcher):
DbAccess.conn = None
self.conf = self.config.get("mysql")
cnf = self.conf
+ pwd = cnf.get('pwd', '')
+ if not pwd:
+ raise ScanError('db_access: attribute pwd is missing')
self.db_connect(cnf.get('host', ''), cnf.get('port', ''),
- cnf.get('user', ''), cnf.get('pwd', ''),
+ cnf.get('user', ''), pwd,
cnf.get('schema', 'nova'))
@with_cursor
diff --git a/app/discover/fetchers/kube/__init__.py b/app/discover/fetchers/kube/__init__.py
new file mode 100644
index 0000000..b0637e9
--- /dev/null
+++ b/app/discover/fetchers/kube/__init__.py
@@ -0,0 +1,9 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
diff --git a/app/discover/fetchers/kube/kube_access.py b/app/discover/fetchers/kube/kube_access.py
new file mode 100644
index 0000000..38bb978
--- /dev/null
+++ b/app/discover/fetchers/kube/kube_access.py
@@ -0,0 +1,28 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from kubernetes.client import Configuration as KubConf, CoreV1Api
+
+from utils.api_access_base import ApiAccessBase
+
+
+class KubeAccess(ApiAccessBase):
+
+ def __init__(self, config=None):
+ super().__init__('Kubernetes', config)
+ self.base_url = 'https://{}:{}'.format(self.host, self.port)
+ self.bearer_token = self.api_config.get('token', '')
+ conf = KubConf()
+ conf.host = self.base_url
+ conf.user = self.api_config.get('user')
+ conf.api_key_prefix['authorization'] = 'Bearer'
+ conf.api_key['authorization'] = self.bearer_token
+ conf.verify_ssl = False
+ self.api = CoreV1Api()
+
diff --git a/app/discover/fetchers/kube/kube_fetch_namespaces.py b/app/discover/fetchers/kube/kube_fetch_namespaces.py
new file mode 100644
index 0000000..951ddb8
--- /dev/null
+++ b/app/discover/fetchers/kube/kube_fetch_namespaces.py
@@ -0,0 +1,32 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from discover.fetchers.kube.kube_access import KubeAccess
+
+
+class KubeFetchNamespaces(KubeAccess):
+
+ def __init__(self, config=None):
+ super().__init__(config)
+
+ def get(self, object_id):
+ namespaces = self.api.list_namespace()
+ return [self.get_namespace(i) for i in namespaces.items]
+
+ @staticmethod
+ def get_namespace(namespace):
+ attrs = ['creation_timestamp', 'self_link', 'uid']
+ namespace_details = {
+ 'name': namespace.metadata.name,
+ 'status': namespace.status.phase
+ }
+ namespace_details.update({x: getattr(namespace.metadata, x, '')
+ for x in attrs})
+ namespace_details['id'] = namespace_details['uid']
+ return namespace_details