summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTim Rozet <trozet@redhat.com>2017-11-27 15:22:25 -0500
committerTim Rozet <trozet@redhat.com>2018-04-04 16:03:49 -0400
commit382fa452e0c9a8189911f615416e1b24badaf5e4 (patch)
tree41973e70a7b054b4a2297bb45918430c0614c9ee
parentb047099b609eac330486f23913e8046d0f22c1ab (diff)
Adds the ability to fetch logs from deployment
Usage: opnfv-pyutil --fetch-logs python3 utils.py --fetch-logs --lib-dir ../lib Eventually all utils.sh functions will be migrated here. Note there is no support here for containers. Will be added later. Change-Id: I223b8592ad09e0370e287ee2801072db31f9aa12 Signed-off-by: Tim Rozet <trozet@redhat.com>
-rw-r--r--apex/common/constants.py2
-rw-r--r--apex/deploy.py13
-rw-r--r--apex/undercloud/undercloud.py19
-rw-r--r--apex/utils.py107
-rw-r--r--build/rpm_specs/opnfv-apex-common.spec3
-rw-r--r--lib/ansible/playbooks/fetch_overcloud_logs.yml25
-rw-r--r--lib/ansible/playbooks/fetch_overcloud_nodes.yml13
-rw-r--r--setup.cfg1
8 files changed, 171 insertions, 12 deletions
diff --git a/apex/common/constants.py b/apex/common/constants.py
index 0aa6a6c..4f72b08 100644
--- a/apex/common/constants.py
+++ b/apex/common/constants.py
@@ -16,7 +16,7 @@ STORAGE_NETWORK = 'storage'
API_NETWORK = 'api'
CONTROLLER = 'controller'
COMPUTE = 'compute'
-
+ANSIBLE_PATH = 'ansible/playbooks'
OPNFV_NETWORK_TYPES = [ADMIN_NETWORK, TENANT_NETWORK, EXTERNAL_NETWORK,
STORAGE_NETWORK, API_NETWORK]
DNS_SERVERS = ["8.8.8.8", "8.8.4.4"]
diff --git a/apex/deploy.py b/apex/deploy.py
index 7bc5568..4facade 100644
--- a/apex/deploy.py
+++ b/apex/deploy.py
@@ -39,7 +39,6 @@ from apex.overcloud import config as oc_cfg
from apex.overcloud import deploy as oc_deploy
APEX_TEMP_DIR = tempfile.mkdtemp(prefix='apex_tmp')
-ANSIBLE_PATH = 'ansible/playbooks'
SDN_IMAGE = 'overcloud-full-opendaylight.qcow2'
@@ -309,7 +308,7 @@ def main():
'virsh_enabled_networks': net_settings.enabled_network_list
}
utils.run_ansible(ansible_args,
- os.path.join(args.lib_dir, ANSIBLE_PATH,
+ os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
'deploy_dependencies.yml'))
uc_external = False
if 'external' in net_settings.enabled_network_list:
@@ -397,7 +396,7 @@ def main():
args.deploy_dir, APEX_TEMP_DIR)
# Install Undercloud
undercloud.configure(net_settings, deploy_settings,
- os.path.join(args.lib_dir, ANSIBLE_PATH,
+ os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
'configure_undercloud.yml'),
APEX_TEMP_DIR, virtual_oc=args.virtual)
@@ -433,7 +432,7 @@ def main():
os.path.basename(opnfv_env),
net_data=net_data)
# Prepare undercloud with containers
- docker_playbook = os.path.join(args.lib_dir, ANSIBLE_PATH,
+ docker_playbook = os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
'prepare_overcloud_containers.yml')
if ds_opts['containers']:
ceph_version = constants.CEPH_VERSION_MAP[ds_opts['os_version']]
@@ -466,7 +465,7 @@ def main():
os.remove(os.path.join(APEX_TEMP_DIR, 'overcloud-full.qcow2'))
raise
- deploy_playbook = os.path.join(args.lib_dir, ANSIBLE_PATH,
+ deploy_playbook = os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
'deploy_overcloud.yml')
virt_env = 'virtual-environment.yaml'
bm_env = 'baremetal-environment.yaml'
@@ -542,7 +541,7 @@ def main():
else:
deploy_vars['overcloudrc_files'] = ['overcloudrc']
- post_undercloud = os.path.join(args.lib_dir, ANSIBLE_PATH,
+ post_undercloud = os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
'post_deploy_undercloud.yml')
logging.info("Executing post deploy configuration undercloud playbook")
try:
@@ -561,7 +560,7 @@ def main():
deploy_vars['sriov'] = ds_opts.get('sriov')
# TODO(trozet): pull all logs and store in tmp dir in overcloud
# playbook
- post_overcloud = os.path.join(args.lib_dir, ANSIBLE_PATH,
+ post_overcloud = os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
'post_deploy_overcloud.yml')
# Run per overcloud node
for node, ip in deploy_vars['overcloud_nodes'].items():
diff --git a/apex/undercloud/undercloud.py b/apex/undercloud/undercloud.py
index e799d37..2972351 100644
--- a/apex/undercloud/undercloud.py
+++ b/apex/undercloud/undercloud.py
@@ -77,15 +77,26 @@ class Undercloud:
self.inject_auth()
self._update_delorean_repo()
- def _set_ip(self):
- ip_out = self.vm.interfaceAddresses(
+ @staticmethod
+ def _get_ip(vm):
+ ip_out = vm.interfaceAddresses(
libvirt.VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE, 0)
if ip_out:
for (name, val) in ip_out.items():
for ipaddr in val['addrs']:
if ipaddr['type'] == libvirt.VIR_IP_ADDR_TYPE_IPV4:
- self.ip = ipaddr['addr']
- return True
+ return ipaddr['addr']
+
+ def _set_ip(self):
+ ip = self._get_ip(self.vm)
+ if ip:
+ self.ip = ip
+ return True
+
+ @staticmethod
+ def get_ip():
+ vm = Undercloud._get_vm()
+ return Undercloud._get_ip(vm)
def start(self):
"""
diff --git a/apex/utils.py b/apex/utils.py
new file mode 100644
index 0000000..cea25fe
--- /dev/null
+++ b/apex/utils.py
@@ -0,0 +1,107 @@
+##############################################################################
+# Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# TODO(trozet) migrate rest of utils.sh here
+
+import argparse
+import datetime
+import logging
+import os
+import sys
+import tempfile
+
+from apex.common import constants
+from apex.common import parsers
+from apex.undercloud import undercloud as uc_lib
+from apex.common import utils
+
+VALID_UTILS = ['fetch_logs']
+START_TIME = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M")
+APEX_TEMP_DIR = tempfile.mkdtemp(prefix="apex-logs-{}-".format(START_TIME))
+
+
+def fetch_logs(args):
+ uc_ip = uc_lib.Undercloud.get_ip()
+ if not uc_ip:
+ raise Exception('No Undercloud IP found')
+ logging.info("Undercloud IP is: {}".format(uc_ip))
+ fetch_vars = dict()
+ fetch_vars['stackrc'] = 'source /home/stack/stackrc'
+ fetch_vars['apex_temp_dir'] = APEX_TEMP_DIR
+ fetch_playbook = os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
+ 'fetch_overcloud_nodes.yml')
+ try:
+ utils.run_ansible(fetch_vars, fetch_playbook, host=uc_ip,
+ user='stack', tmp_dir=APEX_TEMP_DIR)
+ logging.info("Retrieved overcloud nodes info")
+ except Exception:
+ logging.error("Failed to retrieve overcloud nodes. Please check log")
+ raise
+ nova_output = os.path.join(APEX_TEMP_DIR, 'nova_output')
+ fetch_vars['overcloud_nodes'] = parsers.parse_nova_output(nova_output)
+ fetch_vars['SSH_OPTIONS'] = '-o StrictHostKeyChecking=no -o ' \
+ 'GlobalKnownHostsFile=/dev/null -o ' \
+ 'UserKnownHostsFile=/dev/null -o ' \
+ 'LogLevel=error'
+ fetch_playbook = os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
+ 'fetch_overcloud_logs.yml')
+ # Run per overcloud node
+ for node, ip in fetch_vars['overcloud_nodes'].items():
+ logging.info("Executing fetch logs overcloud playbook on "
+ "node {}".format(node))
+ try:
+ utils.run_ansible(fetch_vars, fetch_playbook, host=ip,
+ user='heat-admin', tmp_dir=APEX_TEMP_DIR)
+ logging.info("Logs retrieved for node {}".format(node))
+ except Exception:
+ logging.error("Log retrieval failed "
+ "for node {}. Please check log".format(node))
+ raise
+ logging.info("Log retrieval complete and stored in {}".format(
+ APEX_TEMP_DIR))
+
+
+def execute_actions(args):
+ for action in VALID_UTILS:
+ if hasattr(args, action) and getattr(args, action):
+ util_module = __import__('utils')
+ func = getattr(util_module, action)
+ logging.info("Executing action: {}".format(action))
+ func(args)
+
+
+def main():
+ util_parser = argparse.ArgumentParser()
+ util_parser.add_argument('-f', '--fetch-logs',
+ dest='fetch_logs',
+ required=False,
+ default=False,
+ action='store_true',
+ help='Fetch all overcloud logs')
+ util_parser.add_argument('--lib-dir',
+ default='/usr/share/opnfv-apex',
+ help='Directory path for apex ansible '
+ 'and third party libs')
+ args = util_parser.parse_args(sys.argv[1:])
+ os.makedirs(os.path.dirname('./apex_util.log'), exist_ok=True)
+ formatter = '%(asctime)s %(levelname)s: %(message)s'
+ logging.basicConfig(filename='./apex_clean.log',
+ format=formatter,
+ datefmt='%m/%d/%Y %I:%M:%S %p',
+ level=logging.DEBUG)
+ console = logging.StreamHandler()
+ console.setLevel(logging.DEBUG)
+ console.setFormatter(logging.Formatter(formatter))
+ logging.getLogger('').addHandler(console)
+
+ execute_actions(args)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/build/rpm_specs/opnfv-apex-common.spec b/build/rpm_specs/opnfv-apex-common.spec
index 5b82b72..5d1a59b 100644
--- a/build/rpm_specs/opnfv-apex-common.spec
+++ b/build/rpm_specs/opnfv-apex-common.spec
@@ -65,6 +65,7 @@ install config/inventory/pod_example_settings.yaml %{buildroot}%{_docdir}/opnfv/
%attr(755,root,root) %{_bindir}/opnfv-deploy
%attr(755,root,root) %{_bindir}/opnfv-clean
%attr(755,root,root) %{_bindir}/opnfv-util
+%attr(755,root,root) %{_bindir}/opnfv-pyutil
%{_datadir}/opnfv-apex/
%{_sysconfdir}/bash_completion.d/apex
%{_sysconfdir}/opnfv-apex/os-nosdn-nofeature-noha.yaml
@@ -124,6 +125,8 @@ install config/inventory/pod_example_settings.yaml %{buildroot}%{_docdir}/opnfv/
%doc %{_docdir}/opnfv/inventory.yaml.example
%changelog
+* Tue Apr 03 2018 Tim Rozet <trozet@redhat.com> - 6.0-3
+ Adds fetch logs
* Fri Mar 09 2018 Tim Rozet <trozet@redhat.com> - 6.0-2
Add upstream deploy files with containers
* Wed Feb 14 2018 Tim Rozet <trozet@redhat.com> - 6.0-1
diff --git a/lib/ansible/playbooks/fetch_overcloud_logs.yml b/lib/ansible/playbooks/fetch_overcloud_logs.yml
new file mode 100644
index 0000000..1ab5247
--- /dev/null
+++ b/lib/ansible/playbooks/fetch_overcloud_logs.yml
@@ -0,0 +1,25 @@
+---
+- hosts: all
+ tasks:
+ - name: Archive logs
+ archive:
+ path:
+ - /var/log
+ - /etc/puppet
+ - /etc/nova
+ - /etc/neutron
+ - /etc/heat
+ - /etc/haproxy
+ - /etc/glance
+ - /etc/puppet
+ - /etc/vpp
+ - /etc/os-net-config
+ - /opt/opendaylight/data/log
+ - /opt/opendaylight/etc
+ dest: /root/logging.tar.gz
+ become: yes
+ - name: Fetch /var/log/
+ fetch:
+ src: /root/logging.tar.gz
+ dest: "{{ apex_temp_dir }}/"
+ become: yes
diff --git a/lib/ansible/playbooks/fetch_overcloud_nodes.yml b/lib/ansible/playbooks/fetch_overcloud_nodes.yml
new file mode 100644
index 0000000..bcb5f0f
--- /dev/null
+++ b/lib/ansible/playbooks/fetch_overcloud_nodes.yml
@@ -0,0 +1,13 @@
+---
+- hosts: all
+ tasks:
+ - name: Get overcloud nodes and IPs
+ shell: "{{ stackrc }} && openstack server list -f json"
+ register: nova_list
+ - name: Write nova list output to file
+ local_action: copy content="{{ nova_list.stdout }}" dest="{{ apex_temp_dir }}/nova_output"
+ - name: Get ironic node information
+ shell: "{{ stackrc }} && openstack server list -f json"
+ register: ironic_list
+ - name: Write ironic list output to file
+ local_action: copy content="{{ ironic_list.stdout }}" dest="{{ apex_temp_dir }}/ironic_output"
diff --git a/setup.cfg b/setup.cfg
index 52ad12f..c9e4298 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -25,6 +25,7 @@ setup-hooks =
console_scripts =
opnfv-deploy = apex.deploy:main
opnfv-clean = apex.clean:main
+ opnfv-pyutil = apex.utils:main
[files]
packages =