summaryrefslogtreecommitdiffstats
path: root/ansible/roles/delete_flavor
AgeCommit message (Collapse)AuthorFilesLines
2017-07-26Test Case: OPNFV_YARDSTICK_TC023: VM availability during live migrationchenjiankun1-0/+13
JIRA: YARDSTICK-174 This live migration test case is based on share storage, default share storage is enabled. This test case will do some config work. And do live migration and calculate the migration time and downtime. Change-Id: I6601601edebdd0ac6434ba632b1eba9e9bd4fda0 Signed-off-by: chenjiankun <chenjiankun1@huawei.com>
84' href='#n84'>84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 
###############################################################################
# Copyright (c) 2015 Ericsson AB and others.
# szilard.cserey@ericsson.com
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
###############################################################################

import time
import re
import json

from common import (
    N,
    exec_cmd,
    parse,
    err,
    log,
)

SEARCH_TEXT = '(err)'
LOG_FILE = '/var/log/puppet.log'
GREP_LINES_OF_LEADING_CONTEXT = 100
GREP_LINES_OF_TRAILING_CONTEXT = 100
LIST_OF_CHAR_TO_BE_ESCAPED = ['[', ']', '"']


class DeployNotStart(Exception):
    """Unable to start deployment"""


class NodesGoOffline(Exception):
    """Nodes goes offline during deployment"""


class Deployment(object):

    def __init__(self, dea, yaml_config_dir, env_id, node_id_roles_dict,
                 no_health_check, deploy_timeout):
        self.dea = dea
        self.yaml_config_dir = yaml_config_dir
        self.env_id = env_id
        self.node_id_roles_dict = node_id_roles_dict
        self.no_health_check = no_health_check
        self.deploy_timeout = deploy_timeout
        self.pattern = re.compile(
            '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d')

    def collect_error_logs(self):
        for node_id, roles_blade in self.node_id_roles_dict.iteritems():
            log_list = []
            cmd = ('ssh -q node-%s grep \'"%s"\' %s'
                   % (node_id, SEARCH_TEXT, LOG_FILE))
            results, _ = exec_cmd(cmd, False)
            for result in results.splitlines():
                log_msg = ''

                sub_cmd = '"%s" %s' % (result, LOG_FILE)
                for c in LIST_OF_CHAR_TO_BE_ESCAPED:
                    sub_cmd = sub_cmd.replace(c, '\%s' % c)
                grep_cmd = ('grep -B%s %s'
                            % (GREP_LINES_OF_LEADING_CONTEXT, sub_cmd))
                cmd = ('ssh -q node-%s "%s"' % (node_id, grep_cmd))

                details, _ = exec_cmd(cmd, False)
                details_list = details.splitlines()

                found_prev_log = False
                for i in range(len(details_list) - 2, -1, -1):
                    if self.pattern.match(details_list[i]):
                        found_prev_log = True
                        break
                if found_prev_log:
                    log_msg += '\n'.join(details_list[i:-1]) + '\n'

                grep_cmd = ('grep -A%s %s'
                            % (GREP_LINES_OF_TRAILING_CONTEXT, sub_cmd))
                cmd = ('ssh -q node-%s "%s"' % (node_id, grep_cmd))

                details, _ = exec_cmd(cmd, False)
                details_list = details.splitlines()

                found_next_log = False
                for i in range(1, len(details_list)):
                    if self.pattern.match(details_list[i]):
                        found_next_log = True
                        break
                if found_next_log:
                    log_msg += '\n'.join(details_list[:i])
                else:
                    log_msg += details

                if log_msg:
                    log_list.append(log_msg)

            if log_list:
                role = ('controller' if 'controller' in roles_blade[0]
                        else 'compute host')
                log('_' * 40 + 'Errors in node-%s %s' % (node_id, role)
                    + '_' * 40)
                for log_msg in log_list:
                    print(log_msg + '\n')

    def run_deploy(self):
        SLEEP_TIME = 60
        abort_after = 60 * int(self.deploy_timeout)
        start = time.time()

        log('Starting deployment of environment %s' % self.env_id)
        deploy_id = None
        ready = False
        timeout = False

        attempts = 0
        while attempts < 3:
            try:
                if time.time() > start + abort_after:
                    timeout = True
                    break
                if not deploy_id:
                    deploy_id = self._start_deploy_task()
                sts, prg, msg = self._deployment_status(deploy_id)
                if sts == 'error':
                    log('Error during deployment: {}'.format(msg))
                    break
                if sts == 'running':
                    log('Environment deployment progress: {}%'.format(prg))
                elif sts == 'ready':
                    ready = True
                    break
                time.sleep(SLEEP_TIME)
            except (DeployNotStart, NodesGoOffline) as e:
                log(e)
                attempts += 1
                deploy_id = None
                time.sleep(SLEEP_TIME * attempts)

        if timeout:
            err('Deployment timed out, environment %s is not operational, '
                'snapshot will not be performed'
                % self.env_id)
        if ready:
            log('Environment %s successfully deployed'
                % self.env_id)
        else:
            self.collect_error_logs()
            err('Deployment failed, environment %s is not operational'
                % self.env_id, self.collect_logs)

    def _start_deploy_task(self):
        out, _ = exec_cmd('fuel2 env deploy {}'.format(self.env_id), False)
        id = self._deployment_task_id(out)
        return id

    def _deployment_task_id(self, response):
        response = str(response)
        if response.startswith('Deployment task with id'):
            for s in response.split():
                if s.isdigit():
                    return int(s)
        raise DeployNotStart('Unable to start deployment: {}'.format(response))

    def _deployment_status(self, id):
        task = self._task_fields(id)
        if task['status'] == 'error':
            if task['message'].endswith(
                    'offline. Remove them from environment and try again.'):
                raise NodesGoOffline(task['message'])
        return task['status'], task['progress'], task['message']

    def _task_fields(self, id):
        try:
            out, _ = exec_cmd('fuel2 task show {} -f json'.format(id), False)
            task_info = json.loads(out)
            properties = {}
            # for 9.0 this can be list of dicts or dict
            # see https://bugs.launchpad.net/fuel/+bug/1625518
            if isinstance(task_info, list):
                for d in task_info:
                        properties.update({d['Field']: d['Value']})
            else:
                return task_info
            return properties
        except ValueError as e:
            err('Unable to fetch task info: {}'.format(e))

    def collect_logs(self):
        log('Cleaning out any previous deployment logs')
        exec_cmd('rm -f /var/log/remote/fuel-snapshot-*', False)
        exec_cmd('rm -f /root/deploy-*', False)
        log('Generating Fuel deploy snap-shot')
        if exec_cmd('fuel snapshot < /dev/null &> snapshot.log', False)[1] <> 0:
            log('Could not create a Fuel snapshot')
        else:
            exec_cmd('mv /root/fuel-snapshot* /var/log/remote/', False)

        log('Collecting all Fuel Snapshot & deploy log files')
        r, _ = exec_cmd('tar -czhf /root/deploy-%s.log.tar.gz /var/log/remote' % time.strftime("%Y%m%d-%H%M%S"), False)
        log(r)

    def verify_node_status(self):
        node_list = parse(exec_cmd('fuel --env %s node' % self.env_id))
        failed_nodes = []
        for node in node_list:
            if node[N['status']] != 'ready':
                failed_nodes.append((node[N['id']], node[N['status']]))

        if failed_nodes:
            summary = ''
            for node, status in failed_nodes:
                summary += '[node %s, status %s]\n' % (node, status)
            err('Deployment failed: %s' % summary, self.collect_logs)

    def health_check(self):
        log('Now running sanity and smoke health checks')
        r = exec_cmd('fuel health --env %s --check sanity,smoke --force' % self.env_id)
        log(r)
        if 'failure' in r:
            err('Healthcheck failed!', self.collect_logs)

    def deploy(self):
        self.run_deploy()
        self.verify_node_status()
        if not self.no_health_check:
            self.health_check()
        self.collect_logs()