ansible/post_build_yardstick_image.yml


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

# Copyright (c) 2017 Intel Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
- hosts: localhost

  tasks:
    - group_by:
        key: image_builder

    - name: remove ubuntu policy-rc.d workaround
      file:
        path: "{{ mountdir }}/usr/sbin/policy-rc.d"
        state: absent
      when: "target_os == 'Ubuntu'"

    - name: cleanup fake tmp fstab
      file:
        path: "{{ fake_fstab }}"
        state: absent

    - mount:
        name: "{{ mountdir }}/proc"
        state: unmounted

    - mount:
        name: "{{ mountdir }}"
        state: unmounted

    - name: kpartx -dv to delete all image partition device nodes
      command: kpartx -dv "{{ raw_imgfile }}"
      ignore_errors: true
#!/usr/bin/env python
#
# jose.lausuch@ericsson.com
# valentin.boucher@orange.com
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
#
import functools
import json
import logging
import os
import pkg_resources
import re
import shutil
import subprocess
import sys
import time
from datetime import datetime as dt

import dns.resolver
import requests
from six.moves import urllib
import yaml

from functest.utils import constants
from functest.utils import decorators
from functest.utils.constants import CONST

logger = logging.getLogger(__name__)


# ----------------------------------------------------------
#
#               INTERNET UTILS
#
# -----------------------------------------------------------
def check_internet_connectivity(url='http://www.opnfv.org/'):
    """
    Check if there is access to the internet
    """
    try:
        urllib.request.urlopen(url, timeout=5)
        return True
    except urllib.error.URLError:
        return False


def download_url(url, dest_path):
    """
    Download a file to a destination path given a URL
    """
    name = url.rsplit('/')[-1]
    dest = dest_path + "/" + name
    try:
        response = urllib.request.urlopen(url)
    except (urllib.error.HTTPError, urllib.error.URLError):
        return False

    with open(dest, 'wb') as f:
        shutil.copyfileobj(response, f)
    return True


# ----------------------------------------------------------
#
#               CI UTILS
#
# -----------------------------------------------------------
def get_installer_type():
    """
    Get installer type (fuel, apex, joid, compass)
    """
    try:
        installer = os.environ['INSTALLER_TYPE']
    except KeyError:
        logger.error("Impossible to retrieve the installer type")
        installer = "Unknown_installer"

    return installer


def get_scenario():
    """
    Get scenario
    """
    try:
        scenario = os.environ['DEPLOY_SCENARIO']
    except KeyError:
        logger.info("Impossible to retrieve the scenario."
                    "Use default os-nosdn-nofeature-noha")
        scenario = "os-nosdn-nofeature-noha"

    return scenario


def get_version():
    """
    Get version
    """
    # Use the build tag to retrieve the version
    # By default version is unknown
    # if launched through CI the build tag has the following format
    # jenkins-<project>-<installer>-<pod>-<job>-<branch>-<id>
    # e.g. jenkins-functest-fuel-opnfv-jump-2-daily-master-190
    # jenkins-functest-fuel-baremetal-weekly-master-8
    # use regex to match branch info
    rule = "(dai|week)ly-(.+?)-[0-9]*"
    build_tag = CONST.__getattribute__('BUILD_TAG')
    if not build_tag:
        build_tag = 'none'
    m = re.search(rule, build_tag)
    if m:
        return m.group(2)
    else:
        return "unknown"


def get_pod_name():
    """
    Get PoD Name from env variable NODE_NAME
    """
    try:
        return os.environ['NODE_NAME']
    except KeyError:
        logger.info(
            "Unable to retrieve the POD name from environment. " +
            "Using pod name 'unknown-pod'")
        return "unknown-pod"


def logger_test_results(project, case_name, status, details):
    """
    Format test case results for the logger
    """
    pod_name = get_pod_name()
    scenario = get_scenario()
    version = get_version()
    build_tag = CONST.__getattribute__('BUILD_TAG')
    db_url = CONST.__getattribute__("results_test_db_url")

    logger.info(
        "\n"
        "****************************************\n"
        "\t %(p)s/%(n)s results \n\n"
        "****************************************\n"
        "DB:\t%(db)s\n"
        "pod:\t%(pod)s\n"
        "version:\t%(v)s\n"
        "scenario:\t%(s)s\n"
        "status:\t%(c)s\n"
        "build tag:\t%(b)s\n"
        "details:\t%(d)s\n"
        % {'p': project,
            'n': case_name,
            'db': db_url,
            'pod': pod_name,
            'v': version,
            's': scenario,
            'c': status,
            'b': build_tag,
            'd': details})


@decorators.can_dump_request_to_file
def push_results_to_db(project, case_name,
                       start_date, stop_date, result, details):
    """
    POST results to the Result target DB
    """
    # Retrieve params from CI and conf
    url = CONST.__getattribute__("results_test_db_url")

    try:
        installer = os.environ['INSTALLER_TYPE']
        scenario = os.environ['DEPLOY_SCENARIO']
        pod_name = os.environ['NODE_NAME']
        build_tag = os.environ['BUILD_TAG']
    except KeyError as e:
        logger.error("Please set env var: " + str(e))
        return False
    version = get_version()
    test_start = dt.fromtimestamp(start_date).strftime('%Y-%m-%d %H:%M:%S')
    test_stop = dt.fromtimestamp(stop_date).strftime('%Y-%m-%d %H:%M:%S')

    params = {"project_name": project, "case_name": case_name,
              "pod_name": pod_name, "installer": installer,
              "version": version, "scenario": scenario, "criteria": result,
              "build_tag": build_tag, "start_date": test_start,
              "stop_date": test_stop, "details": details}

    error = None
    headers = {'Content-Type': 'application/json'}
    try:
        r = requests.post(url, data=json.dumps(params, sort_keys=True),
                          headers=headers)
        logger.debug(r)
        r.raise_for_status()
    except requests.RequestException as exc:
        if 'r' in locals():
            error = ("Pushing Result to DB(%s) failed: %s" %
                     (r.url, r.content))
        else:
            error = ("Pushing Result to DB(%s) failed: %s" % (url, exc))
    except Exception as e:
        error = ("Error [push_results_to_db("
                 "DB: '%(db)s', "
                 "project: '%(project)s', "
                 "case: '%(case)s', "
                 "pod: '%(pod)s', "
                 "version: '%(v)s', "
                 "scenario: '%(s)s', "
                 "criteria: '%(c)s', "
                 "build_tag: '%(t)s', "
                 "details: '%(d)s')]: "
                 "%(error)s" %
                 {
                     'db': url,
                     'project': project,
                     'case': case_name,
                     'pod': pod_name,
                     'v': version,
                     's': scenario,
                     'c': result,
                     't': build_tag,
                     'd': details,
                     'error': e
                 })
    finally:
        if error:
            logger.error(error)
            return False
        return True


def get_resolvconf_ns():
    """
    Get nameservers from current resolv.conf
    """
    nameservers = []
    rconf = open("/etc/resolv.conf", "r")
    line = rconf.readline()
    resolver = dns.resolver.Resolver()
    while line:
        ip = re.search(r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b", line)
        if ip:
            resolver.nameservers = [ip.group(0)]
            try:
                result = resolver.query('opnfv.org')[0]
                if result != "":
                    nameservers.append(ip.group())
            except dns.exception.Timeout:
                pass
        line = rconf.readline()
    return nameservers


def get_ci_envvars():
    """
    Get the CI env variables
    """
    ci_env_var = {
        "installer": os.environ.get('INSTALLER_TYPE'),
        "scenario": os.environ.get('DEPLOY_SCENARIO')}
    return ci_env_var


def execute_command_raise(cmd, info=False, error_msg="",
                          verbose=True, output_file=None):
    ret = execute_command(cmd, info, error_msg, verbose, output_file)
    if ret != 0:
        raise Exception(error_msg)


def execute_command(cmd, info=False, error_msg="",
                    verbose=True, output_file=None):
    if not error_msg:
        error_msg = ("The command '%s' failed." % cmd)
    msg_exec = ("Executing command: '%s'" % cmd)
    if verbose:
        if info:
            logger.info(msg_exec)
        else:
            logger.debug(msg_exec)
    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT)
    if output_file:
        f = open(output_file, "w")
    for line in iter(p.stdout.readline, b''):
        if output_file:
            f.write(line)
        else:
            line = line.replace('\n', '')
            print(line)
            sys.stdout.flush()
    if output_file:
        f.close()
    p.stdout.close()
    returncode = p.wait()
    if returncode != 0:
        if verbose:
            logger.error(error_msg)

    return returncode


def get_dict_by_test(testname):
    with open(pkg_resources.resource_filename(
            'functest', 'ci/testcases.yaml')) as f:
        testcases_yaml = yaml.safe_load(f)

    for dic_tier in testcases_yaml.get("tiers"):
        for dic_testcase in dic_tier['testcases']:
            if dic_testcase['case_name'] == testname:
                return dic_testcase

    logger.error('Project %s is not defined in testcases.yaml' % testname)
    return None


def get_criteria_by_test(testname):
    dict = get_dict_by_test(testname)
    if dict:
        return dict['criteria']
    return None


# ----------------------------------------------------------
#
#               YAML UTILS
#
# -----------------------------------------------------------
def get_parameter_from_yaml(parameter, file):
    """
    Returns the value of a given parameter in file.yaml
    parameter must be given in string format with dots
    Example: general.openstack.image_name
    """
    with open(file) as f:
        file_yaml = yaml.safe_load(f)
    f.close()
    value = file_yaml
    for element in parameter.split("."):
        value = value.get(element)
        if value is None:
            raise ValueError("The parameter %s is not defined in"
                             " %s" % (parameter, file))
    return value


def get_functest_config(parameter):
    yaml_ = constants.CONST.__getattribute__('CONFIG_FUNCTEST_YAML')
    return get_parameter_from_yaml(parameter, yaml_)


def merge_dicts(dict1, dict2):
    for k in set(dict1.keys()).union(dict2.keys()):
        if k in dict1 and k in dict2:
            if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
                yield (k, dict(merge_dicts(dict1[k], dict2[k])))
            else:
                yield (k, dict2[k])
        elif k in dict1:
            yield (k, dict1[k])
        else:
            yield (k, dict2[k])


def get_functest_yaml():
    with open(constants.CONST.__getattribute__('CONFIG_FUNCTEST_YAML')) as f:
        functest_yaml = yaml.safe_load(f)
    f.close()
    return functest_yaml


def print_separator():
    logger.info("==============================================")


def timethis(func):
    """Measure the time it takes for a function to complete"""
    @functools.wraps(func)
    def timed(*args, **kwargs):
        ts = time.time()
        result = func(*args, **kwargs)
        te = time.time()
        elapsed = '{0}'.format(te - ts)
        logger.info('{f}(*{a}, **{kw}) took: {t} sec'.format(
            f=func.__name__, a=args, kw=kwargs, t=elapsed))
        return result, elapsed
    return timed