From 221f4482bab3361cdb69161147e4d32d098d9a43 Mon Sep 17 00:00:00 2001 From: chenjiankun Date: Sat, 1 Apr 2017 01:19:16 +0000 Subject: Yardstick virtualenv support JIRA: YARDSTICK-620 Currently we recommend using docker to run yardstick. And it is hard to use virtualenv to install yardstick. So I modify install.sh in yardstick root path. It will support using virtualenv to install yardstick(including API) in linux. In this patch, I make yardstick support read yardstick configuration have priority over constants. Change-Id: I9ea1241b228532a6497451e6c8f232173ddb783e Signed-off-by: chenjiankun (cherry picked from commit b74d875134b988a26441d559a9e700aaa68d6a0c) --- api/api-prepare.sh | 30 ++++++++- api/conf.py | 28 -------- api/resources/env_action.py | 73 ++++++++++----------- api/resources/release_action.py | 6 +- api/resources/samples_action.py | 4 +- api/urls.py | 1 + api/utils/common.py | 2 +- api/utils/daemonthread.py | 4 +- api/utils/influx.py | 8 +-- api/yardstick.ini | 16 ----- docker/Dockerfile | 55 +++------------- etc/yardstick/yardstick.yaml | 33 ++++++++++ install.sh | 45 ++++++++----- tests/unit/common/test_utils.py | 21 +++--- yardstick/__init__.py | 4 +- yardstick/benchmark/contexts/heat.py | 2 +- yardstick/benchmark/contexts/node.py | 9 ++- yardstick/benchmark/contexts/standalone.py | 2 +- yardstick/benchmark/core/task.py | 2 +- yardstick/common/constants.py | 101 +++++++++++++++++------------ yardstick/common/utils.py | 36 +++++----- yardstick/definitions.py | 14 ---- 22 files changed, 244 insertions(+), 252 deletions(-) delete mode 100644 api/conf.py delete mode 100644 api/yardstick.ini create mode 100644 etc/yardstick/yardstick.yaml delete mode 100644 yardstick/definitions.py diff --git a/api/api-prepare.sh b/api/api-prepare.sh index 5cc65c959..f2be48909 100755 --- a/api/api-prepare.sh +++ b/api/api-prepare.sh @@ -8,6 +8,33 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +: ${YARDSTICK_REPO_DIR:='/home/opnfv/repos/yardstick'} + +# generate uwsgi config file +mkdir -p /etc/yardstick +uwsgi_config='/etc/yardstick/yardstick.ini' +if [[ ! -e "${uwsgi_config}" ]];then + + cat << EOF > "${uwsgi_config}" +[uwsgi] +master = true +debug = true +chdir = ${YARDSTICK_REPO_DIR}/api +module = server +plugins = python +processes = 10 +threads = 5 +async = true +max-requests = 5000 +chmod-socket = 666 +callable = app_wrapper +enable-threads = true +close-on-exec = 1 +daemonize= /var/log/yardstick/uwsgi.log +socket = /var/run/yardstick.sock +EOF +fi + # nginx config nginx_config='/etc/nginx/conf.d/yardstick.conf' @@ -24,7 +51,6 @@ server { } } EOF -echo "daemon off;" >> /etc/nginx/nginx.conf fi # nginx service start when boot @@ -42,7 +68,7 @@ autorestart = true [program:yardstick_uwsgi] user = root -directory = /home/opnfv/repos/yardstick/api +directory = /etc/yardstick command = uwsgi -i yardstick.ini autorestart = true EOF diff --git a/api/conf.py b/api/conf.py deleted file mode 100644 index a4f332533..000000000 --- a/api/conf.py +++ /dev/null @@ -1,28 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -from __future__ import absolute_import -from pyroute2 import IPDB - - -# configuration for influxdb -with IPDB() as ip: - GATEWAY_IP = ip.routes['default'].gateway -PORT = 8086 - -TEST_CASE_PATH = '../tests/opnfv/test_cases/' - -SAMPLE_PATH = '../samples/' - -TEST_CASE_PRE = 'opnfv_yardstick_' - -TEST_SUITE_PATH = '../tests/opnfv/test_suites/' - -TEST_SUITE_PRE = 'opnfv_' - -OUTPUT_CONFIG_FILE_PATH = '/etc/yardstick/yardstick.conf' diff --git a/api/resources/env_action.py b/api/resources/env_action.py index 9d1686a1d..f6f43e5ac 100644 --- a/api/resources/env_action.py +++ b/api/resources/env_action.py @@ -19,17 +19,17 @@ import glob from six.moves import configparser from oslo_serialization import jsonutils +from docker import Client -from api import conf as api_conf from api.database.handler import AsyncTaskHandler from api.utils import influx from api.utils.common import result_handler -from docker import Client -from yardstick.common import constants as config +from yardstick.common import constants as consts from yardstick.common import utils as yardstick_utils from yardstick.common import openstack_utils from yardstick.common.httpClient import HttpClient + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -46,12 +46,12 @@ def createGrafanaContainer(args): def _create_grafana(task_id): _create_task(task_id) - client = Client(base_url=config.DOCKER_URL) + client = Client(base_url=consts.DOCKER_URL) try: - if not _check_image_exist(client, '%s:%s' % (config.GRAFANA_IMAGE, - config.GRAFANA_TAGS)): - client.pull(config.GRAFANA_IMAGE, config.GRAFANA_TAGS) + image = '{}:{}'.format(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG) + if not _check_image_exist(client, image): + client.pull(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG) _create_grafana_container(client) @@ -64,13 +64,12 @@ def _create_grafana(task_id): _update_task_status(task_id) except Exception as e: _update_task_error(task_id, str(e)) - logger.debug('Error: %s', e) + logger.exception('Error: %s', e) def _create_dashboard(): - url = 'http://admin:admin@%s:3000/api/dashboards/db' % api_conf.GATEWAY_IP - path = os.path.join(config.YARDSTICK_REPOS_DIR, 'dashboard', - '*dashboard.json') + url = 'http://admin:admin@%s:3000/api/dashboards/db' % consts.GRAFANA_IP + path = os.path.join(consts.REPOS_DIR, 'dashboard', '*dashboard.json') for i in sorted(glob.iglob(path)): with open(i) as f: @@ -79,12 +78,12 @@ def _create_dashboard(): def _create_data_source(): - url = 'http://admin:admin@%s:3000/api/datasources' % api_conf.GATEWAY_IP + url = 'http://admin:admin@%s:3000/api/datasources' % consts.GRAFANA_IP data = { "name": "yardstick", "type": "influxdb", "access": "proxy", - "url": "http://%s:8086" % api_conf.GATEWAY_IP, + "url": "http://%s:8086" % consts.INFLUXDB_IP, "password": "root", "user": "root", "database": "yardstick", @@ -101,8 +100,8 @@ def _create_grafana_container(client): port_bindings = {k: k for k in ports} host_config = client.create_host_config(port_bindings=port_bindings) - container = client.create_container(image='%s:%s' % (config.GRAFANA_IMAGE, - config.GRAFANA_TAGS), + container = client.create_container(image='%s:%s' % (consts.GRAFANA_IMAGE, + consts.GRAFANA_TAG), ports=ports, detach=True, tty=True, @@ -126,14 +125,14 @@ def createInfluxDBContainer(args): def _create_influxdb(task_id): _create_task(task_id) - client = Client(base_url=config.DOCKER_URL) + client = Client(base_url=consts.DOCKER_URL) try: _change_output_to_influxdb() - if not _check_image_exist(client, '%s:%s' % (config.INFLUXDB_IMAGE, - config.INFLUXDB_TAG)): - client.pull(config.INFLUXDB_IMAGE, tag=config.INFLUXDB_TAG) + if not _check_image_exist(client, '%s:%s' % (consts.INFLUXDB_IMAGE, + consts.INFLUXDB_TAG)): + client.pull(consts.INFLUXDB_IMAGE, tag=consts.INFLUXDB_TAG) _create_influxdb_container(client) @@ -153,8 +152,8 @@ def _create_influxdb_container(client): port_bindings = {k: k for k in ports} host_config = client.create_host_config(port_bindings=port_bindings) - container = client.create_container(image='%s:%s' % (config.INFLUXDB_IMAGE, - config.INFLUXDB_TAG), + container = client.create_container(image='%s:%s' % (consts.INFLUXDB_IMAGE, + consts.INFLUXDB_TAG), ports=ports, detach=True, tty=True, @@ -165,24 +164,26 @@ def _create_influxdb_container(client): def _config_influxdb(): try: client = influx.get_data_db_client() - client.create_user(config.USER, config.PASSWORD, config.DATABASE) - client.create_database(config.DATABASE) + client.create_user(consts.INFLUXDB_USER, + consts.INFLUXDB_PASS, + consts.INFLUXDB_DB_NAME) + client.create_database(consts.INFLUXDB_DB_NAME) logger.info('Success to config influxDB') except Exception as e: logger.debug('Failed to config influxDB: %s', e) def _change_output_to_influxdb(): - yardstick_utils.makedirs(config.YARDSTICK_CONFIG_DIR) + yardstick_utils.makedirs(consts.CONF_DIR) parser = configparser.ConfigParser() - parser.read(config.YARDSTICK_CONFIG_SAMPLE_FILE) + parser.read(consts.CONF_SAMPLE_FILE) parser.set('DEFAULT', 'dispatcher', 'influxdb') parser.set('dispatcher_influxdb', 'target', - 'http://%s:8086' % api_conf.GATEWAY_IP) + 'http://%s:8086' % consts.INFLUXDB_IP) - with open(config.YARDSTICK_CONFIG_FILE, 'w') as f: + with open(consts.CONF_FILE, 'w') as f: parser.write(f) @@ -213,7 +214,7 @@ def _prepare_env_daemon(task_id): _create_directories() - rc_file = config.OPENSTACK_RC_FILE + rc_file = consts.OPENRC if not _already_source_openrc(): _get_remote_rc_file(rc_file, installer_ip, installer_type) @@ -240,12 +241,12 @@ def _check_variables(installer_ip, installer_type): if installer_type == 'undefined': raise SystemExit('Missing INSTALLER_TYPE') - elif installer_type not in config.INSTALLERS: + elif installer_type not in consts.INSTALLERS: raise SystemExit('INSTALLER_TYPE is not correct') def _create_directories(): - yardstick_utils.makedirs(config.YARDSTICK_CONFIG_DIR) + yardstick_utils.makedirs(consts.CONF_DIR) def _source_file(rc_file): @@ -254,7 +255,7 @@ def _source_file(rc_file): def _get_remote_rc_file(rc_file, installer_ip, installer_type): - os_fetch_script = os.path.join(config.RELENG_DIR, config.OS_FETCH_SCRIPT) + os_fetch_script = os.path.join(consts.RELENG_DIR, consts.FETCH_SCRIPT) try: cmd = [os_fetch_script, '-d', rc_file, '-i', installer_type, @@ -287,17 +288,15 @@ def _append_external_network(rc_file): def _clean_images(): - cmd = [config.CLEAN_IMAGES_SCRIPT] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - cwd=config.YARDSTICK_REPOS_DIR) + cmd = [consts.CLEAN_IMAGES_SCRIPT] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=consts.REPOS_DIR) output = p.communicate()[0] logger.debug('The result is: %s', output) def _load_images(): - cmd = [config.LOAD_IMAGES_SCRIPT] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - cwd=config.YARDSTICK_REPOS_DIR) + cmd = [consts.LOAD_IMAGES_SCRIPT] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=consts.REPOS_DIR) output = p.communicate()[0] logger.debug('The result is: %s', output) diff --git a/api/resources/release_action.py b/api/resources/release_action.py index c5aa20afc..9016d4aa2 100644 --- a/api/resources/release_action.py +++ b/api/resources/release_action.py @@ -11,8 +11,8 @@ import uuid import os import logging -from api import conf from api.utils import common as common_utils +from yardstick.common import constants as consts logger = logging.getLogger(__name__) @@ -24,8 +24,8 @@ def runTestCase(args): except KeyError: return common_utils.error_handler('Lack of testcase argument') - testcase_name = conf.TEST_CASE_PRE + testcase - testcase = os.path.join(conf.TEST_CASE_PATH, testcase_name + '.yaml') + testcase_name = consts.TESTCASE_PRE + testcase + testcase = os.path.join(consts.TESTCASE_DIR, testcase_name + '.yaml') task_id = str(uuid.uuid4()) diff --git a/api/resources/samples_action.py b/api/resources/samples_action.py index 490e48b25..3093864e0 100644 --- a/api/resources/samples_action.py +++ b/api/resources/samples_action.py @@ -11,8 +11,8 @@ import uuid import os import logging -from api import conf from api.utils import common as common_utils +from yardstick.common import constants as consts logger = logging.getLogger(__name__) @@ -24,7 +24,7 @@ def runTestCase(args): except KeyError: return common_utils.error_handler('Lack of testcase argument') - testcase = os.path.join(conf.SAMPLE_PATH, testcase_name + '.yaml') + testcase = os.path.join(consts.SAMPLE_CASE_DIR, testcase_name + '.yaml') task_id = str(uuid.uuid4()) diff --git a/api/urls.py b/api/urls.py index 3ccb67dbc..b9ddd4c72 100644 --- a/api/urls.py +++ b/api/urls.py @@ -7,6 +7,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## from __future__ import absolute_import + from api import views from api.utils.common import Url diff --git a/api/utils/common.py b/api/utils/common.py index 3e9bf8f8b..f8b0d40ba 100644 --- a/api/utils/common.py +++ b/api/utils/common.py @@ -11,10 +11,10 @@ import collections import logging from flask import jsonify +import six from api.utils.daemonthread import DaemonThread from yardstick.cmd.cli import YardstickCLI -import six logger = logging.getLogger(__name__) diff --git a/api/utils/daemonthread.py b/api/utils/daemonthread.py index 0049834eb..3d5625547 100644 --- a/api/utils/daemonthread.py +++ b/api/utils/daemonthread.py @@ -11,7 +11,7 @@ import threading import os import errno -from api import conf +from yardstick.common import constants as consts from api.database.handlers import TasksHandler @@ -43,7 +43,7 @@ class DaemonThread(threading.Thread): def _handle_testsuite_file(task_id): try: - os.remove(os.path.join(conf.TEST_SUITE_PATH, task_id + '.yaml')) + os.remove(os.path.join(consts.TESTSUITE_DIR, task_id + '.yaml')) except OSError as e: if e.errno != errno.ENOENT: raise diff --git a/api/utils/influx.py b/api/utils/influx.py index 08996b9c9..9bc6e9abe 100644 --- a/api/utils/influx.py +++ b/api/utils/influx.py @@ -14,7 +14,7 @@ import six.moves.configparser as ConfigParser from six.moves.urllib.parse import urlsplit from influxdb import InfluxDBClient -from api import conf +from yardstick.common import constants as consts logger = logging.getLogger(__name__) @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) def get_data_db_client(): parser = ConfigParser.ConfigParser() try: - parser.read(conf.OUTPUT_CONFIG_FILE_PATH) + parser.read(consts.CONF_FILE) if parser.get('DEFAULT', 'dispatcher') != 'influxdb': raise RuntimeError @@ -35,10 +35,10 @@ def get_data_db_client(): def _get_client(parser): ip = _get_ip(parser.get('dispatcher_influxdb', 'target')) - username = parser.get('dispatcher_influxdb', 'username') + user = parser.get('dispatcher_influxdb', 'username') password = parser.get('dispatcher_influxdb', 'password') db_name = parser.get('dispatcher_influxdb', 'db_name') - return InfluxDBClient(ip, conf.PORT, username, password, db_name) + return InfluxDBClient(ip, consts.INFLUXDB_PORT, user, password, db_name) def _get_ip(url): diff --git a/api/yardstick.ini b/api/yardstick.ini deleted file mode 100644 index d2e8956e2..000000000 --- a/api/yardstick.ini +++ /dev/null @@ -1,16 +0,0 @@ -[uwsgi] -master = true -debug = true -chdir = /home/opnfv/repos/yardstick/api -module = server -plugins = python -processes = 10 -threads = 5 -async = true -max-requests = 5000 -chmod-socket = 666 -callable = app_wrapper -enable-threads = true -close-on-exec = 1 -daemonize= /var/log/yardstick/uwsgi.log -socket = /var/run/yardstick.sock diff --git a/docker/Dockerfile b/docker/Dockerfile index 551693554..3be28e4b2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,69 +15,32 @@ ARG BRANCH=master # GIT repo directory ENV REPOS_DIR /home/opnfv/repos +ENV IMAGE_DIR /home/opnfv/images/ # Set work directory -WORKDIR /home/opnfv/repos # Yardstick repo ENV YARDSTICK_REPO_DIR ${REPOS_DIR}/yardstick ENV RELENG_REPO_DIR ${REPOS_DIR}/releng -RUN sed -i -e 's/^deb /deb [arch=amd64] /g' /etc/apt/sources.list -RUN sed -i -e 's/^deb-src /# deb-src /g' /etc/apt/sources.list -RUN echo "APT::Default-Release \"trusty\";" > /etc/apt/apt.conf.d/default-distro -RUN echo "\n\ -deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ xenial-updates universe \n\ -deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty main universe multiverse restricted \n\ -deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main universe multiverse restricted \n\ -deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-security main universe multiverse restricted \n\ -deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-proposed main universe multiverse restricted" >> /etc/apt/sources.list -RUN echo "vm.mmap_min_addr = 0" > /etc/sysctl.d/mmap_min_addr.conf -RUN dpkg --add-architecture arm64 - -RUN apt-get update && apt-get install -y \ - qemu-user-static/xenial \ - libc6:arm64 \ - wget \ - expect \ - curl \ - git \ - sshpass \ - qemu-utils \ - kpartx \ - libffi-dev \ - libssl-dev \ - libzmq-dev \ - python \ - python-dev \ - libxml2-dev \ - libxslt1-dev \ - nginx \ - uwsgi \ - uwsgi-plugin-python \ - supervisor \ - python-setuptools && \ - easy_install -U setuptools==30.0.0 - -RUN apt-get -y autoremove && \ - apt-get clean +RUN apt-get update && apt-get install -y git RUN mkdir -p ${REPOS_DIR} RUN git config --global http.sslVerify false RUN git clone --depth 1 -b $BRANCH https://gerrit.opnfv.org/gerrit/yardstick ${YARDSTICK_REPO_DIR} RUN git clone --depth 1 https://gerrit.opnfv.org/gerrit/releng ${RELENG_REPO_DIR} -# install yardstick + dependencies -RUN cd ${YARDSTICK_REPO_DIR} && easy_install -U pip -RUN cd ${YARDSTICK_REPO_DIR} && pip install -r requirements.txt -RUN cd ${YARDSTICK_REPO_DIR} && pip install . +WORKDIR ${YARDSTICK_REPO_DIR} +RUN ${YARDSTICK_REPO_DIR}/install.sh -RUN ${YARDSTICK_REPO_DIR}/api/api-prepare.sh +RUN echo "daemon off;" >> /etc/nginx/nginx.conf EXPOSE 5000 -ADD http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img /home/opnfv/images/ -ADD http://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img /home/opnfv/images/ +ADD http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img ${IMAGE_DIR} +ADD http://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img ${IMAGE_DIR} COPY ./exec_tests.sh /usr/local/bin/ + +WORKDIR ${REPOS_DIR} CMD ["/usr/bin/supervisord"] diff --git a/etc/yardstick/yardstick.yaml b/etc/yardstick/yardstick.yaml new file mode 100644 index 000000000..5b39fa08b --- /dev/null +++ b/etc/yardstick/yardstick.yaml @@ -0,0 +1,33 @@ +dir: + conf: /etc/yardstick + repos: /home/opnfv/repos/yardstick + releng: /home/opnfv/repos/releng + log: /tmp/yardstick + +file: + openrc: /etc/yardstick/openstack.creds + fetch_script: utils/fetch_os_creds.sh + clean_image_script: tests/ci/clean_images.sh + load_image_script: tests/ci/load_images.sh + output_file: /tmp/yardstick.out + html_file: /tmp/yardstick.htm + +influxdb: + ip: 172.17.0.1 + port: 8086 + username: root + password: root + db_name: yardstick + image: tutum/influxdb + tag: 0.13 + +grafana: + ip: 172.17.0.1 + port: 3000 + username: admin + password: admin + image: grafana/grafana + tag: 3.1.1 + +api: + server_ip: 172.17.0.1 diff --git a/install.sh b/install.sh index a918340ca..75618dd27 100755 --- a/install.sh +++ b/install.sh @@ -8,8 +8,26 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +# fit for arm64 +source_file=/etc/apt/sources.list +sed -i -e 's/^deb \([^/[]\)/deb [arch=amd64] \1/g' "${source_file}" +sed -i -e 's/^deb-src /# deb-src /g' "${source_file}" +echo "APT::Default-Release \"trusty\";" > /etc/apt/apt.conf.d/default-distro + +sub_source_file=/etc/apt/sources.list.d/yardstick.list +touch "${sub_source_file}" +echo -e "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ xenial-updates universe +deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty main universe multiverse restricted +deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main universe multiverse restricted +deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-security main universe multiverse restricted +deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-proposed main universe multiverse restricted" > "${sub_source_file}" +echo "vm.mmap_min_addr = 0" > /etc/sysctl.d/mmap_min_addr.conf +dpkg --add-architecture arm64 + # install tools apt-get update && apt-get install -y \ + qemu-user-static/xenial \ + libc6:arm64 \ wget \ expect \ curl \ @@ -19,38 +37,31 @@ apt-get update && apt-get install -y \ kpartx \ libffi-dev \ libssl-dev \ + libzmq-dev \ python \ python-dev \ - python-pip \ - flake8 libxml2-dev \ libxslt1-dev \ nginx \ uwsgi \ uwsgi-plugin-python \ supervisor \ + python-pip \ + vim \ python-setuptools && \ - easy_install -U setuptools + easy_install -U setuptools==30.0.0 apt-get -y autoremove && apt-get clean +git config --global http.sslVerify false -# fit for arm64 -source_file=/etc/apt/sources.list -sed -i -e 's/^deb \([^/[]\)/deb [arch=amd64] \1/g' "${source_file}" -sed -i -e 's/^deb-src /# deb-src /g' "${source_file}" - -sub_source_file=/etc/apt/sources.list.d/yardstick.list -touch "${sub_source_file}" -echo -e "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty main universe multiverse restricted -deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main universe multiverse restricted -deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-security main universe multiverse restricted -deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-proposed main universe multiverse restricted" > "${sub_source_file}" -echo "vm.mmap_min_addr = 0" > /etc/sysctl.d/mmap_min_addr.conf -dpkg --add-architecture arm64 -apt-get install -y qemu-user-static libc6:arm64 # install yardstick + dependencies easy_install -U pip pip install -r requirements.txt pip install . + +/bin/bash "$(pwd)/api/api-prepare.sh" + +service nginx restart +uwsgi -i /etc/yardstick/yardstick.ini diff --git a/tests/unit/common/test_utils.py b/tests/unit/common/test_utils.py index 267c71312..8f52b53b0 100644 --- a/tests/unit/common/test_utils.py +++ b/tests/unit/common/test_utils.py @@ -87,24 +87,21 @@ class ImportModulesFromPackageTestCase(unittest.TestCase): class GetParaFromYaml(unittest.TestCase): - def test_get_para_from_yaml_file_not_exist(self): - file_path = '/etc/yardstick/hello.yaml' - args = 'hello.world' - para = utils.get_para_from_yaml(file_path, args) - self.assertIsNone(para) - - def test_get_para_from_yaml_para_not_found(self): + @mock.patch('yardstick.common.utils.os.environ.get') + def test_get_param_para_not_found(self, get_env): file_path = 'config_sample.yaml' - file_path = self._get_file_abspath(file_path) + get_env.return_value = self._get_file_abspath(file_path) args = 'releng.file' - self.assertIsNone(utils.get_para_from_yaml(file_path, args)) + default = 'hello' + self.assertTrue(utils.get_param(args, default), default) - def test_get_para_from_yaml_para_exists(self): + @mock.patch('yardstick.common.utils.os.environ.get') + def test_get_param_para_exists(self, get_env): file_path = 'config_sample.yaml' - file_path = self._get_file_abspath(file_path) + get_env.return_value = self._get_file_abspath(file_path) args = 'releng.dir' para = '/home/opnfv/repos/releng' - self.assertEqual(para, utils.get_para_from_yaml(file_path, args)) + self.assertEqual(para, utils.get_param(args)) def _get_file_abspath(self, filename): curr_path = os.path.dirname(os.path.abspath(__file__)) diff --git a/yardstick/__init__.py b/yardstick/__init__.py index e19be36fd..2f5ae9feb 100644 --- a/yardstick/__init__.py +++ b/yardstick/__init__.py @@ -20,8 +20,8 @@ from yardstick.common import utils as yardstick_utils # without having to install apexlake. sys.path.append(os.path.dirname(apexlake.__file__)) -yardstick_utils.makedirs(constants.YARDSTICK_LOG_DIR) -LOG_FILE = os.path.join(constants.YARDSTICK_LOG_DIR, 'yardstick.log') +yardstick_utils.makedirs(constants.LOG_DIR) +LOG_FILE = os.path.join(constants.LOG_DIR, 'yardstick.log') LOG_FORMATTER = ('%(asctime)s ' '%(name)s %(filename)s:%(lineno)d ' '%(levelname)s %(message)s') diff --git a/yardstick/benchmark/contexts/heat.py b/yardstick/benchmark/contexts/heat.py index 571a769eb..604df80d1 100644 --- a/yardstick/benchmark/contexts/heat.py +++ b/yardstick/benchmark/contexts/heat.py @@ -25,7 +25,7 @@ from yardstick.benchmark.contexts.model import PlacementGroup, ServerGroup from yardstick.benchmark.contexts.model import Server from yardstick.benchmark.contexts.model import update_scheduler_hints from yardstick.orchestrator.heat import HeatTemplate, get_short_key_uuid -from yardstick.definitions import YARDSTICK_ROOT_PATH +from yardstick.common.constants import YARDSTICK_ROOT_PATH LOG = logging.getLogger(__name__) diff --git a/yardstick/benchmark/contexts/node.py b/yardstick/benchmark/contexts/node.py index 8bf915609..a4a2cfc57 100644 --- a/yardstick/benchmark/contexts/node.py +++ b/yardstick/benchmark/contexts/node.py @@ -19,7 +19,7 @@ import pkg_resources from yardstick import ssh from yardstick.benchmark.contexts.base import Context -from yardstick.common.constants import YARDSTICK_ROOT_PATH +from yardstick.common import constants as consts LOG = logging.getLogger(__name__) @@ -57,7 +57,7 @@ class NodeContext(Context): except IOError as ioerror: if ioerror.errno == errno.ENOENT: self.file_path = \ - os.path.join(YARDSTICK_ROOT_PATH, self.file_path) + os.path.join(consts.YARDSTICK_ROOT_PATH, self.file_path) cfg = self.read_config_file() else: raise @@ -108,8 +108,7 @@ class NodeContext(Context): def _do_ansible_job(self, path): cmd = 'ansible-playbook -i inventory.ini %s' % path - base = '/home/opnfv/repos/yardstick/ansible' - p = subprocess.Popen(cmd, shell=True, cwd=base) + p = subprocess.Popen(cmd, shell=True, cwd=consts.ANSIBLE_DIR) p.communicate() def _get_server(self, attr_name): @@ -164,7 +163,7 @@ class NodeContext(Context): def _execute_local_script(self, info): script, options = self._get_script(info) - script = os.path.join(YARDSTICK_ROOT_PATH, script) + script = os.path.join(consts.YARDSTICK_ROOT_PATH, script) cmd = ['bash', script, options] p = subprocess.Popen(cmd, stdout=subprocess.PIPE) diff --git a/yardstick/benchmark/contexts/standalone.py b/yardstick/benchmark/contexts/standalone.py index 674e57f54..78eaac7ee 100644 --- a/yardstick/benchmark/contexts/standalone.py +++ b/yardstick/benchmark/contexts/standalone.py @@ -20,7 +20,7 @@ import collections import yaml from yardstick.benchmark.contexts.base import Context -from yardstick.definitions import YARDSTICK_ROOT_PATH +from yardstick.common.constants import YARDSTICK_ROOT_PATH LOG = logging.getLogger(__name__) diff --git a/yardstick/benchmark/core/task.py b/yardstick/benchmark/core/task.py index 40122764c..3a151dbba 100644 --- a/yardstick/benchmark/core/task.py +++ b/yardstick/benchmark/core/task.py @@ -503,7 +503,7 @@ def check_environment(): auth_url = os.environ.get('OS_AUTH_URL', None) if not auth_url: try: - source_env(constants.OPENSTACK_RC_FILE) + source_env(constants.OPENRC) except IOError as e: if e.errno != errno.EEXIST: raise diff --git a/yardstick/common/constants.py b/yardstick/common/constants.py index 6550cc8fd..33266e233 100644 --- a/yardstick/common/constants.py +++ b/yardstick/common/constants.py @@ -9,61 +9,78 @@ from __future__ import absolute_import import os -DOCKER_URL = 'unix://var/run/docker.sock' - -# database config -USER = 'root' -PASSWORD = 'root' -DATABASE = 'yardstick' - -INFLUXDB_IMAGE = 'tutum/influxdb' -INFLUXDB_TAG = '0.13' +from yardstick.common.utils import get_param -GRAFANA_IMAGE = 'grafana/grafana' -GRAFANA_TAGS = '3.1.1' dirname = os.path.dirname abspath = os.path.abspath join = os.path.join sep = os.path.sep -INSTALLERS = ['apex', 'compass', 'fuel', 'joid'] - +try: + SERVER_IP = get_param('api.server_ip') +except KeyError: + try: + from pyroute2 import IPDB + except ImportError: + SERVER_IP = '172.17.0.1' + else: + with IPDB() as ip: + SERVER_IP = ip.routes['default'].gateway + +# dir +CONF_DIR = get_param('dir.conf', '/etc/yardstick') +REPOS_DIR = get_param('dir.repos', '/home/opnfv/repos/yardstick') +RELENG_DIR = get_param('dir.releng', '/home/opnfv/repos/releng') +LOG_DIR = get_param('dir.log', '/tmp/yardstick/') YARDSTICK_ROOT_PATH = dirname(dirname(dirname(abspath(__file__)))) + sep - +CONF_SAMPLE_DIR = join(REPOS_DIR, 'etc/yardstick/') +ANSIBLE_DIR = join(REPOS_DIR, 'ansible') +SAMPLE_CASE_DIR = join(REPOS_DIR, 'samples') TESTCASE_DIR = join(YARDSTICK_ROOT_PATH, 'tests/opnfv/test_cases/') - TESTSUITE_DIR = join(YARDSTICK_ROOT_PATH, 'tests/opnfv/test_suites/') -YARDSTICK_REPOS_DIR = '/home/opnfv/repos/yardstick' - -YARDSTICK_LOG_DIR = '/tmp/yardstick/' - -YARDSTICK_CONFIG_DIR = '/etc/yardstick/' - -YARDSTICK_CONFIG_FILE = join(YARDSTICK_CONFIG_DIR, 'yardstick.conf') - -YARDSTICK_CONFIG_SAMPLE_DIR = join(YARDSTICK_ROOT_PATH, 'etc/yardstick/') - -YARDSTICK_CONFIG_SAMPLE_FILE = join(YARDSTICK_CONFIG_SAMPLE_DIR, - 'yardstick.conf.sample') - -RELENG_DIR = '/home/opnfv/repos/releng' - -OS_FETCH_SCRIPT = 'utils/fetch_os_creds.sh' - -CLEAN_IMAGES_SCRIPT = 'tests/ci/clean_images.sh' - -LOAD_IMAGES_SCRIPT = 'tests/ci/load_images.sh' - -OPENSTACK_RC_FILE = join(YARDSTICK_CONFIG_DIR, 'openstack.creds') +# file +OPENRC = get_param('file.openrc', '/etc/yardstick/yardstick.conf') +CONF_FILE = join(CONF_DIR, 'yardstick.conf') +CONF_SAMPLE_FILE = join(CONF_SAMPLE_DIR, 'yardstick.conf.sample') +FETCH_SCRIPT = get_param('file.fetch_script', 'utils/fetch_os_creds.sh') +FETCH_SCRIPT = join(RELENG_DIR, FETCH_SCRIPT) +CLEAN_IMAGES_SCRIPT = get_param('file.clean_image_script', + 'tests/ci/clean_images.sh') +CLEAN_IMAGES_SCRIPT = join(REPOS_DIR, CLEAN_IMAGES_SCRIPT) +LOAD_IMAGES_SCRIPT = get_param('file.load_image_script', + 'tests/ci/load_images.sh') +LOAD_IMAGES_SCRIPT = join(REPOS_DIR, LOAD_IMAGES_SCRIPT) +DEFAULT_OUTPUT_FILE = get_param('file.output_file', '/tmp/yardstick.out') +DEFAULT_HTML_FILE = get_param('file.html_file', '/tmp/yardstick.htm') + +# influxDB +INFLUXDB_IP = get_param('influxdb.ip', SERVER_IP) +INFLUXDB_PORT = get_param('influxdb.port', 8086) +INFLUXDB_USER = get_param('influxdb.username', 'root') +INFLUXDB_PASS = get_param('influxdb.password', 'root') +INFLUXDB_DB_NAME = get_param('influxdb.db_name', 'yardstick') +INFLUXDB_IMAGE = get_param('influxdb.image', 'tutum/influxdb') +INFLUXDB_TAG = get_param('influxdb.tag', '0.13') + +# grafana +GRAFANA_IP = get_param('grafana.ip', SERVER_IP) +GRAFANA_PORT = get_param('grafana.port', 3000) +GRAFANA_USER = get_param('grafana.username', 'admin') +GRAFANA_PASS = get_param('grafana.password', 'admin') +GRAFANA_IMAGE = get_param('grafana.image', 'grafana/grafana') +GRAFANA_TAG = get_param('grafana.tag', '3.1.1') + +# api +DOCKER_URL = 'unix://var/run/docker.sock' +INSTALLERS = ['apex', 'compass', 'fuel', 'joid'] +SQLITE = 'sqlite:////tmp/yardstick.db' BASE_URL = 'http://localhost:5000' ENV_ACTION_API = BASE_URL + '/yardstick/env/action' ASYNC_TASK_API = BASE_URL + '/yardstick/asynctask' -SQLITE = 'sqlite:////tmp/yardstick.db' - -DEFAULT_OUTPUT_FILE = '/tmp/yardstick.out' - -DEFAULT_HTML_FILE = '/tmp/yardstick.htm' +# general +TESTCASE_PRE = 'opnfv_yardstick_' +TESTSUITE_PRE = 'opnfv_' diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py index d2be8000e..7035f3374 100644 --- a/yardstick/common/utils.py +++ b/yardstick/common/utils.py @@ -84,26 +84,30 @@ def import_modules_from_package(package): try_append_module(module_name, sys.modules) -def get_para_from_yaml(file_path, args): - - def func(a, b): - if a is None: - return None - return a.get(b) - - if os.path.exists(file_path): +def parse_yaml(file_path): + try: with open(file_path) as f: value = yaml.safe_load(f) - value = reduce(func, args.split('.'), value) + except IOError: + return {} + except OSError as e: + if e.errno != errno.EEXIST: + raise + else: + return value - if value is None: - print('parameter not found') - return None - return value - else: - print('file not exist') - return None +def get_param(key, default=''): + + conf_file = os.environ.get('CONF_FILE', '/etc/yardstick/yardstick.yaml') + + conf = parse_yaml(conf_file) + try: + return reduce(lambda a, b: a[b], key.split('.'), conf) + except KeyError: + if not default: + raise + return default def makedirs(d): diff --git a/yardstick/definitions.py b/yardstick/definitions.py deleted file mode 100644 index 64a4a80d4..000000000 --- a/yardstick/definitions.py +++ /dev/null @@ -1,14 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -from __future__ import absolute_import -import os - -dirname = os.path.dirname -YARDSTICK_ROOT_PATH = dirname(dirname(os.path.abspath(__file__))) -YARDSTICK_ROOT_PATH += os.path.sep -- cgit 1.2.3-korg