diff options
-rw-r--r-- | docker/Dockerfile | 5 | ||||
-rw-r--r-- | docs/testing/developer/design/apidocs/index.rst (renamed from docs/development/design/apidocs/index.rst) | 0 | ||||
-rw-r--r-- | docs/testing/developer/design/apidocs/qtip_restful_api.rst (renamed from docs/development/design/apidocs/qtip_restful_api.rst) | 0 | ||||
-rw-r--r-- | docs/testing/developer/design/compute_QPI.rst (renamed from docs/development/design/compute_QPI.rst) | 0 | ||||
-rw-r--r-- | docs/testing/developer/design/dashboard.rst (renamed from docs/development/design/dashboard.rst) | 0 | ||||
-rw-r--r-- | docs/testing/developer/design/index.rst (renamed from docs/development/design/index.rst) | 0 | ||||
-rw-r--r-- | docs/testing/developer/design/integration_with_yardstick.rst (renamed from docs/development/design/integration_with_yardstick.rst) | 0 | ||||
-rw-r--r-- | docs/testing/developer/overview/.gitkeep (renamed from docs/development/overview/.gitkeep) | 0 | ||||
-rw-r--r-- | docs/testing/developer/requirement/.gitkeep (renamed from docs/development/requirement/.gitkeep) | 0 | ||||
-rw-r--r-- | docs/testing/user/.gitkeep (renamed from docs/release/installation/.gitkeep) | 0 | ||||
-rw-r--r-- | docs/testing/user/configguide/configuration.rst (renamed from docs/release/configguide/configuration.rst) | 0 | ||||
-rw-r--r-- | docs/testing/user/configguide/index.rst (renamed from docs/release/configguide/index.rst) | 0 | ||||
-rw-r--r-- | docs/testing/user/installation/.gitkeep (renamed from docs/release/scenarios/.gitkeep) | 0 | ||||
-rw-r--r-- | docs/testing/user/scenarios/.gitkeep (renamed from qtip/driver/ansible/__init__.py) | 0 | ||||
-rw-r--r-- | docs/testing/user/userguide/_01-compute.rst (renamed from docs/release/userguide/_01-compute.rst) | 0 | ||||
-rw-r--r-- | docs/testing/user/userguide/_02-network.rst (renamed from docs/release/userguide/_02-network.rst) | 0 | ||||
-rw-r--r-- | docs/testing/user/userguide/_03-storage.rst (renamed from docs/release/userguide/_03-storage.rst) | 0 | ||||
-rw-r--r-- | docs/testing/user/userguide/_testcase_description.rst (renamed from docs/release/userguide/_testcase_description.rst) | 0 | ||||
-rw-r--r-- | docs/testing/user/userguide/annex.rst (renamed from docs/release/userguide/annex.rst) | 0 | ||||
-rw-r--r-- | docs/testing/user/userguide/benchmark-suites.rst (renamed from docs/release/userguide/benchmark-suites.rst) | 0 | ||||
-rw-r--r-- | docs/testing/user/userguide/index.rst (renamed from docs/release/userguide/index.rst) | 0 | ||||
-rw-r--r-- | docs/testing/user/userguide/introduction.rst (renamed from docs/release/userguide/introduction.rst) | 0 | ||||
-rw-r--r-- | qtip/api/__main__.py | 9 | ||||
-rw-r--r-- | qtip/api/swagger/swagger.yaml | 9 | ||||
-rw-r--r-- | qtip/cli/commands/cmd_report.py | 27 | ||||
-rw-r--r-- | qtip/cli/entry.py | 29 | ||||
-rw-r--r-- | qtip/driver/ansible.py (renamed from qtip/driver/ansible/ansible.py) | 0 | ||||
-rw-r--r-- | qtip/driver/playbook/bwn_ng.yaml (renamed from qtip/driver/ansible/playbook/bwn_ng.yaml) | 0 | ||||
-rw-r--r-- | qtip/driver/playbook/dhrystone/clean.yaml | 27 | ||||
-rw-r--r-- | qtip/driver/playbook/dhrystone/run.yaml | 63 | ||||
-rw-r--r-- | qtip/driver/playbook/dhrystone/setup.yaml | 66 | ||||
-rw-r--r-- | qtip/driver/playbook/inxi.yaml (renamed from qtip/driver/ansible/playbook/inxi.yaml) | 0 | ||||
-rw-r--r-- | qtip/driver/playbook/prepare_env.yaml (renamed from qtip/driver/ansible/playbook/prepare_env.yaml) | 14 | ||||
-rw-r--r-- | qtip/driver/playbook/top.yaml (renamed from qtip/driver/ansible/playbook/top.yaml) | 0 | ||||
-rw-r--r-- | qtip/reporter/console.py | 15 | ||||
-rw-r--r-- | qtip/reporter/templates/report.j2 | 22 | ||||
-rw-r--r-- | qtip/reporter/templates/timeline.j2 | 2 | ||||
-rwxr-xr-x | qtip/scripts/generate_host_file.sh (renamed from qtip/scripts/fetch_compute_ips.sh) | 39 | ||||
-rwxr-xr-x | qtip/scripts/qtip_creds.sh | 24 | ||||
-rw-r--r-- | qtip/util/env.py | 153 | ||||
-rw-r--r-- | tests/data/reporter/timeline.pickle | 3 | ||||
-rw-r--r-- | tests/unit/cli/cmd_report.py | 23 | ||||
-rw-r--r-- | tests/unit/cli/options_test.py | 9 | ||||
-rw-r--r-- | tests/unit/reporter/console_test.py | 22 | ||||
-rw-r--r-- | tests/unit/util/env_test.py | 94 | ||||
-rw-r--r-- | tests/unit/util/logger_test.py | 9 | ||||
-rw-r--r-- | third-party/License/.gitrepo | 6 | ||||
-rw-r--r-- | third-party/License/README.md | 5 |
48 files changed, 507 insertions, 168 deletions
diff --git a/docker/Dockerfile b/docker/Dockerfile index f2a543e6..13665131 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,6 +22,9 @@ RUN apt-get update && apt-get install -y \ gcc \ ssh \ expect \ + build-essential\ + libssl-dev\ + libffi-dev\ python-matplotlib \ python-dev \ python-pip \ @@ -57,7 +60,7 @@ RUN git config --global http.sslVerify false RUN git clone -b $BRANCH https://gerrit.opnfv.org/gerrit/qtip $REPOS_DIR/qtip RUN git clone https://gerrit.opnfv.org/gerrit/releng $REPOS_DIR/releng -RUN pip install -r $REPOS_DIR/qtip/requirements.txt +RUN pip install -U -r $REPOS_DIR/qtip/requirements.txt #Config supervisor RUN mkdir -p /var/log/supervisor diff --git a/docs/development/design/apidocs/index.rst b/docs/testing/developer/design/apidocs/index.rst index 241a2680..241a2680 100644 --- a/docs/development/design/apidocs/index.rst +++ b/docs/testing/developer/design/apidocs/index.rst diff --git a/docs/development/design/apidocs/qtip_restful_api.rst b/docs/testing/developer/design/apidocs/qtip_restful_api.rst index 06c01292..06c01292 100644 --- a/docs/development/design/apidocs/qtip_restful_api.rst +++ b/docs/testing/developer/design/apidocs/qtip_restful_api.rst diff --git a/docs/development/design/compute_QPI.rst b/docs/testing/developer/design/compute_QPI.rst index 2e5aa87c..2e5aa87c 100644 --- a/docs/development/design/compute_QPI.rst +++ b/docs/testing/developer/design/compute_QPI.rst diff --git a/docs/development/design/dashboard.rst b/docs/testing/developer/design/dashboard.rst index 60c4720d..60c4720d 100644 --- a/docs/development/design/dashboard.rst +++ b/docs/testing/developer/design/dashboard.rst diff --git a/docs/development/design/index.rst b/docs/testing/developer/design/index.rst index b6dd0c01..b6dd0c01 100644 --- a/docs/development/design/index.rst +++ b/docs/testing/developer/design/index.rst diff --git a/docs/development/design/integration_with_yardstick.rst b/docs/testing/developer/design/integration_with_yardstick.rst index a8298d6f..a8298d6f 100644 --- a/docs/development/design/integration_with_yardstick.rst +++ b/docs/testing/developer/design/integration_with_yardstick.rst diff --git a/docs/development/overview/.gitkeep b/docs/testing/developer/overview/.gitkeep index e69de29b..e69de29b 100644 --- a/docs/development/overview/.gitkeep +++ b/docs/testing/developer/overview/.gitkeep diff --git a/docs/development/requirement/.gitkeep b/docs/testing/developer/requirement/.gitkeep index e69de29b..e69de29b 100644 --- a/docs/development/requirement/.gitkeep +++ b/docs/testing/developer/requirement/.gitkeep diff --git a/docs/release/installation/.gitkeep b/docs/testing/user/.gitkeep index e69de29b..e69de29b 100644 --- a/docs/release/installation/.gitkeep +++ b/docs/testing/user/.gitkeep diff --git a/docs/release/configguide/configuration.rst b/docs/testing/user/configguide/configuration.rst index 78e96492..78e96492 100644 --- a/docs/release/configguide/configuration.rst +++ b/docs/testing/user/configguide/configuration.rst diff --git a/docs/release/configguide/index.rst b/docs/testing/user/configguide/index.rst index d5e05d63..d5e05d63 100644 --- a/docs/release/configguide/index.rst +++ b/docs/testing/user/configguide/index.rst diff --git a/docs/release/scenarios/.gitkeep b/docs/testing/user/installation/.gitkeep index e69de29b..e69de29b 100644 --- a/docs/release/scenarios/.gitkeep +++ b/docs/testing/user/installation/.gitkeep diff --git a/qtip/driver/ansible/__init__.py b/docs/testing/user/scenarios/.gitkeep index e69de29b..e69de29b 100644 --- a/qtip/driver/ansible/__init__.py +++ b/docs/testing/user/scenarios/.gitkeep diff --git a/docs/release/userguide/_01-compute.rst b/docs/testing/user/userguide/_01-compute.rst index 56be5488..56be5488 100644 --- a/docs/release/userguide/_01-compute.rst +++ b/docs/testing/user/userguide/_01-compute.rst diff --git a/docs/release/userguide/_02-network.rst b/docs/testing/user/userguide/_02-network.rst index 00fe5b0a..00fe5b0a 100644 --- a/docs/release/userguide/_02-network.rst +++ b/docs/testing/user/userguide/_02-network.rst diff --git a/docs/release/userguide/_03-storage.rst b/docs/testing/user/userguide/_03-storage.rst index b1490432..b1490432 100644 --- a/docs/release/userguide/_03-storage.rst +++ b/docs/testing/user/userguide/_03-storage.rst diff --git a/docs/release/userguide/_testcase_description.rst b/docs/testing/user/userguide/_testcase_description.rst index d60ca949..d60ca949 100644 --- a/docs/release/userguide/_testcase_description.rst +++ b/docs/testing/user/userguide/_testcase_description.rst diff --git a/docs/release/userguide/annex.rst b/docs/testing/user/userguide/annex.rst index e8bf5555..e8bf5555 100644 --- a/docs/release/userguide/annex.rst +++ b/docs/testing/user/userguide/annex.rst diff --git a/docs/release/userguide/benchmark-suites.rst b/docs/testing/user/userguide/benchmark-suites.rst index 84d1c647..84d1c647 100644 --- a/docs/release/userguide/benchmark-suites.rst +++ b/docs/testing/user/userguide/benchmark-suites.rst diff --git a/docs/release/userguide/index.rst b/docs/testing/user/userguide/index.rst index 4be3e498..4be3e498 100644 --- a/docs/release/userguide/index.rst +++ b/docs/testing/user/userguide/index.rst diff --git a/docs/release/userguide/introduction.rst b/docs/testing/user/userguide/introduction.rst index 3147f0aa..3147f0aa 100644 --- a/docs/release/userguide/introduction.rst +++ b/docs/testing/user/userguide/introduction.rst diff --git a/qtip/api/__main__.py b/qtip/api/__main__.py index 89298e6d..aa2941a7 100644 --- a/qtip/api/__main__.py +++ b/qtip/api/__main__.py @@ -1,3 +1,12 @@ +############################################################################## +# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + import connexion diff --git a/qtip/api/swagger/swagger.yaml b/qtip/api/swagger/swagger.yaml index 97a9c352..a5a815f1 100644 --- a/qtip/api/swagger/swagger.yaml +++ b/qtip/api/swagger/swagger.yaml @@ -1,3 +1,12 @@ +############################################################################## +# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + swagger: '2.0' info: title: QTIP-API diff --git a/qtip/cli/commands/cmd_report.py b/qtip/cli/commands/cmd_report.py new file mode 100644 index 00000000..c780e847 --- /dev/null +++ b/qtip/cli/commands/cmd_report.py @@ -0,0 +1,27 @@ +############################################################################## +# Copyright (c) 2017 taseer94@gmail.com and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import click + +from qtip.cli.entry import Context + +pass_context = click.make_pass_decorator(Context, ensure=False) + + +@click.group() +@pass_context +def cli(ctx): + """ View Qtip results""" + pass + + +@cli.command('show') +@pass_context +def show(ctx): + pass diff --git a/qtip/cli/entry.py b/qtip/cli/entry.py index 9f51f25e..6cf78b58 100644 --- a/qtip/cli/entry.py +++ b/qtip/cli/entry.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2016 ZTE Corp and others. +# Copyright (c) 2017 taseer94@gmail.com and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -7,33 +7,19 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +import click import os import sys -import click CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) +# TODO (taseer) define user friendly error messages +sys.tracebacklimit = 0 -class Context(object): - - ''' TODO(taseer) implementation''' - - def __init__(self): - self.verbose = False - self.debug = False - - def log(self, msg, *args): - ''' Log message to stderr ''' - pass - def verbose(self, msg, *args): - ''' Log message to stderr when verbose ''' - pass - - def debug(self, msg, *args): - ''' Log message to debug ''' - pass +class Context(object): + """ Load configuration and pass to subcommands """ pass_context = click.make_pass_decorator(Context, ensure=True) @@ -70,4 +56,5 @@ class QtipCli(click.MultiCommand): @click.version_option('dev') @pass_context def cli(ctx, verbose, debug): - pass + if debug: + sys.tracebacklimit = 8 diff --git a/qtip/driver/ansible/ansible.py b/qtip/driver/ansible.py index cd17625d..cd17625d 100644 --- a/qtip/driver/ansible/ansible.py +++ b/qtip/driver/ansible.py diff --git a/qtip/driver/ansible/playbook/bwn_ng.yaml b/qtip/driver/playbook/bwn_ng.yaml index 99477856..99477856 100644 --- a/qtip/driver/ansible/playbook/bwn_ng.yaml +++ b/qtip/driver/playbook/bwn_ng.yaml diff --git a/qtip/driver/playbook/dhrystone/clean.yaml b/qtip/driver/playbook/dhrystone/clean.yaml new file mode 100644 index 00000000..72bfab7e --- /dev/null +++ b/qtip/driver/playbook/dhrystone/clean.yaml @@ -0,0 +1,27 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: hosts + become: yes + remote_user: root + + tasks: + - name: Checking home directory + shell: echo $HOME + register: home_dir + + - name: Cleaning tempT + file: + path: '{{ home_dir.stdout }}/tempT' + state: absent + + - name: Cleaning qtip_result + file: + path: '{{ home_dir.stdout }}/qtip_result' + state: absent diff --git a/qtip/driver/playbook/dhrystone/run.yaml b/qtip/driver/playbook/dhrystone/run.yaml new file mode 100644 index 00000000..55de6597 --- /dev/null +++ b/qtip/driver/playbook/dhrystone/run.yaml @@ -0,0 +1,63 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: hosts + become: yes + remote_user: root + + tasks: + - name: Get current timestamp + set_fact: + timestamp: "{{ lookup('pipe', 'date +%Y-%m-%d-%H-%M') }}" + + - name: Checking home directory + shell: echo $HOME + register: home_dir + + - name: Fetch hostname + shell: hostname + register: host_name + + - name: Make UnixBench + shell: make --directory $HOME/tempT/UnixBench/ + + - name: Make some directories needed + file: + path: '{{ home_dir.stdout }}/qtip_result/{{ timestamp }}/{{ host_name.stdout }}' + state: directory + + - include: ../inxi.yaml + + - include: ../top.yaml + + - name: Run dhrystone + shell: ./Run -v dhrystone + args: + chdir: '{{ home_dir.stdout }}/tempT/UnixBench/' + + - name: Copying result to qtip result directory + shell: cp -r $HOME/tempT/UnixBench/results/* ./ + args: + chdir: '{{ home_dir.stdout }}/qtip_result/{{ timestamp }}/{{ host_name.stdout }}' + + - name: Copy top log to qtip result directory + shell: mv $HOME/qtip_result/top.log ./ + args: + chdir: '{{ home_dir.stdout }}/qtip_result/{{ timestamp }}/{{ host_name.stdout }}' + + - name: Copy inxi log to qtip result directory + shell: mv $HOME/qtip_result/inxi.log ./ + args: + chdir: '{{ home_dir.stdout }}/qtip_result/{{ timestamp }}/{{ host_name.stdout }}' + + - name: Fetch result files to local manchine + synchronize: + mode: pull + src: '{{ home_dir.stdout }}/qtip_result/' + dest: '{{ result_dir }}/dhrystone/logs/' diff --git a/qtip/driver/playbook/dhrystone/setup.yaml b/qtip/driver/playbook/dhrystone/setup.yaml new file mode 100644 index 00000000..430670c1 --- /dev/null +++ b/qtip/driver/playbook/dhrystone/setup.yaml @@ -0,0 +1,66 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: localhost + connection: local + gather_facts: no + + tasks: + - name: Making dhrystone directory + file: + path: '{{ result_dir }}/dhrystone/logs/' + state: directory + +- hosts: hosts + become: yes + remote_user: root + + tasks: + - name: Checking home directory + shell: echo $HOME + register: home_dir + + - name: Cleaning tempT directory + file: + path: '{{ home_dir.stdout }}/tempT' + state: absent + + - name: Cleaning qtip_result directory + file: + path: '{{ home_dir.stdout }}/qtip_result' + state: absent + + - include: ../prepare_env.yaml + + - name: Installing UnixBench dependencies if CentOS + yum: + name: '{{ item }}' + state: present + when: ansible_os_family == "RedHat" + with_items: + - git + - gcc + - patch + - perl-Time-HiRes + + - name: Installing UnixBench dependencies if Ubuntu + apt: + name: '{{ item }}' + state: present + when: ansible_os_family == "Debian" + with_items: + - git + - gcc + - patch + - perl + + - name: Clone unixbench + git: + repo: https://github.com/kdlucas/byte-unixbench.git + dest: '{{ home_dir.stdout }}/tempT' diff --git a/qtip/driver/ansible/playbook/inxi.yaml b/qtip/driver/playbook/inxi.yaml index f6a0311d..f6a0311d 100644 --- a/qtip/driver/ansible/playbook/inxi.yaml +++ b/qtip/driver/playbook/inxi.yaml diff --git a/qtip/driver/ansible/playbook/prepare_env.yaml b/qtip/driver/playbook/prepare_env.yaml index 0595d988..1ec71520 100644 --- a/qtip/driver/ansible/playbook/prepare_env.yaml +++ b/qtip/driver/playbook/prepare_env.yaml @@ -10,7 +10,7 @@ yum: name: epel-release state: present - when: ansible_os_family == "RedHat" + when: ansible_os_family == "RedHat" - name: Software Properties Common apt: @@ -40,3 +40,15 @@ name: python-selinux state: present when: ansible_os_family == "Debian" + +- name: Install rsync when CentOS + yum: + name: rsync + state: present + when: ansible_os_family == "RedHat" + +- name: Install rsync when Ubuntu + apt: + name: rsync + state: present + when: ansible_os_family == "Debian" diff --git a/qtip/driver/ansible/playbook/top.yaml b/qtip/driver/playbook/top.yaml index 8de7e3d6..8de7e3d6 100644 --- a/qtip/driver/ansible/playbook/top.yaml +++ b/qtip/driver/playbook/top.yaml diff --git a/qtip/reporter/console.py b/qtip/reporter/console.py index 24c98e74..2b5130a6 100644 --- a/qtip/reporter/console.py +++ b/qtip/reporter/console.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2016 ZTE Corp and others. +# Copyright (c) 2017 taseer94@gmail.com and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -7,6 +7,9 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +from jinja2 import Environment +from jinja2 import FileSystemLoader +from os import path from qtip.base import BaseActor @@ -16,8 +19,12 @@ class ConsoleReporter(BaseActor): """ def __init__(self, config, parent=None): super(ConsoleReporter, self).__init__(config, parent=parent) - # TODO(yujunz) remove PoC code - self._fmt = "{title}: {description}" + + # TODO (taseer) load template from config + tpl_loader = FileSystemLoader(path.join(path.dirname(__file__), 'templates')) + env = Environment(loader=tpl_loader) + self._template = env.get_template('timeline.j2') def render(self, var_dict): - return self._fmt.format(**var_dict) + out = self._template.render(var_dict) + return out diff --git a/qtip/reporter/templates/report.j2 b/qtip/reporter/templates/report.j2 new file mode 100644 index 00000000..766e6dde --- /dev/null +++ b/qtip/reporter/templates/report.j2 @@ -0,0 +1,22 @@ +{{ title }} + +Plan: {{ plan.name }} + +{{ qpi.name }}: {{ qpi.score }} +Sections: +{% for section in sections %} + {{ section.name }}: {{ section.score }} + + Formula: {{ section.formula }} + Metrics: + {% for metric in section.metrics %} + {{ metric.name }}: {{ metric.score }} + Formula: {{ metric.formula }} + Workloads: + {% for workload in workloads %} + {{ workload.name }}: {{ workload.score }} + {% endfor %} + {% endfor %} +{% endfor %} + +{{ signature }} diff --git a/qtip/reporter/templates/timeline.j2 b/qtip/reporter/templates/timeline.j2 index 9c18a996..d4c95c46 100644 --- a/qtip/reporter/templates/timeline.j2 +++ b/qtip/reporter/templates/timeline.j2 @@ -1,4 +1,4 @@ -{% title %} +{{ title }} {% for phase in phases %} {{ phase.name|upper }}{{ "TIME" }} {% for cp in phase.checkpoints %} diff --git a/qtip/scripts/fetch_compute_ips.sh b/qtip/scripts/generate_host_file.sh index a2618ec9..ecc4d55f 100755 --- a/qtip/scripts/fetch_compute_ips.sh +++ b/qtip/scripts/generate_host_file.sh @@ -10,17 +10,15 @@ usage(){ - echo "usage: $0 [-v] -i <installer_type> -a <installer_ip>" >&2 - echo "[-v] Virtualized deployment" >&2 + echo "usage: $0 -t <installer_type> -i <installer_ipaddr> -d <dest_hostfile>" >&2 } info() { - logger -s -t "fetch_compute_info.info" "$*" + logger -s -t "generate_host_file.info" "$*" } - error() { - logger -s -t "fetch_compute_info.error" "$*" + logger -s -t "generate_host_file.error" "$*" exit 1 } @@ -40,11 +38,11 @@ verify_connectivity(){ :${DEPLOY_TYPE:=''} #Getoptions -while getopts ":i:a:h:v" optchar; do +while getopts ":t:i:d:" optchar; do case "${optchar}" in - i) installer_type=${OPTARG} ;; - a) installer_ip=${OPTARG} ;; - v) DEPLOY_TYPE="virt" ;; + t) installer_type=${OPTARG} ;; + i) installer_ipaddr=${OPTARG} ;; + d) dest_hostfile=${OPTARG} ;; *) echo "Non-option argument: '-${OPTARG}'" >&2 usage exit 2 @@ -54,9 +52,9 @@ done #set vars from env if not provided by user as options installer_type=${installer_type:-$INSTALLER_TYPE} -installer_ip=${installer_ip:-$INSTALLER_IP} +installer_ipaddr=${installer_ipaddr:-$INSTALLER_IP} -if [ -z $installer_type ] || [ -z $installer_ip ]; then +if [ -z $installer_type ] || [ -z $installer_ipaddr ]; then usage exit 2 fi @@ -65,9 +63,9 @@ ssh_options="-oUserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" #Start fetching compute ip if [ "$installer_type" == "fuel" ]; then - verify_connectivity $installer_ip + verify_connectivity $installer_ipaddr - env=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ip} \ + env=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ipaddr} \ 'fuel env'|grep operational|head -1|awk '{print $1}') &> /dev/null if [ -z $env ]; then error "No operational environment detected in Fuel" @@ -75,7 +73,7 @@ if [ "$installer_type" == "fuel" ]; then env_id="${FUEL_ENV:-$env}" # Check if compute is alive (online='True') - IPS=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ip} \ + IPS=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ipaddr} \ "fuel node --env ${env_id} | grep compute | grep 'True\| 1' | awk -F\| '{print \$5}' " | \ sed 's/ //g') &> /dev/null @@ -86,8 +84,8 @@ elif [ "$installer_type" == "apex" ]; then elif [ "$installer_type" == "compass" ]; then # need test - verify_connectivity $installer_ip - IPS=$(sshpass -p'root' ssh 2>/dev/null $ssh_options root@${installer_ip} \ + verify_connectivity $installer_ipaddr + IPS=$(sshpass -p'root' ssh 2>/dev/null $ssh_options root@${installer_ipaddr} \ 'mysql -ucompass -pcompass -Dcompass -e"select * from cluster;"' \ | awk -F"," '{for(i=1;i<NF;i++)if($i~/\"host[4-5]\"/) {print $(i+1);}}' \ | grep -oP "\d+.\d+.\d+.\d+") @@ -108,10 +106,11 @@ if [ -z "$IPS" ]; then error "The compute node $IPS are not up. Please check that the POD is correctly deployed." else echo "-------- all compute node ips: --------" - rm $HOME/ips.log - touch $HOME/ips.log - echo "$IPS" > $HOME/qtip/ips.log - echo $IPS + rm $dest_hostfile + touch $dest_hostfile + echo "[hosts]" >> $dest_hostfile + echo "$IPS" >> $dest_hostfile + cat $dest_hostfile fi exit 0 diff --git a/qtip/scripts/qtip_creds.sh b/qtip/scripts/qtip_creds.sh index af051ac5..239c60c1 100755 --- a/qtip/scripts/qtip_creds.sh +++ b/qtip/scripts/qtip_creds.sh @@ -1,30 +1,34 @@ #! /bin/bash DEST_IP=$1 +PRIVATE_KEY=$2 +PUBLIC_KEY=$2.pub +KEYNAME=$(basename $PRIVATE_KEY) + echo $INSTALLER_TYPE echo $INSTALLER_IP sshoptions="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" case "$INSTALLER_TYPE" in apex) - scp $sshoptions -i $APEX_KEY ./config/QtipKey.pub stack@$INSTALLER_IP:/home/stack - scp $sshoptions -i $APEX_KEY ./config/QtipKey stack@$INSTALLER_IP:/home/stack - ssh $sshoptions -i $APEX_KEY stack@$INSTALLER_IP "ssh-copy-id $sshoptions -i /home/stack/QtipKey.pub heat-admin@$DEST_IP && rm -rf /home/stack/QtipKey && rm -rf /home/stack/QtipKey.pub" + scp $sshoptions -i $APEX_KEY $PUBLIC_KEY stack@$INSTALLER_IP:/home/stack + scp $sshoptions -i $APEX_KEY $PRIVATE_KEY stack@$INSTALLER_IP:/home/stack + ssh $sshoptions -i $APEX_KEY stack@$INSTALLER_IP "ssh-copy-id $sshoptions -i /home/stack/$KEYNAME.pub heat-admin@$DEST_IP && rm -rf /home/stack/$KEYNAME && rm -rf /home/stack/$KEYNAME.pub" ;; fuel) PSWD="r00tme" - sshpass -p $PSWD scp $sshoptions ./config/QtipKey.pub root@$INSTALLER_IP:/root - sshpass -p $PSWD scp $sshoptions ./config/QtipKey root@$INSTALLER_IP:/root + sshpass -p $PSWD scp $sshoptions $PUBLIC_KEY root@$INSTALLER_IP:/root + sshpass -p $PSWD scp $sshoptions $PRIVATE_KEY root@$INSTALLER_IP:/root sshpass -p $PSWD ssh $sshoptions root@$INSTALLER_IP "grep -q '\-F /dev/null ' /usr/bin/ssh-copy-id || sed -i 's/\(ssh -i.*$\)/\1\n -F \/dev\/null \\\/g' `which ssh-copy-id`" - sshpass -p $PSWD ssh $sshoptions root@$INSTALLER_IP "ssh-copy-id $sshoptions -i /root/QtipKey root@$DEST_IP && rm -rf /root/QtipKey && rm -rf /root/QtipKey.pub" + sshpass -p $PSWD ssh $sshoptions root@$INSTALLER_IP "ssh-copy-id $sshoptions -i /root/$KEYNAME root@$DEST_IP && rm -rf /root/$KEYNAME && rm -rf /root/$KEYNAME.pub" ;; compass) PSWD="root" - sshpass -p $PSWD scp $sshoptions ./config/QtipKey.pub root@$INSTALLER_IP:/root - sshpass -p $PSWD scp $sshoptions ./config/QtipKey root@$INSTALLER_IP:/root - sshpass -p $PSWD ssh $sshoptions root@$INSTALLER_IP "ssh-copy-id $sshoptions -i /root/QtipKey.pub root@$DEST_IP && rm -rf /root/QtipKey && rm -rf /root/QtipKey.pub" + sshpass -p $PSWD scp $sshoptions $PUBLIC_KEY root@$INSTALLER_IP:/root + sshpass -p $PSWD scp $sshoptions $PRIVATE_KEY root@$INSTALLER_IP:/root + sshpass -p $PSWD ssh $sshoptions root@$INSTALLER_IP "ssh-copy-id $sshoptions -i /root/$KEYNAME.pub root@$DEST_IP && rm -rf /root/$KEYNAME && rm -rf /root/$KEYNAME.pub" ;; joid) PSWD="joid";; *) echo "Unkown installer $INSTALLER_TYPE specified";; -esac +esac
\ No newline at end of file diff --git a/qtip/util/env.py b/qtip/util/env.py index 4e7a31c7..0585a4c1 100644 --- a/qtip/util/env.py +++ b/qtip/util/env.py @@ -7,131 +7,94 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## import os -import paramiko import socket import time from os import path -from os.path import expanduser + +import paramiko SCRIPT_DIR = path.join(path.dirname(__file__), path.pardir, 'scripts') -CONFIG_DIR = path.join(path.dirname(__file__), path.pardir, path.pardir, - 'config') -PRIVATE_KEY = CONFIG_DIR + '/QtipKey' -PUBLIC_KEY = CONFIG_DIR + '/QtipKey.pub' -IPS_FILE = expanduser('~') + "/qtip/ips.log" -HOST_FILE = CONFIG_DIR + "/host" +KEYNAME = 'QtipKey' +PRIVATE_KEY = '{0}/qtip/{1}'.format(os.environ['HOME'], KEYNAME) +PUBLIC_KEY = PRIVATE_KEY + '.pub' +HOST_FILE = '{0}/qtip/hosts'.format(os.environ['HOME']) -def fetch_compute_ips_via_installer(): - clean_file(IPS_FILE) +def all_files_exist(*files): + if len(files) == 0: + return False + flag = True + for f_item in files: + flag &= path.isfile(f_item) + print("Is {0} existed: {1}".format(f_item, flag)) + return flag + +def clean_file(*files): + if len(files) == 0: + print('Nothing to clean') + return False + + def clean(f): + try: + if all_files_exist(f): + os.remove(f) + print("Removed: {0}".format(f)) + else: + print("Not exists: {0}".format(f)) + return True + except OSError as error: + print("Not able to Remove: {0}".format(f), error) + return False + + results = map(clean, files) + return len(results) == len(files) and False not in results + + +def generate_host_file(hostfile=HOST_FILE): installer_type = str(os.environ['INSTALLER_TYPE'].lower()) installer_ip = str(os.environ['INSTALLER_IP']) + if installer_type not in ["fuel"]: - raise RuntimeError("%s is not supported" % installer_type) + raise ValueError("%s is not supported" % installer_type) if not installer_ip: - raise RuntimeError("undefine environment variable INSTALLER_IP") + raise ValueError("The value of environment variable INSTALLER_IP is empty") - cmd = "bash %s/fetch_compute_ips.sh -i %s -a %s" % \ - (SCRIPT_DIR, installer_type, installer_ip) + cmd = "bash %s/generate_host_file.sh -i %s -a %s -d %s" % \ + (SCRIPT_DIR, installer_type, installer_ip, hostfile) os.system(cmd) - if path.isfile(IPS_FILE): - return True - else: - return False + return all_files_exist(hostfile) -def parse_ips(): - ip_list = [] - with open(IPS_FILE, "r") as outfile: - data = outfile.read() - if data: - ip_list.extend(data.rstrip('\n').split('\n')) - return ip_list +def generate_keypair(keyname='QtipKey'): + """Generating ssh keypair""" + cmd = "ssh-keygen -t rsa -N "" -f {0} -q -b 2048".format(keyname) + os.system(cmd) + return all_files_exist(PRIVATE_KEY, PUBLIC_KEY) -def ssh_test(ip): +def pass_keypair(ip, private_key=PRIVATE_KEY): os.system('ssh-keyscan %s >> /root/.ssh/known_hosts' % ip) time.sleep(2) - ssh_cmd = '%s/qtip_creds.sh %s' % (SCRIPT_DIR, ip) + ssh_cmd = '%s/qtip_creds.sh %s %s' % (SCRIPT_DIR, ip, private_key) os.system(ssh_cmd) + +def ssh_is_ok(ip, private_key=PRIVATE_KEY, attempts=100): ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(ip, key_filename='{0}/QtipKey'.format(CONFIG_DIR)) + ssh.connect(ip, key_filename=private_key) - for attempts in range(100): + for attempt in range(attempts): try: stdin, stdout, stderr = ssh.exec_command('uname') if not stderr.readlines(): - print("{0}: SSH test successful") + print("{0}: SSH test successful".format(ip)) return True except socket.error: - if attempts == 99: + if attempt == (attempts - 1): return False + print("%s times ssh test......failed" % attempt) time.sleep(2) - - -def ping_test(ip, attempts=30): - ping_cmd = 'ping -D -c1 {0}'.format(ip) - for i in range(attempts): - if os.system(ping_cmd): - print('\nWaiting for machine\n') - time.sleep(10) - else: - print('\n\n %s is UP \n\n ' % ip) - return True - if i == 29: - return False - - -def check_nodes_connectivity(): - ip_list = parse_ips() - for ip in ip_list: - if not ping_test(ip): - raise RuntimeError("{0}: Ping test failed".format(ip)) - if not ssh_test(ip): - raise RuntimeError("{0}: SSH test failed".format(ip)) - - -def generate_host_file(): - ip_list = parse_ips() - with open(HOST_FILE, 'w') as host_file: - for index, item in enumerate(ip_list): - host_file.write("[host_{0}]\n".format(index)) - host_file.write(item + '\n') - - -def generate_keypair(): - """Generating ssh keypair""" - if not clean_keypair(): - raise RuntimeError("Cann't remove old keypair") - - cmd = "ssh-keygen -t rsa -N "" -f {0} -q".format(PRIVATE_KEY) - os.system(cmd) - - if path.isfile(PRIVATE_KEY) and path.isfile(PUBLIC_KEY): - return True - else: - return False - - -def clean_file(file_path): - try: - if path.isfile(file_path): - os.remove(file_path) - print("Removed: " + file_path) - else: - print("Not exists: " + file_path) - except OSError, error: - print("Not able to Remove: " + file_path, error) - return False - return True - - -def clean_keypair(): - flag = True - flag &= clean_file(PRIVATE_KEY) - flag &= clean_file(PUBLIC_KEY) - return flag + return False diff --git a/tests/data/reporter/timeline.pickle b/tests/data/reporter/timeline.pickle new file mode 100644 index 00000000..5c870d93 --- /dev/null +++ b/tests/data/reporter/timeline.pickle @@ -0,0 +1,3 @@ +VTimeline\u000a\u000aMONITOR TIME\u000a\u000aT00 1\u000a\u000a\u000aINSPECTOR TIME\u000a\u000aT01 2\u000a\u000aT02 5\u000a\u000aT03 8\u000a\u000a\u000aCONTROLLER TIME\u000a\u000aT04 11\u000a\u000a\u000aNOTIFIER TIME\u000a\u000aT05 16\u000a\u000a\u000aEVALUATOR TIME\u000a\u000aT06 40\u000a\u000a\u000aTotal: 312ms +p0 +. diff --git a/tests/unit/cli/cmd_report.py b/tests/unit/cli/cmd_report.py new file mode 100644 index 00000000..e010b960 --- /dev/null +++ b/tests/unit/cli/cmd_report.py @@ -0,0 +1,23 @@ +############################################################### +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import pytest +from click.testing import CliRunner + +from qtip.cli.entry import cli + + +@pytest.fixture(scope="module") +def runner(): + return CliRunner() + + +def test_show(runner): + result = runner.invoke(cli, ['report', 'show']) + assert result.output == '' diff --git a/tests/unit/cli/options_test.py b/tests/unit/cli/options_test.py index f9472814..9dbbe6f3 100644 --- a/tests/unit/cli/options_test.py +++ b/tests/unit/cli/options_test.py @@ -1,5 +1,5 @@ ############################################################### -# Copyright (c) 2016 ZTE Corp and others. +# Copyright (c) 2017 taseer94@gmail.com and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -8,8 +8,9 @@ ############################################################################## import pytest -from click.testing import CliRunner +import sys +from click.testing import CliRunner from qtip.cli.entry import cli @@ -28,5 +29,5 @@ class TestClass(object): assert 'dev' in result.output def test_debug(self, runner): - result = runner.invoke(cli, ['-d']) - assert '' in result.output + runner.invoke(cli, ['-d']) + assert sys.tracebacklimit == 8 diff --git a/tests/unit/reporter/console_test.py b/tests/unit/reporter/console_test.py index 8150239e..d2816690 100644 --- a/tests/unit/reporter/console_test.py +++ b/tests/unit/reporter/console_test.py @@ -7,7 +7,10 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +import pickle import pytest +import os + from qtip.reporter.console import ConsoleReporter @@ -21,9 +24,16 @@ def test_constructor(console_reporter): def test_render(console_reporter): - var_dict = { - 'title': 'fake title', - 'description': 'fake description' - } - output = console_reporter.render(var_dict=var_dict) - assert output == 'fake title: fake description' + var_dict = {'title': 'Timeline', 'total': '312ms', 'phases': [{'name': 'Monitor ', + 'checkpoints': [{'name': 'T00 ', 'timestamp': '1'}]}, + {'name': 'Inspector ', 'checkpoints': [{'name': 'T01 ', 'timestamp': '2'}, + {'name': 'T02 ', 'timestamp': '5'}, {'name': 'T03 ', 'timestamp': '8'}]}, + {'name': 'Controller ', 'checkpoints': [{'name': 'T04 ', 'timestamp': '11'}]}, + {'name': 'Notifier ', 'checkpoints': [{'name': 'T05 ', 'timestamp': '16'}]}, + {'name': 'Evaluator ', 'checkpoints': [{'name': 'T06 ', 'timestamp': '40'}]}]} + + result = console_reporter.render(var_dict=var_dict) + path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, + os.pardir, 'tests/data/reporter/') + timeline = pickle.load(open(path + 'timeline.pickle', 'rb')) + assert result == timeline diff --git a/tests/unit/util/env_test.py b/tests/unit/util/env_test.py new file mode 100644 index 00000000..38ac988b --- /dev/null +++ b/tests/unit/util/env_test.py @@ -0,0 +1,94 @@ +############################################################### +# Copyright (c) 2017 ZTE Corporation +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import time + +import mock +import pytest + +from qtip.util import env + + +def test_all_files_exist(tmpdir): + exist_file = tmpdir.mkdir('qtip').join('hello.txt') + exist_file.write("hello") + non_exist_file = tmpdir.strpath + '/tmp.txt' + assert env.all_files_exist() is False + assert env.all_files_exist(str(exist_file)) + assert env.all_files_exist(non_exist_file) is False + assert env.all_files_exist(str(exist_file), non_exist_file) is False + + +def test_clean_file(tmpdir): + exist_file = tmpdir.mkdir('qtip').join('hello.txt') + exist_file.write("hello") + non_exist_file = tmpdir.strpath + '/tmp.txt' + + assert env.clean_file() is False + assert env.clean_file(str(exist_file)) + assert env.clean_file(non_exist_file) + + +def test_generate_host_file_without_setenv(monkeypatch): + def setenv(*args): + monkeypatch.setenv('INSTALLER_TYPE', args[0]) + monkeypatch.setenv('INSTALLER_IP', args[1]) + + with pytest.raises(KeyError) as excinfo: + env.generate_host_file() + assert 'INSTALLER_TYPE' in str(excinfo.value) + + with pytest.raises(ValueError) as excinfo: + setenv('fuel_1', '10.20.0.2') + env.generate_host_file() + assert 'fuel_1 is not supported' in str(excinfo.value) + + with pytest.raises(ValueError) as excinfo: + setenv('fuel', '') + env.generate_host_file() + assert 'The value of environment variable INSTALLER_IP is empty' \ + in str(excinfo.value) + + +def test_generate_host_file(monkeypatch, tmpdir): + monkeypatch.setenv('INSTALLER_TYPE', 'fuel') + monkeypatch.setenv('INSTALLER_IP', '10.20.0.2') + hostfile = tmpdir.mkdir('qtip').join('hosts') + hostfile.write('') + assert env.generate_host_file(str(hostfile)) + + +def test_generate_keypair(): + with mock.patch('os.system') as mock_os: + env.generate_keypair() + assert mock_os.call_count == 1 + + +def test_pass_keypair(monkeypatch): + monkeypatch.setattr(time, 'sleep', lambda s: None) + with mock.patch('os.system') as mock_os: + env.pass_keypair('10.20.0.10') + assert mock_os.call_count == 2 + + +@pytest.mark.parametrize("stderrinfo, expected", [ + ('', True), + ('sorry', False) +]) +@mock.patch('paramiko.SSHClient') +def test_ssh_is_ok(mock_sshclient, stderrinfo, expected): + stderr = mock.MagicMock() + stderr.readlines.return_value = stderrinfo + test_ssh_client = mock_sshclient.return_value + test_ssh_client.exec_command.return_value = ('', '', stderr) + result = env.ssh_is_ok('10.20.0.3') + assert result == expected + test_ssh_client.connect.assert_called_once_with( + '10.20.0.3', key_filename=env.PRIVATE_KEY) + test_ssh_client.exec_command.assert_called_with('uname') diff --git a/tests/unit/util/logger_test.py b/tests/unit/util/logger_test.py index 339b2bf6..78c4c109 100644 --- a/tests/unit/util/logger_test.py +++ b/tests/unit/util/logger_test.py @@ -1,3 +1,12 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + import pytest from qtip.util import logger diff --git a/third-party/License/.gitrepo b/third-party/License/.gitrepo index 29cd5a75..03fab6f7 100644 --- a/third-party/License/.gitrepo +++ b/third-party/License/.gitrepo @@ -5,7 +5,7 @@ ; [subrepo] remote = git@github.com:openzero-zte/License.git - branch = master - commit = 61489dae4453b66887d0d90a2244610a30f7e53c - parent = 3e443dff14a2be02b914e66f27b549d0ed4cc600 + branch = develop + commit = 88b1440008b6acac8ad65afc93606cddef63cea9 + parent = 473deae9a10162f000c49ca49b4e31b28c4bf0d8 cmdver = 0.3.0 diff --git a/third-party/License/README.md b/third-party/License/README.md index 0232de7e..57444c1f 100644 --- a/third-party/License/README.md +++ b/third-party/License/README.md @@ -5,6 +5,7 @@ A script for checking and adding license header according to [OPNFV contribution ## Quick Start ``` -$ cd <project-folder> -$ curl https://raw.githubusercontent.com/Justin-chi/License/master/add_license.sh |bash +$ cd <project-root> +$ curl https://raw.githubusercontent.com/openzero-team/License/develop/add_license.sh |bash ``` + |