diff options
28 files changed, 749 insertions, 82 deletions
diff --git a/qtip/cli/commands/cmd_project.py b/qtip/cli/commands/cmd_project.py index b32e71cd..325594ad 100644 --- a/qtip/cli/commands/cmd_project.py +++ b/qtip/cli/commands/cmd_project.py @@ -41,7 +41,8 @@ def cli(): @cli.command(help="Create new testing project") @click.argument('project_name') @click.option('--project-template', - type=click.Choice(['compute', 'doctor']), + # TODO(yujunz) create template list by directory name + type=click.Choice(['compute', 'doctor', 'storage']), default='compute', help='Choose project template') @click.option('--pod-name', diff --git a/resources/ansible_roles/qtip-common/defaults/main.yml b/resources/ansible_roles/qtip-common/defaults/main.yml new file mode 100644 index 00000000..47b8d107 --- /dev/null +++ b/resources/ansible_roles/qtip-common/defaults/main.yml @@ -0,0 +1,10 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +output: "{{ qtip_results }}"
\ No newline at end of file diff --git a/resources/ansible_roles/qtip-generator/files/storage/ansible.cfg b/resources/ansible_roles/qtip-generator/files/storage/ansible.cfg new file mode 100644 index 00000000..a80d4ae4 --- /dev/null +++ b/resources/ansible_roles/qtip-generator/files/storage/ansible.cfg @@ -0,0 +1,447 @@ +# config file for ansible -- https://ansible.com/ +# =============================================== + +# nearly all parameters can be overridden in ansible-playbook +# or with command line flags. ansible will read ANSIBLE_CONFIG, +# ansible.cfg in the current working directory, .ansible.cfg in +# the home directory or /etc/ansible/ansible.cfg, whichever it +# finds first + +[defaults] + +# some basic default values... + +inventory = hosts +library = {{ qtip_package }}/qtip/ansible_library/ +#module_utils = /usr/share/my_module_utils/ +#remote_tmp = ~/.ansible/tmp +#local_tmp = ~/.ansible/tmp +#forks = 5 +#poll_interval = 15 +#sudo_user = root +#ask_sudo_pass = True +#ask_pass = True +#transport = smart +#remote_port = 22 +#module_lang = C +#module_set_locale = False + +# plays will gather facts by default, which contain information about +# the remote system. +# +# smart - gather by default, but don't regather if already gathered +# implicit - gather by default, turn off with gather_facts: False +# explicit - do not gather by default, must say gather_facts: True +gathering = smart + +# This only affects the gathering done by a play's gather_facts directive, +# by default gathering retrieves all facts subsets +# all - gather all subsets +# network - gather min and network facts +# hardware - gather hardware facts (longest facts to retrieve) +# virtual - gather min and virtual facts +# facter - import facts from facter +# ohai - import facts from ohai +# You can combine them using comma (ex: network,virtual) +# You can negate them using ! (ex: !hardware,!facter,!ohai) +# A minimal set of facts is always gathered. +#gather_subset = all + +# some hardware related facts are collected +# with a maximum timeout of 10 seconds. This +# option lets you increase or decrease that +# timeout to something more suitable for the +# environment. +# gather_timeout = 10 + +# additional paths to search for roles in, colon separated +#roles_path = /etc/ansible/roles +roles_path = {{ qtip_package }}/resources/ansible_roles + +# uncomment this to disable SSH key host checking +#host_key_checking = False + +# change the default callback +#stdout_callback = skippy +# enable additional callbacks +callback_whitelist = profile_tasks + +# Determine whether includes in tasks and handlers are "static" by +# default. As of 2.0, includes are dynamic by default. Setting these +# values to True will make includes behave more like they did in the +# 1.x versions. +#task_includes_static = True +#handler_includes_static = True + +# Controls if a missing handler for a notification event is an error or a warning +#error_on_missing_handler = True + +# change this for alternative sudo implementations +#sudo_exe = sudo + +# What flags to pass to sudo +# WARNING: leaving out the defaults might create unexpected behaviours +#sudo_flags = -H -S -n + +# SSH timeout +#timeout = 10 + +# default user to use for playbooks if user is not specified +# (/usr/bin/ansible will use current user as default) +#remote_user = root + +# logging is off by default unless this path is defined +# if so defined, consider logrotate +#log_path = /var/log/ansible.log + +# default module name for /usr/bin/ansible +#module_name = command + +# use this shell for commands executed under sudo +# you may need to change this to bin/bash in rare instances +# if sudo is constrained +#executable = /bin/sh + +# if inventory variables overlap, does the higher precedence one win +# or are hash values merged together? The default is 'replace' but +# this can also be set to 'merge'. +#hash_behaviour = replace + +# by default, variables from roles will be visible in the global variable +# scope. To prevent this, the following option can be enabled, and only +# tasks and handlers within the role will see the variables there +#private_role_vars = yes + +# list any Jinja2 extensions to enable here: +#jinja2_extensions = jinja2.ext.do,jinja2.ext.i18n + +# if set, always use this private key file for authentication, same as +# if passing --private-key to ansible or ansible-playbook +#private_key_file = /path/to/file + +# If set, configures the path to the Vault password file as an alternative to +# specifying --vault-password-file on the command line. +#vault_password_file = /path/to/vault_password_file + +# format of string {{ ansible_managed }} available within Jinja2 +# templates indicates to users editing templates files will be replaced. +# replacing {file}, {host} and {uid} and strftime codes with proper values. +#ansible_managed = Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S by {uid} on {host} +# {file}, {host}, {uid}, and the timestamp can all interfere with idempotence +# in some situations so the default is a static string: +#ansible_managed = Ansible managed + +# by default, ansible-playbook will display "Skipping [host]" if it determines a task +# should not be run on a host. Set this to "False" if you don't want to see these "Skipping" +# messages. NOTE: the task header will still be shown regardless of whether or not the +# task is skipped. +#display_skipped_hosts = True + +# by default, if a task in a playbook does not include a name: field then +# ansible-playbook will construct a header that includes the task's action but +# not the task's args. This is a security feature because ansible cannot know +# if the *module* considers an argument to be no_log at the time that the +# header is printed. If your environment doesn't have a problem securing +# stdout from ansible-playbook (or you have manually specified no_log in your +# playbook on all of the tasks where you have secret information) then you can +# safely set this to True to get more informative messages. +#display_args_to_stdout = False + +# by default (as of 1.3), Ansible will raise errors when attempting to dereference +# Jinja2 variables that are not set in templates or action lines. Uncomment this line +# to revert the behavior to pre-1.3. +#error_on_undefined_vars = False + +# by default (as of 1.6), Ansible may display warnings based on the configuration of the +# system running ansible itself. This may include warnings about 3rd party packages or +# other conditions that should be resolved if possible. +# to disable these warnings, set the following value to False: +#system_warnings = True + +# by default (as of 1.4), Ansible may display deprecation warnings for language +# features that should no longer be used and will be removed in future versions. +# to disable these warnings, set the following value to False: +#deprecation_warnings = True + +# (as of 1.8), Ansible can optionally warn when usage of the shell and +# command module appear to be simplified by using a default Ansible module +# instead. These warnings can be silenced by adjusting the following +# setting or adding warn=yes or warn=no to the end of the command line +# parameter string. This will for example suggest using the git module +# instead of shelling out to the git command. +# command_warnings = False + + +# set plugin path directories here, separate with colons +action_plugins = {{ qtip_package }}/qtip/ansible_library/plugins/action +#cache_plugins = /usr/share/ansible/plugins/cache +#callback_plugins = /usr/share/ansible/plugins/callback +#connection_plugins = /usr/share/ansible/plugins/connection +#lookup_plugins = /usr/share/ansible/plugins/lookup +#inventory_plugins = /usr/share/ansible/plugins/inventory +#vars_plugins = /usr/share/ansible/plugins/vars +filter_plugins = {{ qtip_package }}/qtip/ansible_library/plugins/filter +#test_plugins = /usr/share/ansible/plugins/test +#terminal_plugins = /usr/share/ansible/plugins/terminal +#strategy_plugins = /usr/share/ansible/plugins/strategy + + +# by default, ansible will use the 'linear' strategy but you may want to try +# another one +#strategy = free + +# by default callbacks are not loaded for /bin/ansible, enable this if you +# want, for example, a notification or logging callback to also apply to +# /bin/ansible runs +#bin_ansible_callbacks = False + + +# don't like cows? that's unfortunate. +# set to 1 if you don't want cowsay support or export ANSIBLE_NOCOWS=1 +#nocows = 1 + +# set which cowsay stencil you'd like to use by default. When set to 'random', +# a random stencil will be selected for each task. The selection will be filtered +# against the `cow_whitelist` option below. +#cow_selection = default +#cow_selection = random + +# when using the 'random' option for cowsay, stencils will be restricted to this list. +# it should be formatted as a comma-separated list with no spaces between names. +# NOTE: line continuations here are for formatting purposes only, as the INI parser +# in python does not support them. +#cow_whitelist=bud-frogs,bunny,cheese,daemon,default,dragon,elephant-in-snake,elephant,eyes,\ +# hellokitty,kitty,luke-koala,meow,milk,moofasa,moose,ren,sheep,small,stegosaurus,\ +# stimpy,supermilker,three-eyes,turkey,turtle,tux,udder,vader-koala,vader,www + +# don't like colors either? +# set to 1 if you don't want colors, or export ANSIBLE_NOCOLOR=1 +#nocolor = 1 + +# if set to a persistent type (not 'memory', for example 'redis') fact values +# from previous runs in Ansible will be stored. This may be useful when +# wanting to use, for example, IP information from one group of servers +# without having to talk to them in the same playbook run to get their +# current IP information. +fact_caching = jsonfile +fact_caching_connection = {{ qtip_cache }} +fact_caching_timeout = 86400 + + +# retry files +# When a playbook fails by default a .retry file will be created in ~/ +# You can disable this feature by setting retry_files_enabled to False +# and you can change the location of the files by setting retry_files_save_path + +#retry_files_enabled = False +#retry_files_save_path = ~/.ansible-retry + +# squash actions +# Ansible can optimise actions that call modules with list parameters +# when looping. Instead of calling the module once per with_ item, the +# module is called once with all items at once. Currently this only works +# under limited circumstances, and only with parameters named 'name'. +squash_actions = apk,apt,dnf,homebrew,pacman,pkgng,yum,zypper + +# prevents logging of task data, off by default +#no_log = False + +# prevents logging of tasks, but only on the targets, data is still logged on the master/controller +#no_target_syslog = False + +# controls whether Ansible will raise an error or warning if a task has no +# choice but to create world readable temporary files to execute a module on +# the remote machine. This option is False by default for security. Users may +# turn this on to have behaviour more like Ansible prior to 2.1.x. See +# https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user +# for more secure ways to fix this than enabling this option. +#allow_world_readable_tmpfiles = False + +# controls the compression level of variables sent to +# worker processes. At the default of 0, no compression +# is used. This value must be an integer from 0 to 9. +#var_compression_level = 9 + +# controls what compression method is used for new-style ansible modules when +# they are sent to the remote system. The compression types depend on having +# support compiled into both the controller's python and the client's python. +# The names should match with the python Zipfile compression types: +# * ZIP_STORED (no compression. available everywhere) +# * ZIP_DEFLATED (uses zlib, the default) +# These values may be set per host via the ansible_module_compression inventory +# variable +#module_compression = 'ZIP_DEFLATED' + +# This controls the cutoff point (in bytes) on --diff for files +# set to 0 for unlimited (RAM may suffer!). +#max_diff_size = 1048576 + +# This controls how ansible handles multiple --tags and --skip-tags arguments +# on the CLI. If this is True then multiple arguments are merged together. If +# it is False, then the last specified argument is used and the others are ignored. +#merge_multiple_cli_flags = False + +# Controls showing custom stats at the end, off by default +#show_custom_stats = True + +# Controlls which files to ignore when using a directory as inventory with +# possibly multiple sources (both static and dynamic) +#inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo + +# This family of modules use an alternative execution path optimized for network appliances +# only update this setting if you know how this works, otherwise it can break module execution +#network_group_modules=['eos', 'nxos', 'ios', 'iosxr', 'junos', 'vyos'] + +# This keeps facts from polluting the main namespace as variables. +# Setting to True keeps them under the ansible_facts namespace, the default is False +#restrict_facts_namespace: True + +[privilege_escalation] +#become=True +#become_method=sudo +#become_user=root +#become_ask_pass=False + +[paramiko_connection] + +# uncomment this line to cause the paramiko connection plugin to not record new host +# keys encountered. Increases performance on new host additions. Setting works independently of the +# host key checking setting above. +#record_host_keys=False + +# by default, Ansible requests a pseudo-terminal for commands executed under sudo. Uncomment this +# line to disable this behaviour. +#pty=False + +# paramiko will default to looking for SSH keys initially when trying to +# authenticate to remote devices. This is a problem for some network devices +# that close the connection after a key failure. Uncomment this line to +# disable the Paramiko look for keys function +#look_for_keys = False + +# When using persistent connections with Paramiko, the connection runs in a +# background process. If the host doesn't already have a valid SSH key, by +# default Ansible will prompt to add the host key. This will cause connections +# running in background processes to fail. Uncomment this line to have +# Paramiko automatically add host keys. +#host_key_auto_add = True + +[ssh_connection] + +# ssh arguments to use +# Leaving off ControlPersist will result in poor performance, so use +# paramiko on older platforms rather than removing it, -C controls compression use +#ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s +ssh_args = -o StrictHostKeyChecking=no + +# The base directory for the ControlPath sockets. +# This is the "%(directory)s" in the control_path option +# +# Example: +# control_path_dir = /tmp/.ansible/cp +#control_path_dir = ~/.ansible/cp + +# The path to use for the ControlPath sockets. This defaults to a hashed string of the hostname, +# port and username (empty string in the config). The hash mitigates a common problem users +# found with long hostames and the conventional %(directory)s/ansible-ssh-%%h-%%p-%%r format. +# In those cases, a "too long for Unix domain socket" ssh error would occur. +# +# Example: +# control_path = %(directory)s/%%h-%%r +#control_path = + +# Enabling pipelining reduces the number of SSH operations required to +# execute a module on the remote server. This can result in a significant +# performance improvement when enabled, however when using "sudo:" you must +# first disable 'requiretty' in /etc/sudoers +# +# By default, this option is disabled to preserve compatibility with +# sudoers configurations that have requiretty (the default on many distros). +# +#pipelining = False + +# Control the mechanism for transferring files (old) +# * smart = try sftp and then try scp [default] +# * True = use scp only +# * False = use sftp only +#scp_if_ssh = smart + +# Control the mechanism for transferring files (new) +# If set, this will override the scp_if_ssh option +# * sftp = use sftp to transfer files +# * scp = use scp to transfer files +# * piped = use 'dd' over SSH to transfer files +# * smart = try sftp, scp, and piped, in that order [default] +#transfer_method = smart + +# if False, sftp will not use batch mode to transfer files. This may cause some +# types of file transfer failures impossible to catch however, and should +# only be disabled if your sftp version has problems with batch mode +#sftp_batch_mode = False + +[persistent_connection] + +# Configures the persistent connection timeout value in seconds. This value is +# how long the persistent connection will remain idle before it is destroyed. +# If the connection doesn't receive a request before the timeout value +# expires, the connection is shutdown. The default value is 30 seconds. +#connect_timeout = 30 + +# Configures the persistent connection retries. This value configures the +# number of attempts the ansible-connection will make when trying to connect +# to the local domain socket. The default value is 30. +#connect_retries = 30 + +# Configures the amount of time in seconds to wait between connection attempts +# to the local unix domain socket. This value works in conjunction with the +# connect_retries value to define how long to try to connect to the local +# domain socket when setting up a persistent connection. The default value is +# 1 second. +#connect_interval = 1 + +[accelerate] +#accelerate_port = 5099 +#accelerate_timeout = 30 +#accelerate_connect_timeout = 5.0 + +# The daemon timeout is measured in minutes. This time is measured +# from the last activity to the accelerate daemon. +#accelerate_daemon_timeout = 30 + +# If set to yes, accelerate_multi_key will allow multiple +# private keys to be uploaded to it, though each user must +# have access to the system via SSH to add a new key. The default +# is "no". +#accelerate_multi_key = yes + +[selinux] +# file systems that require special treatment when dealing with security context +# the default behaviour that copies the existing context or uses the user default +# needs to be changed to use the file system dependent context. +#special_context_filesystems=nfs,vboxsf,fuse,ramfs,9p + +# Set this to yes to allow libvirt_lxc connections to work without SELinux. +#libvirt_lxc_noseclabel = yes + +[colors] +#highlight = white +#verbose = blue +#warn = bright purple +#error = red +#debug = dark gray +#deprecate = purple +#skip = cyan +#unreachable = red +#ok = green +#changed = yellow +#diff_add = green +#diff_remove = red +#diff_lines = cyan + + +[diff] +# Always print diff when running ( same as always running with -D/--diff ) +# always = no + +# Set how many context lines to show in diff +# context = 3 diff --git a/resources/ansible_roles/qtip-generator/files/storage/fixtures/case.json b/resources/ansible_roles/qtip-generator/files/storage/fixtures/case.json new file mode 100644 index 00000000..d58f51bc --- /dev/null +++ b/resources/ansible_roles/qtip-generator/files/storage/fixtures/case.json @@ -0,0 +1,14 @@ +{ + "run": "", + "name": "storage", + "ci_loop": "", + "tags": "", + "url": "", + "domains": "", + "dependencies": "", + "version": "", + "criteria": "", + "tier": "", + "blocking": "", + "description": "" +}
\ No newline at end of file diff --git a/resources/ansible_roles/qtip-generator/files/storage/fixtures/pod.json b/resources/ansible_roles/qtip-generator/files/storage/fixtures/pod.json new file mode 100644 index 00000000..654b5828 --- /dev/null +++ b/resources/ansible_roles/qtip-generator/files/storage/fixtures/pod.json @@ -0,0 +1,6 @@ +{ + "details": "", + "role": "", + "name": "qtip-pod", + "mode": "" +}
\ No newline at end of file diff --git a/resources/ansible_roles/qtip-generator/files/storage/fixtures/project.json b/resources/ansible_roles/qtip-generator/files/storage/fixtures/project.json new file mode 100644 index 00000000..ecd03e83 --- /dev/null +++ b/resources/ansible_roles/qtip-generator/files/storage/fixtures/project.json @@ -0,0 +1,4 @@ +{ + "name": "qtip", + "description": "Platform Performance Benchmarking" +}
\ No newline at end of file diff --git a/resources/ansible_roles/qtip-generator/files/storage/group_vars/all.yml b/resources/ansible_roles/qtip-generator/files/storage/group_vars/all.yml new file mode 100644 index 00000000..8b90b498 --- /dev/null +++ b/resources/ansible_roles/qtip-generator/files/storage/group_vars/all.yml @@ -0,0 +1,22 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +qtip_resources: "{{ qtip_package }}/resources" +qtip_results: results +qtip_fixtures: fixtures +qtip_dump: dump +qtip_cache: "{{ qtip_cache }}" +qtip_workdir: "{% raw %}{{ ansible_env.HOME }}{% endraw %}/qtip-workdir-{{ lookup('pipe', 'date +%Y%m%d-%H%M') }}" + +installer_type: "{{ installer_type }}" +installer_host: "{{ installer_host }}" +installer_group: + manual: dummy-group + fuel: fuel-masters + apex: apex-underclouds diff --git a/resources/ansible_roles/qtip-generator/files/storage/host_vars/localhost.yml b/resources/ansible_roles/qtip-generator/files/storage/host_vars/localhost.yml new file mode 100644 index 00000000..228c00e6 --- /dev/null +++ b/resources/ansible_roles/qtip-generator/files/storage/host_vars/localhost.yml @@ -0,0 +1,16 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# set `testapi_url` to enable reporting to testapi +# testapi_url: http://localhost:8000/api/v1 +project_name: qtip +case_name: "{{ case_name|default('storage') }}" +pod_name: "{{ pod_name|default('qtip-pod') }}" +scenario: "{{ scenario|default('generic') }}" +version: "{{ version|default('master') }}" diff --git a/resources/ansible_roles/qtip-generator/files/storage/hosts b/resources/ansible_roles/qtip-generator/files/storage/hosts new file mode 100644 index 00000000..0347e450 --- /dev/null +++ b/resources/ansible_roles/qtip-generator/files/storage/hosts @@ -0,0 +1,13 @@ +localhost ansible_connection=local + +{% if installer_type != 'manual' %} +[{{ installer_group[installer_type] }}] +{{ installer_host }} +{% else %} +[SUT] +# Add hosts in system under test, example:: +# +# host-under-test ansible_host=192.168.10.1 ansible_port=5022 ansile_user=root +# +# See http://docs.ansible.com/ansible/intro_inventory.html for details +{% endif %} diff --git a/resources/ansible_roles/qtip-generator/files/storage/run.yml b/resources/ansible_roles/qtip-generator/files/storage/run.yml new file mode 100644 index 00000000..5fdf72f6 --- /dev/null +++ b/resources/ansible_roles/qtip-generator/files/storage/run.yml @@ -0,0 +1,21 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- + +- hosts: localhost + + roles: + # prepare local environment + - { role: qtip, tasks: setup-local, tags: [setup] } + - storperf + +- hosts: SUT + roles: + - ceph-info diff --git a/resources/ansible_roles/qtip-generator/files/storage/setup.yml b/resources/ansible_roles/qtip-generator/files/storage/setup.yml new file mode 100644 index 00000000..f5920c1f --- /dev/null +++ b/resources/ansible_roles/qtip-generator/files/storage/setup.yml @@ -0,0 +1,22 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Prepare connection to SUT (System Under Test) + +--- +{% if installer_type == 'manual' %} +- hosts: localhost + gather_facts: no + tasks: + - debug: "msg='please add hosts to SUT group in `hosts` file manually.'" +{% else %} +- hosts: [{{ installer_group[installer_type] }}] + roles: + - { role: qtip, tasks: generate-inventory } +{% endif %} diff --git a/resources/ansible_roles/qtip-generator/files/storage/teardown.yml b/resources/ansible_roles/qtip-generator/files/storage/teardown.yml new file mode 100644 index 00000000..dc659930 --- /dev/null +++ b/resources/ansible_roles/qtip-generator/files/storage/teardown.yml @@ -0,0 +1,14 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: SUT + + roles: + # teardown environment + - { role: qtip, tasks: teardown } diff --git a/resources/ansible_roles/qtip-generator/files/storage/templates/hosts b/resources/ansible_roles/qtip-generator/files/storage/templates/hosts new file mode 100644 index 00000000..49920006 --- /dev/null +++ b/resources/ansible_roles/qtip-generator/files/storage/templates/hosts @@ -0,0 +1,25 @@ +{% raw %} +localhost ansible_connection=local + +[{{ installer_group[installer_type] }}] +{{ installer_host }} + +[SUT:children] # system under test +compute + +[node-groups:children] +{% for group in hosts|sort %} +{{ group }} +{% endfor %} + +[node-groups:vars] +ansible_ssh_common_args=-F ./ssh.cfg + +{% for group in hosts|sort %} +[{{ group }}] +{% for host in hosts[group]|sort %} +{{ host }} +{% endfor %} + +{% endfor %} +{% endraw %} diff --git a/resources/ansible_roles/qtip-generator/files/storage/templates/ssh.cfg b/resources/ansible_roles/qtip-generator/files/storage/templates/ssh.cfg new file mode 100644 index 00000000..1b7c0183 --- /dev/null +++ b/resources/ansible_roles/qtip-generator/files/storage/templates/ssh.cfg @@ -0,0 +1,16 @@ +{% raw %} +# Connect to target node through jump host +# +# OpenSSH 7.3: +# ProxyJump jumphost +# before OpenSSH 7.3 +# ProxyCommand ssh -o 'ForwardAgent yes' jumphost 'ssh-add && nc %h %p' + +{% for (name, host) in hosts_meta.items() %} +Host {{ name }} + HostName {{ host.ansible_ssh_host }} + User {{ host.ansible_user }} + ProxyCommand ssh -o 'ForwardAgent yes' {{ installer_host }} 'ssh-add && nc %h %p' + +{% endfor %} +{% endraw %}
\ No newline at end of file diff --git a/tests/ci/storperf/default_job.json b/resources/ansible_roles/storperf/files/default_job.json index 4e5e2dfd..4e5e2dfd 100644 --- a/tests/ci/storperf/default_job.json +++ b/resources/ansible_roles/storperf/files/default_job.json diff --git a/tests/ci/storperf/default_stack.json b/resources/ansible_roles/storperf/files/default_stack.json index 79d8fc46..79d8fc46 100644 --- a/tests/ci/storperf/default_stack.json +++ b/resources/ansible_roles/storperf/files/default_stack.json diff --git a/tests/ci/storperf/prepare.sh b/resources/ansible_roles/storperf/files/prepare.sh index fe0708ad..fe0708ad 100755 --- a/tests/ci/storperf/prepare.sh +++ b/resources/ansible_roles/storperf/files/prepare.sh diff --git a/tests/ci/storperf/start_job.sh b/resources/ansible_roles/storperf/files/start_job.sh index 15785036..4455609e 100755 --- a/tests/ci/storperf/start_job.sh +++ b/resources/ansible_roles/storperf/files/start_job.sh @@ -7,6 +7,11 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +set -o errexit +set -o pipefail +set -o nounset +set -x + usage(){ echo "usage: $0 -s <stack_json_path> -j <job_json_path>" >&2 } @@ -32,13 +37,10 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" stack_json=${stack_json:-"$script_dir/default_stack.json"} job_json=${job_json:-"$script_dir/default_job.json"} -if [[ -z $WORKSPACE ]];then - WORKSPACE=`pwd` -fi +WORKSPACE=${WORKSPACE:-$(pwd)} nova_vm_mapping() { - rm $WORKSPACE/nova_vm.json openstack server list --name storperf-agent -c ID -c Host --long -f json > $WORKSPACE/nova_vm.json echo ========================================================================== @@ -47,6 +49,14 @@ nova_vm_mapping() echo ========================================================================== } +storperf_api="http://storperf-master:5000/api/v1.0" + +echo "QTIP: Waiting for storperf api ready" +while [ $(curl -s -o /dev/null -I -w "%{http_code}" -X GET ${storperf_api}/configurations) != "200" ] +do + sleep 1 +done + echo ========================================================================== echo "Start to create storperf stack" cat ${stack_json} 1>&2 @@ -54,7 +64,7 @@ echo ========================================================================== curl -X POST --header 'Content-Type: application/json' \ --header 'Accept: application/json' -d @${stack_json} \ - 'http://storperf-httpfrontend:5000/api/v1.0/configurations' + ${storperf_api}/configurations nova_vm_mapping @@ -66,15 +76,16 @@ echo ========================================================================== JOB=$(curl -s -X POST --header 'Content-Type: application/json' \ --header 'Accept: application/json' \ - -d @${job_json} 'http://storperf-httpfrontend:5000/api/v1.0/jobs' | \ + -d @${job_json} ${storperf_api}/jobs | \ awk '/job_id/ {print $2}' | sed 's/"//g') echo "JOB ID: $JOB" if [[ -z "$JOB" ]]; then echo "Oops, JOB ID is empty!" + exit 1 else echo "checking job status..." - curl -s -X GET "http://storperf-httpfrontend:5000/api/v1.0/jobs?id=$JOB&type=status" \ + curl -s -X GET "${storperf_api}/jobs?id=$JOB&type=status" \ -o $WORKSPACE/status.json cat $WORKSPACE/status.json @@ -85,7 +96,7 @@ else do sleep 180 mv $WORKSPACE/status.json $WORKSPACE/old-status.json - curl -s -X GET "http://storperf-httpfrontend:5000/api/v1.0/jobs?id=$JOB&type=status" \ + curl -s -X GET "${storperf_api}/jobs?id=$JOB&type=status" \ -o $WORKSPACE/status.json JOB_STATUS=`cat $WORKSPACE/status.json | awk '/Status/ {print $2}' | cut -d\" -f2` diff $WORKSPACE/status.json $WORKSPACE/old-status.json >/dev/null @@ -101,10 +112,10 @@ else echo ========================================================================== echo Final report echo ========================================================================== - curl -s -X GET "http://storperf-httpfrontend:5000/api/v1.0/jobs?id=$JOB&type=metadata" \ + curl -s -X GET "${storperf_api}/jobs?id=$JOB&type=metadata" \ -o $WORKSPACE/report.json cat $WORKSPACE/report.json fi echo "Deleting stack for cleanup" -curl -s -X DELETE --header 'Accept: application/json' 'http://storperf-httpfrontend:5000/api/v1.0/configurations'
\ No newline at end of file +curl -s -X DELETE --header 'Accept: application/json' ${storperf_api}/configurations diff --git a/tests/ci/storperf/storperf_requirements.txt b/resources/ansible_roles/storperf/files/storperf_requirements.txt index 4a279ae5..4a279ae5 100644 --- a/tests/ci/storperf/storperf_requirements.txt +++ b/resources/ansible_roles/storperf/files/storperf_requirements.txt diff --git a/resources/ansible_roles/storperf/meta/main.yml b/resources/ansible_roles/storperf/meta/main.yml new file mode 100644 index 00000000..ecdf8697 --- /dev/null +++ b/resources/ansible_roles/storperf/meta/main.yml @@ -0,0 +1,14 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- + +allow_duplicates: yes +dependencies: +- { role: qtip-common, basename: storperf } diff --git a/resources/ansible_roles/storperf/tasks/main.yml b/resources/ansible_roles/storperf/tasks/main.yml new file mode 100644 index 00000000..b326dc47 --- /dev/null +++ b/resources/ansible_roles/storperf/tasks/main.yml @@ -0,0 +1,15 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +- name: preparing stack for storperf testing + local_action: "shell {{ role_path }}/files/prepare.sh chdir={{ output }}" + +- name: running storperf testing + local_action: "shell {{ role_path }}/files/start_job.sh chdir={{ output }}" diff --git a/resources/ansible_roles/storperf/vars/main.yml b/resources/ansible_roles/storperf/vars/main.yml new file mode 100644 index 00000000..ab934905 --- /dev/null +++ b/resources/ansible_roles/storperf/vars/main.yml @@ -0,0 +1,11 @@ +############################################################################# +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################# + +--- +workdir: "{{ qtip_workdir }}/storperf" diff --git a/tests/ci/experimental.sh b/tests/ci/experimental.sh index 379e8602..3dd15d56 100644 --- a/tests/ci/experimental.sh +++ b/tests/ci/experimental.sh @@ -1,3 +1,36 @@ #!/usr/bin/env bash +############################################################################## +# Copyright (c) 2017 ZTE and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## -source tests/ci/verify_storage.sh +set -o errexit +set -o pipefail +set -o nounset +set -x + +export DEPLOY_SCENARIO='generic' +export DOCKER_TAG='latest' +export CI_DEBUG='false' +export TEST_SUITE='storage' +export TESTAPI_URL='' + +export WORKSPACE=${WORKSPACE:-$(pwd)} + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +qtip_repo='/home/opnfv/repos/qtip' + +source $script_dir/launch_containers_by_testsuite.sh + +container_id=$(docker ps | grep "opnfv/qtip:${DOCKER_TAG}" | awk '{print $1}' | head -1) + +echo "QTIP: Copying current submit patch to the container ${container_id}" +cd $WORKSPACE +docker cp . ${container_id}:${qtip_repo} +docker exec ${container_id} bash -c "cd ${qtip_repo} && pip install -U -e ." +docker exec -t ${container_id} bash -x ${qtip_repo}/tests/ci/run_${TEST_SUITE}_qpi.sh +echo "QTIP: Verify ${TEST_SUITE} done!" +exit 0 diff --git a/tests/ci/launch_containers_by_testsuite.sh b/tests/ci/launch_containers_by_testsuite.sh index b55ba09c..08b80028 100755 --- a/tests/ci/launch_containers_by_testsuite.sh +++ b/tests/ci/launch_containers_by_testsuite.sh @@ -7,6 +7,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +export ENV_FILE="${WORKSPACE}/env_file" + if [[ -e $ENV_FILE ]];then rm $ENV_FILE fi @@ -51,8 +53,8 @@ if [[ "$TEST_SUITE" == 'compute' ]];then elif [[ "$TEST_SUITE" == 'storage' ]];then script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - git clone --depth 1 https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng - $WORKSPACE/releng/utils/fetch_os_creds.sh -i $INSTALLER_TYPE -a $INSTALLER_IP -d $WORKSPACE/openrc + curl https://git.opnfv.org/releng/plain/utils/fetch_os_creds.sh | bash -s -- \ + -i $INSTALLER_TYPE -a $INSTALLER_IP -d $WORKSPACE/openrc grep "export" $WORKSPACE/openrc | sed "s/export //" >> $ENV_FILE echo "DOCKER_TAG=$DOCKER_TAG" >> $ENV_FILE diff --git a/tests/ci/run_storage_qpi.sh b/tests/ci/run_storage_qpi.sh index d4859d56..955d6751 100755 --- a/tests/ci/run_storage_qpi.sh +++ b/tests/ci/run_storage_qpi.sh @@ -7,8 +7,20 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -#TODO: These will be replaced by qtip cli script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -$script_dir/storperf/prepare.sh -$script_dir/storperf/start_job.sh
\ No newline at end of file +cd /home/opnfv + +qtip create --project-template storage --pod-name ${NODE_NAME} --installer-type ${INSTALLER_TYPE} \ +--installer-host ${INSTALLER_IP} --scenario ${SCENARIO} workspace + +cd /home/opnfv/workspace/ + +qtip setup +eval `ssh-agent` +if [[ -z $testapi_url ]];then + qtip run -vv +else + qtip run --extra-vars "testapi_url=${TESTAPI_URL}" +fi +qtip teardown diff --git a/tests/ci/storperf/containers.sh b/tests/ci/storperf/containers.sh index 26d582bf..62f8d4d5 100755 --- a/tests/ci/storperf/containers.sh +++ b/tests/ci/storperf/containers.sh @@ -19,11 +19,4 @@ launch_containers() echo "Launch new qtip and storperf containers" docker-compose -f qtip-storperf-docker-compose.yaml pull docker-compose -f qtip-storperf-docker-compose.yaml up -d - - echo "QTIP: Waiting for StorPerf to become active" - - while [ $(curl -s -o /dev/null -I -w "%{http_code}" -X GET http://127.0.0.1:5000/api/v1.0/configurations) != "200" ] - do - sleep 1 - done }
\ No newline at end of file diff --git a/tests/ci/storperf/qtip-storperf-docker-compose.yaml b/tests/ci/storperf/qtip-storperf-docker-compose.yaml index fc837139..9396a414 100644 --- a/tests/ci/storperf/qtip-storperf-docker-compose.yaml +++ b/tests/ci/storperf/qtip-storperf-docker-compose.yaml @@ -11,35 +11,15 @@ version: '2' services: storperf-master: - container_name: "storperf-master" + container_name: storperf-master image: opnfv/storperf-master:${DOCKER_TAG} ports: - - "8000:8000" + - 5000:5000 env_file: ${ENV_FILE} - storperf-reporting: - container_name: "storperf-reporting" - image: opnfv/storperf-reporting:${DOCKER_TAG} - ports: - - "5080:5000" - - storperf-swaggerui: - container_name: "storperf-swaggerui" - image: "schickling/swagger-ui" - - storperf-httpfrontend: - container_name: "storperf-httpfrontend" - image: opnfv/storperf-httpfrontend:${DOCKER_TAG} - ports: - - "5000:5000" - links: - - storperf-master - - storperf-reporting - - storperf-swaggerui - qtip: container_name: qtip image: opnfv/qtip:${DOCKER_TAG} env_file: ${ENV_FILE} links: - - storperf-httpfrontend
\ No newline at end of file + - storperf-master diff --git a/tests/ci/verify_storage.sh b/tests/ci/verify_storage.sh deleted file mode 100755 index 32e94f9e..00000000 --- a/tests/ci/verify_storage.sh +++ /dev/null @@ -1,35 +0,0 @@ -#! /bin/bash -x -############################################################################## -# Copyright (c) 2017 ZTE and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -TEST_SUITE=storage -QTIP_REPO=/home/opnfv/repos/qtip - -export DOCKER_TAG=${DOCKER_TAG:-latest} -export ENV_FILE=$WORKSPACE/env_file - -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -source $script_dir/launch_containers_by_testsuite.sh - -container_id=$(docker ps | grep "opnfv/qtip:${DOCKER_TAG}" | awk '{print $1}' | head -1) - -if [[ -z "$container_id" ]]; then - echo "QTIP: The container opnfv/qtip has not been properly started. Exiting..." - exit 1 -else - echo "QTIP: Copying current submit patch to the container ${container_id}" - cd $WORKSPACE - docker cp . ${container_id}:${QTIP_REPO} - docker exec ${container_id} bash -c "cd ${QTIP_REPO} && pip install -U -e ." - docker exec -t ${container_id} bash -c "bash ${QTIP_REPO}/tests/ci/run_${TEST_SUITE}_qpi.sh" - echo "QTIP: Verify ${TEST_SUITE} done!" - exit 0 -fi - - |