aboutsummaryrefslogtreecommitdiffstats
path: root/tests/integration
diff options
context:
space:
mode:
Diffstat (limited to 'tests/integration')
-rw-r--r--tests/integration/hosts6
-rw-r--r--tests/integration/hosts.j221
-rw-r--r--tests/integration/hosts.sample83
-rw-r--r--tests/integration/reports/qpi-report46
-rw-r--r--tests/integration/reports/system-info (renamed from tests/integration/reports/inxi-system-info)0
-rw-r--r--tests/integration/run.yaml70
-rw-r--r--tests/integration/setup.yaml (renamed from tests/integration/compute.yaml)23
-rw-r--r--tests/integration/specs/compute.yaml36
-rw-r--r--tests/integration/ssh.cfg.j214
-rw-r--r--tests/integration/ssh.cfg.sample56
-rw-r--r--tests/integration/tasks/inxi.yaml15
-rw-r--r--tests/integration/tasks/openssl.yaml53
-rw-r--r--tests/integration/templates/inxi-system-info.j216
-rw-r--r--tests/integration/templates/qpi-report.j218
-rw-r--r--tests/integration/templates/system-info.j216
15 files changed, 433 insertions, 40 deletions
diff --git a/tests/integration/hosts b/tests/integration/hosts
deleted file mode 100644
index 9b91eea6..00000000
--- a/tests/integration/hosts
+++ /dev/null
@@ -1,6 +0,0 @@
-[fuel-master]
-fuel-master
-
-[local]
-localhost ansible_connection=local
-
diff --git a/tests/integration/hosts.j2 b/tests/integration/hosts.j2
new file mode 100644
index 00000000..868a4e57
--- /dev/null
+++ b/tests/integration/hosts.j2
@@ -0,0 +1,21 @@
+[fuel-master]
+fuel-master
+
+[local]
+localhost ansible_connection=local
+
+[fuel-groups:children]
+{% for group in hosts|sort %}
+{{ group }}
+{% endfor %}
+
+[fuel-groups:vars]
+ansible_ssh_common_args=-F ./ssh.cfg
+
+{% for group in hosts|sort %}
+[{{ group }}]
+{% for host in hosts[group]|sort %}
+{{ host }}
+{% endfor %}
+
+{% endfor %}
diff --git a/tests/integration/hosts.sample b/tests/integration/hosts.sample
new file mode 100644
index 00000000..e37d38ec
--- /dev/null
+++ b/tests/integration/hosts.sample
@@ -0,0 +1,83 @@
+[fuel-master]
+fuel-master
+
+[local]
+localhost ansible_connection=local
+
+[fuel-groups:children]
+ceph-osd
+cluster-1
+compute
+controller
+hw-zte-servers
+mongo
+node-1
+node-2
+node-3
+node-4
+node-5
+node-6
+node-7
+
+[fuel-groups:vars]
+ansible_ssh_common_args=-F ./ssh.cfg
+
+[ceph-osd]
+node-2
+node-4
+node-6
+node-7
+
+[cluster-1]
+node-1
+node-2
+node-3
+node-4
+node-5
+node-6
+node-7
+
+[compute]
+node-2
+node-4
+node-6
+node-7
+
+[controller]
+node-1
+node-3
+node-5
+
+[hw-zte-servers]
+node-1
+node-2
+node-3
+node-4
+node-5
+node-6
+node-7
+
+[mongo]
+node-1
+
+[node-1]
+node-1
+
+[node-2]
+node-2
+
+[node-3]
+node-3
+
+[node-4]
+node-4
+
+[node-5]
+node-5
+
+[node-6]
+node-6
+
+[node-7]
+node-7
+
diff --git a/tests/integration/reports/qpi-report b/tests/integration/reports/qpi-report
new file mode 100644
index 00000000..cb9e9308
--- /dev/null
+++ b/tests/integration/reports/qpi-report
@@ -0,0 +1,46 @@
+Sample QPI Report
+
+Host: node-26
+QPI: 1.41147857985
+Spec: compute
+
+- SSL: 1.41147857985
+ - ssl_rsa: 1.41147857985
+ - rsa_sign_512: 1.25261808935
+ - rsa_verify_512: 1.33973907536
+ - rsa_sign_1024: 1.31933223495
+ - rsa_verify_1024: 1.46972456907
+ - rsa_sign_2048: 1.8615470852
+ - rsa_verify_2048: 1.3683903146
+ - rsa_sign_4096: 1.31537708129
+ - rsa_verify_4096: 1.36510018898
+
+Host: node-28
+QPI: 1.28082308651
+Spec: compute
+
+- SSL: 1.28082308651
+ - ssl_rsa: 1.28082308651
+ - rsa_sign_512: 1.13628081136
+ - rsa_verify_512: 1.24882238433
+ - rsa_sign_1024: 1.16100601465
+ - rsa_verify_1024: 1.33382620817
+ - rsa_sign_2048: 1.72057174888
+ - rsa_verify_2048: 1.23917640038
+ - rsa_sign_4096: 1.16846229187
+ - rsa_verify_4096: 1.2384388324
+
+Host: node-27
+QPI: 1.41542492777
+Spec: compute
+
+- SSL: 1.41542492777
+ - ssl_rsa: 1.41542492777
+ - rsa_sign_512: 1.25857845591
+ - rsa_verify_512: 1.34193319426
+ - rsa_sign_1024: 1.32097981222
+ - rsa_verify_1024: 1.4807103336
+ - rsa_sign_2048: 1.86378923767
+ - rsa_verify_2048: 1.36600306932
+ - rsa_sign_4096: 1.31635651322
+ - rsa_verify_4096: 1.37504880601
diff --git a/tests/integration/reports/inxi-system-info b/tests/integration/reports/system-info
index 371243e2..371243e2 100644
--- a/tests/integration/reports/inxi-system-info
+++ b/tests/integration/reports/system-info
diff --git a/tests/integration/run.yaml b/tests/integration/run.yaml
new file mode 100644
index 00000000..7eb141d7
--- /dev/null
+++ b/tests/integration/run.yaml
@@ -0,0 +1,70 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+---
+# Execute compute benchmark plan and collect data
+# - system information
+# - test condition
+# - performance metrics
+- hosts: compute
+ tasks:
+ - name: check ssh connection
+ ping:
+ # collect system information
+ - name: collect system information
+ include: tasks/inxi.yaml
+ - name: ssl metrics
+ include: tasks/openssl.yaml
+
+- hosts: compute
+ tasks:
+ - name: calculate QPI of compute
+ calculate:
+ metrics:
+ ssl_rsa: "{{ openssl_rsa_metrics }}"
+ spec: # TODO(yujunz) load spec from file
+ name: compute
+ description: QTIP Performance Index of compute
+ formula: weighted arithmetic mean
+ sections: # split based on different application
+ - name: SSL
+ description: cryptography and SSL/TLS performance
+ formula: geometric mean
+ metrics:
+ - name: ssl_rsa
+ formual: geometric mean
+ workloads:
+ - name: rsa_sign_512
+ description: RSA signature 512 bits
+ baseline: 14982.3
+ - name: rsa_verify_512
+ baseline: 180619.2
+ - name: rsa_sign_1024
+ baseline: 5037.7
+ - name: rsa_verify_1024
+ baseline: 67359.9
+ - name: rsa_sign_2048
+ baseline: 713.6
+ - name: rsa_verify_2048
+ baseline: 23458.0
+ - name: rsa_sign_4096
+ baseline: 102.1
+ - name: rsa_verify_4096
+ baseline: 6402.9
+ register: qpi_result
+ delegate_to: localhost
+
+# Generate and publish report
+- hosts: local
+ tasks:
+ - name: create system information report
+ template: src=templates/system-info.j2 dest=reports/system-info
+ - name: create qpi report
+ template: src=templates/qpi-report.j2 dest=reports/qpi-report
+ # TODO(yujunz) push test result to testapi
diff --git a/tests/integration/compute.yaml b/tests/integration/setup.yaml
index 4cb71e9f..784d6cc5 100644
--- a/tests/integration/compute.yaml
+++ b/tests/integration/setup.yaml
@@ -7,26 +7,15 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
----
+# Prepare connection to SUT (System Under Test)
- hosts: fuel-master
gather_facts: no
tasks:
- name: collect facts of fuel hosts
fuel:
- - name: add compute node to ansible inventory
- add_host:
- name: "{{ hosts_meta[item]['ip'] }}"
- groups: compute
- ansible_user: root
- ansible_ssh_common_args: '-o StrictHostKeyChecking=No -o ProxyJump=fuel-master'
- with_items: "{{ hosts.compute }}"
-- hosts: compute
- tasks:
- - name: check ssh connection
- ping:
- - include: tasks/inxi.yaml
-- hosts: local
- tasks:
- - name: create system information report
- local_action: template src=templates/inxi-system-info.j2 dest=reports/inxi-system-info
+ - name: update inventory file
+ template: src=./hosts.j2 dest=./hosts
+ delegate_to: localhost
+ - name: update ssh.cfg file
+ template: src=./ssh.cfg.j2 dest=./ssh.cfg
delegate_to: localhost
diff --git a/tests/integration/specs/compute.yaml b/tests/integration/specs/compute.yaml
new file mode 100644
index 00000000..0266ac8d
--- /dev/null
+++ b/tests/integration/specs/compute.yaml
@@ -0,0 +1,36 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+name: compute
+description: QTIP Performance Index of compute
+formula: weighted arithmetic mean
+sections: # split based on different application
+- name: SSL
+ description: cryptography and SSL/TLS performance
+ formula: geometric mean
+ metrics:
+ - name: rsa
+ formual: geometric mean
+ workloads:
+ - name: rsa_sign_512
+ description: RSA signature 512 bits
+ baseline: 14982.3
+ - name: rsa_verify_512
+ baseline: 180619.2
+ - name: rsa_sign_1024
+ baseline: 5037.7
+ - name: rsa_verify_1024
+ baseline: 67359.9
+ - name: rsa_sign_2048
+ baseline: 713.6
+ - name: rsa_verify_2048
+ baseline: 23458.0
+ - name: rsa_sign_4096
+ baeline: 102.1
+ - name: rsa_verify_4096
+ baseline: 6402.9
diff --git a/tests/integration/ssh.cfg.j2 b/tests/integration/ssh.cfg.j2
new file mode 100644
index 00000000..2fe049e2
--- /dev/null
+++ b/tests/integration/ssh.cfg.j2
@@ -0,0 +1,14 @@
+# Connect to target node through jump host
+#
+# OpenSSH 7.3:
+# ProxyJump jumphost
+# before OpenSSH 7.3
+# ProxyCommand ssh -o 'ForwardAgent yes' jumphost 'ssh-add && nc %h %p'
+
+{% for (name, host) in hosts_meta.items() %}
+Host {{ name }}
+ HostName {{ host.ansible_ssh_host }}
+ User root
+ ProxyCommand ssh -o 'ForwardAgent yes' fuel-master 'ssh-add && nc %h %p'
+
+{% endfor %}
diff --git a/tests/integration/ssh.cfg.sample b/tests/integration/ssh.cfg.sample
new file mode 100644
index 00000000..efa45ab6
--- /dev/null
+++ b/tests/integration/ssh.cfg.sample
@@ -0,0 +1,56 @@
+Host node-5
+ HostName 10.20.5.12
+ User root
+ # Use `ProxyCommand` for OpenSSH before 7.3
+ ProxyCommand ssh -o 'ForwardAgent yes' fuel-master 'ssh-add && nc %h %p'
+ # `ProxyJump` is available since OpenSSH 7.3
+ # ProxyJump fuel-master
+
+Host node-4
+ HostName 10.20.5.14
+ User root
+ # Use `ProxyCommand` for OpenSSH before 7.3
+ ProxyCommand ssh -o 'ForwardAgent yes' fuel-master 'ssh-add && nc %h %p'
+ # `ProxyJump` is available since OpenSSH 7.3
+ # ProxyJump fuel-master
+
+Host node-7
+ HostName 10.20.5.15
+ User root
+ # Use `ProxyCommand` for OpenSSH before 7.3
+ ProxyCommand ssh -o 'ForwardAgent yes' fuel-master 'ssh-add && nc %h %p'
+ # `ProxyJump` is available since OpenSSH 7.3
+ # ProxyJump fuel-master
+
+Host node-6
+ HostName 10.20.5.16
+ User root
+ # Use `ProxyCommand` for OpenSSH before 7.3
+ ProxyCommand ssh -o 'ForwardAgent yes' fuel-master 'ssh-add && nc %h %p'
+ # `ProxyJump` is available since OpenSSH 7.3
+ # ProxyJump fuel-master
+
+Host node-1
+ HostName 10.20.5.10
+ User root
+ # Use `ProxyCommand` for OpenSSH before 7.3
+ ProxyCommand ssh -o 'ForwardAgent yes' fuel-master 'ssh-add && nc %h %p'
+ # `ProxyJump` is available since OpenSSH 7.3
+ # ProxyJump fuel-master
+
+Host node-3
+ HostName 10.20.5.11
+ User root
+ # Use `ProxyCommand` for OpenSSH before 7.3
+ ProxyCommand ssh -o 'ForwardAgent yes' fuel-master 'ssh-add && nc %h %p'
+ # `ProxyJump` is available since OpenSSH 7.3
+ # ProxyJump fuel-master
+
+Host node-2
+ HostName 10.20.5.13
+ User root
+ # Use `ProxyCommand` for OpenSSH before 7.3
+ ProxyCommand ssh -o 'ForwardAgent yes' fuel-master 'ssh-add && nc %h %p'
+ # `ProxyJump` is available since OpenSSH 7.3
+ # ProxyJump fuel-master
+
diff --git a/tests/integration/tasks/inxi.yaml b/tests/integration/tasks/inxi.yaml
index f8951dc1..6c2db6a4 100644
--- a/tests/integration/tasks/inxi.yaml
+++ b/tests/integration/tasks/inxi.yaml
@@ -16,6 +16,19 @@
command: inxi -b -c0 -n
register: inxi_log
+# TODO(yujunz) normalize system information, test condition and performance metrics for future data mining
+# e.g. convert "2 Deca core Intel Xeon E5-2650 v3s (-HT-MCP-SMP-) speed/max: 1200/3000 MHz" to
+# ---
+# processor:
+# id:
+# vendor: Intel
+# product_family: Xeon
+# processor_number: E5-2650 v3s
+# number_of_cores: 2
+# number_of_threads: None # set `None` when data is not available
+# base_frequency_mhz: 1200
+# max_turbo_frequency_mhz: 3000
+# cache_mb: None
- name: collect system information from inxi
collect:
string: "{{ inxi_log.stdout }}"
@@ -27,4 +40,4 @@
- '.+\sKernel:\s+(?P<kernel>.+)\sConsole'
- '.+\s+HDD Total Size:\s+(?P<disk>.+)\s'
- '.+\sproduct:\s+(?P<product>.+)\sv'
- register: inxi_info
+ register: system_info
diff --git a/tests/integration/tasks/openssl.yaml b/tests/integration/tasks/openssl.yaml
new file mode 100644
index 00000000..0e70913e
--- /dev/null
+++ b/tests/integration/tasks/openssl.yaml
@@ -0,0 +1,53 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: install openssl - Cryptography and SSL/TLS Toolkit
+ package:
+ name: openssl
+ state: present
+
+- name: RSA signatures speed measurement
+ command: openssl speed rsa
+ register: openssl_rsa_log
+
+#- name: AES speed measurement
+# command: openssl speed -evp aes-128-cbc
+# register: openssl_aes_log
+
+- name: collect ssl metrics from openssl
+ collect:
+ string: "{{ openssl_rsa_log.stdout }}"
+ patterns:
+ - |-
+ ^rsa\s+512\sbits\s.+\s+
+ ?(?P<rsa_sign_512>\d+\.\d)\s+
+ ?(?P<rsa_verify_512>\d+\.\d)$
+ - |-
+ ^rsa\s+1024\sbits\s.+\s+
+ ?(?P<rsa_sign_1024>\d+\.\d)\s+
+ ?(?P<rsa_verify_1024>\d+\.\d)$
+ - |-
+ ^rsa\s+2048\sbits\s.+\s+
+ ?(?P<rsa_sign_2048>\d+\.\d)\s+
+ ?(?P<rsa_verify_2048>\d+\.\d)$
+ - |-
+ ^rsa\s+4096\sbits\s.+\s+
+ ?(?P<rsa_sign_4096>\d+\.\d)\s+
+ ?(?P<rsa_verify_4096>\d+\.\d)$
+ register: openssl_rsa_metrics
+
+# - filename: AES-128-CBC_dump
+# grep:
+# - |-
+# ^aes-128-cbc\s+
+# ?(?P<aes_128_cbc_16_bytes>\d+\.\w+)\s+
+# ?(?P<aes_128_cbc_64_bytes>\d+\.\w+)\s+
+# ?(?P<aes_128_cbc_256_bytes>\d+\.\w+)\s+
+# ?(?P<aes_128_cbc_1024_bytes>\d+\.\w+)\s+
+# ?(?P<aes_128_cbc_8192_bytes>\d+\.\w+)$
diff --git a/tests/integration/templates/inxi-system-info.j2 b/tests/integration/templates/inxi-system-info.j2
deleted file mode 100644
index 35c8661f..00000000
--- a/tests/integration/templates/inxi-system-info.j2
+++ /dev/null
@@ -1,16 +0,0 @@
-System Information from inxi
-============================
-
-{% for host in groups['compute'] %}
-{{ hostvars[host].ansible_hostname }}
------------------------------
-
-{{ ('CPU Brand', hostvars[host].inxi_info.cpu[0])|justify }}
-{{ ('Disk', hostvars[host].inxi_info.disk[0])|justify }}
-{{ ('Host Name', hostvars[host].inxi_info.hostname[0])|justify }}
-{{ ('Kernel', hostvars[host].inxi_info.kernel[0])|justify }}
-{{ ('Memory', hostvars[host].inxi_info.memory[0])|justify }}
-{{ ('Operating System', hostvars[host].inxi_info.os[0])|justify }}
-{{ ('Product', hostvars[host].inxi_info.product[0])|justify }}
-
-{% endfor %}
diff --git a/tests/integration/templates/qpi-report.j2 b/tests/integration/templates/qpi-report.j2
new file mode 100644
index 00000000..afe9bfb8
--- /dev/null
+++ b/tests/integration/templates/qpi-report.j2
@@ -0,0 +1,18 @@
+Sample QPI Report
+{% for host in groups['compute'] %}
+{% set qpi_result = hostvars[host].qpi_result %}
+
+Host: {{ hostvars[host].ansible_hostname }}
+QPI: {{ qpi_result.score }}
+Spec: {{ qpi_result.spec.name }}
+
+{% for section in qpi_result.section_results %}
+- {{ section.name }}: {{ section.result.score }}
+{% for metric in section.result.metric_results %}
+ - {{ metric.name }}: {{ metric.result.score }}
+{% for workload in metric.result.workload_results %}
+ - {{ workload.name }}: {{ workload.score }}
+{% endfor %}
+{% endfor %}
+{% endfor %}
+{% endfor %}
diff --git a/tests/integration/templates/system-info.j2 b/tests/integration/templates/system-info.j2
new file mode 100644
index 00000000..305a2af2
--- /dev/null
+++ b/tests/integration/templates/system-info.j2
@@ -0,0 +1,16 @@
+System Information from inxi
+============================
+
+{% for host in groups['compute'] %}
+{{ hostvars[host].ansible_hostname }}
+-----------------------------
+
+{{ ('CPU Brand', hostvars[host].system_info.cpu[0])|justify }}
+{{ ('Disk', hostvars[host].system_info.disk[0])|justify }}
+{{ ('Host Name', hostvars[host].system_info.hostname[0])|justify }}
+{{ ('Kernel', hostvars[host].system_info.kernel[0])|justify }}
+{{ ('Memory', hostvars[host].system_info.memory[0])|justify }}
+{{ ('Operating System', hostvars[host].system_info.os[0])|justify }}
+{{ ('Product', hostvars[host].system_info.product[0])|justify }}
+
+{% endfor %}