aboutsummaryrefslogtreecommitdiffstats
path: root/tests/integration
diff options
context:
space:
mode:
Diffstat (limited to 'tests/integration')
-rw-r--r--tests/integration/compute.yaml57
-rw-r--r--tests/integration/reports/qpi-report46
-rw-r--r--tests/integration/reports/system-info (renamed from tests/integration/reports/inxi-system-info)0
-rw-r--r--tests/integration/specs/compute.yaml36
-rw-r--r--tests/integration/tasks/inxi.yaml15
-rw-r--r--tests/integration/tasks/openssl.yaml53
-rw-r--r--tests/integration/templates/inxi-system-info.j216
-rw-r--r--tests/integration/templates/qpi-report.j218
-rw-r--r--tests/integration/templates/system-info.j216
9 files changed, 238 insertions, 19 deletions
diff --git a/tests/integration/compute.yaml b/tests/integration/compute.yaml
index 4cb71e9f..1f5ec8dd 100644
--- a/tests/integration/compute.yaml
+++ b/tests/integration/compute.yaml
@@ -8,6 +8,7 @@
##############################################################################
---
+# Prepare connection to SUT (System Under Test)
- hosts: fuel-master
gather_facts: no
tasks:
@@ -20,13 +21,65 @@
ansible_user: root
ansible_ssh_common_args: '-o StrictHostKeyChecking=No -o ProxyJump=fuel-master'
with_items: "{{ hosts.compute }}"
+
+# Execute compute benchmark plan and collect data
+# - system information
+# - test condition
+# - performance metrics
- hosts: compute
tasks:
- name: check ssh connection
ping:
- - include: tasks/inxi.yaml
+ # collect system information
+ - name: collect system information
+ include: tasks/inxi.yaml
+ - name: ssl metrics
+ include: tasks/openssl.yaml
+
+- hosts: compute
+ tasks:
+ - name: calculate QPI of compute
+ calculate:
+ metrics:
+ ssl_rsa: "{{ openssl_rsa_metrics }}"
+ spec: # TODO(yujunz) load spec from file
+ name: compute
+ description: QTIP Performance Index of compute
+ formula: weighted arithmetic mean
+ sections: # split based on different application
+ - name: SSL
+ description: cryptography and SSL/TLS performance
+ formula: geometric mean
+ metrics:
+ - name: ssl_rsa
+ formual: geometric mean
+ workloads:
+ - name: rsa_sign_512
+ description: RSA signature 512 bits
+ baseline: 14982.3
+ - name: rsa_verify_512
+ baseline: 180619.2
+ - name: rsa_sign_1024
+ baseline: 5037.7
+ - name: rsa_verify_1024
+ baseline: 67359.9
+ - name: rsa_sign_2048
+ baseline: 713.6
+ - name: rsa_verify_2048
+ baseline: 23458.0
+ - name: rsa_sign_4096
+ baseline: 102.1
+ - name: rsa_verify_4096
+ baseline: 6402.9
+ register: qpi_result
+ delegate_to: localhost
+
+# Generate and publish report
- hosts: local
tasks:
- name: create system information report
- local_action: template src=templates/inxi-system-info.j2 dest=reports/inxi-system-info
+ local_action: template src=templates/system-info.j2 dest=reports/system-info
delegate_to: localhost
+ # TODO(yujunz) push test result to testapi
+ - name: create qpi report
+ template: src=templates/qpi-report.j2 dest=reports/qpi-report
diff --git a/tests/integration/reports/qpi-report b/tests/integration/reports/qpi-report
new file mode 100644
index 00000000..cb9e9308
--- /dev/null
+++ b/tests/integration/reports/qpi-report
@@ -0,0 +1,46 @@
+Sample QPI Report
+
+Host: node-26
+QPI: 1.41147857985
+Spec: compute
+
+- SSL: 1.41147857985
+ - ssl_rsa: 1.41147857985
+ - rsa_sign_512: 1.25261808935
+ - rsa_verify_512: 1.33973907536
+ - rsa_sign_1024: 1.31933223495
+ - rsa_verify_1024: 1.46972456907
+ - rsa_sign_2048: 1.8615470852
+ - rsa_verify_2048: 1.3683903146
+ - rsa_sign_4096: 1.31537708129
+ - rsa_verify_4096: 1.36510018898
+
+Host: node-28
+QPI: 1.28082308651
+Spec: compute
+
+- SSL: 1.28082308651
+ - ssl_rsa: 1.28082308651
+ - rsa_sign_512: 1.13628081136
+ - rsa_verify_512: 1.24882238433
+ - rsa_sign_1024: 1.16100601465
+ - rsa_verify_1024: 1.33382620817
+ - rsa_sign_2048: 1.72057174888
+ - rsa_verify_2048: 1.23917640038
+ - rsa_sign_4096: 1.16846229187
+ - rsa_verify_4096: 1.2384388324
+
+Host: node-27
+QPI: 1.41542492777
+Spec: compute
+
+- SSL: 1.41542492777
+ - ssl_rsa: 1.41542492777
+ - rsa_sign_512: 1.25857845591
+ - rsa_verify_512: 1.34193319426
+ - rsa_sign_1024: 1.32097981222
+ - rsa_verify_1024: 1.4807103336
+ - rsa_sign_2048: 1.86378923767
+ - rsa_verify_2048: 1.36600306932
+ - rsa_sign_4096: 1.31635651322
+ - rsa_verify_4096: 1.37504880601
diff --git a/tests/integration/reports/inxi-system-info b/tests/integration/reports/system-info
index 371243e2..371243e2 100644
--- a/tests/integration/reports/inxi-system-info
+++ b/tests/integration/reports/system-info
diff --git a/tests/integration/specs/compute.yaml b/tests/integration/specs/compute.yaml
new file mode 100644
index 00000000..0266ac8d
--- /dev/null
+++ b/tests/integration/specs/compute.yaml
@@ -0,0 +1,36 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+name: compute
+description: QTIP Performance Index of compute
+formula: weighted arithmetic mean
+sections: # split based on different application
+- name: SSL
+ description: cryptography and SSL/TLS performance
+ formula: geometric mean
+ metrics:
+ - name: rsa
+ formual: geometric mean
+ workloads:
+ - name: rsa_sign_512
+ description: RSA signature 512 bits
+ baseline: 14982.3
+ - name: rsa_verify_512
+ baseline: 180619.2
+ - name: rsa_sign_1024
+ baseline: 5037.7
+ - name: rsa_verify_1024
+ baseline: 67359.9
+ - name: rsa_sign_2048
+ baseline: 713.6
+ - name: rsa_verify_2048
+ baseline: 23458.0
+ - name: rsa_sign_4096
+ baeline: 102.1
+ - name: rsa_verify_4096
+ baseline: 6402.9
diff --git a/tests/integration/tasks/inxi.yaml b/tests/integration/tasks/inxi.yaml
index f8951dc1..6c2db6a4 100644
--- a/tests/integration/tasks/inxi.yaml
+++ b/tests/integration/tasks/inxi.yaml
@@ -16,6 +16,19 @@
command: inxi -b -c0 -n
register: inxi_log
+# TODO(yujunz) normalize system information, test condition and performance metrics for future data mining
+# e.g. convert "2 Deca core Intel Xeon E5-2650 v3s (-HT-MCP-SMP-) speed/max: 1200/3000 MHz" to
+# ---
+# processor:
+# id:
+# vendor: Intel
+# product_family: Xeon
+# processor_number: E5-2650 v3s
+# number_of_cores: 2
+# number_of_threads: None # set `None` when data is not available
+# base_frequency_mhz: 1200
+# max_turbo_frequency_mhz: 3000
+# cache_mb: None
- name: collect system information from inxi
collect:
string: "{{ inxi_log.stdout }}"
@@ -27,4 +40,4 @@
- '.+\sKernel:\s+(?P<kernel>.+)\sConsole'
- '.+\s+HDD Total Size:\s+(?P<disk>.+)\s'
- '.+\sproduct:\s+(?P<product>.+)\sv'
- register: inxi_info
+ register: system_info
diff --git a/tests/integration/tasks/openssl.yaml b/tests/integration/tasks/openssl.yaml
new file mode 100644
index 00000000..0e70913e
--- /dev/null
+++ b/tests/integration/tasks/openssl.yaml
@@ -0,0 +1,53 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: install openssl - Cryptography and SSL/TLS Toolkit
+ package:
+ name: openssl
+ state: present
+
+- name: RSA signatures speed measurement
+ command: openssl speed rsa
+ register: openssl_rsa_log
+
+#- name: AES speed measurement
+# command: openssl speed -evp aes-128-cbc
+# register: openssl_aes_log
+
+- name: collect ssl metrics from openssl
+ collect:
+ string: "{{ openssl_rsa_log.stdout }}"
+ patterns:
+ - |-
+ ^rsa\s+512\sbits\s.+\s+
+ ?(?P<rsa_sign_512>\d+\.\d)\s+
+ ?(?P<rsa_verify_512>\d+\.\d)$
+ - |-
+ ^rsa\s+1024\sbits\s.+\s+
+ ?(?P<rsa_sign_1024>\d+\.\d)\s+
+ ?(?P<rsa_verify_1024>\d+\.\d)$
+ - |-
+ ^rsa\s+2048\sbits\s.+\s+
+ ?(?P<rsa_sign_2048>\d+\.\d)\s+
+ ?(?P<rsa_verify_2048>\d+\.\d)$
+ - |-
+ ^rsa\s+4096\sbits\s.+\s+
+ ?(?P<rsa_sign_4096>\d+\.\d)\s+
+ ?(?P<rsa_verify_4096>\d+\.\d)$
+ register: openssl_rsa_metrics
+
+# - filename: AES-128-CBC_dump
+# grep:
+# - |-
+# ^aes-128-cbc\s+
+# ?(?P<aes_128_cbc_16_bytes>\d+\.\w+)\s+
+# ?(?P<aes_128_cbc_64_bytes>\d+\.\w+)\s+
+# ?(?P<aes_128_cbc_256_bytes>\d+\.\w+)\s+
+# ?(?P<aes_128_cbc_1024_bytes>\d+\.\w+)\s+
+# ?(?P<aes_128_cbc_8192_bytes>\d+\.\w+)$
diff --git a/tests/integration/templates/inxi-system-info.j2 b/tests/integration/templates/inxi-system-info.j2
deleted file mode 100644
index 35c8661f..00000000
--- a/tests/integration/templates/inxi-system-info.j2
+++ /dev/null
@@ -1,16 +0,0 @@
-System Information from inxi
-============================
-
-{% for host in groups['compute'] %}
-{{ hostvars[host].ansible_hostname }}
------------------------------
-
-{{ ('CPU Brand', hostvars[host].inxi_info.cpu[0])|justify }}
-{{ ('Disk', hostvars[host].inxi_info.disk[0])|justify }}
-{{ ('Host Name', hostvars[host].inxi_info.hostname[0])|justify }}
-{{ ('Kernel', hostvars[host].inxi_info.kernel[0])|justify }}
-{{ ('Memory', hostvars[host].inxi_info.memory[0])|justify }}
-{{ ('Operating System', hostvars[host].inxi_info.os[0])|justify }}
-{{ ('Product', hostvars[host].inxi_info.product[0])|justify }}
-
-{% endfor %}
diff --git a/tests/integration/templates/qpi-report.j2 b/tests/integration/templates/qpi-report.j2
new file mode 100644
index 00000000..afe9bfb8
--- /dev/null
+++ b/tests/integration/templates/qpi-report.j2
@@ -0,0 +1,18 @@
+Sample QPI Report
+{% for host in groups['compute'] %}
+{% set qpi_result = hostvars[host].qpi_result %}
+
+Host: {{ hostvars[host].ansible_hostname }}
+QPI: {{ qpi_result.score }}
+Spec: {{ qpi_result.spec.name }}
+
+{% for section in qpi_result.section_results %}
+- {{ section.name }}: {{ section.result.score }}
+{% for metric in section.result.metric_results %}
+ - {{ metric.name }}: {{ metric.result.score }}
+{% for workload in metric.result.workload_results %}
+ - {{ workload.name }}: {{ workload.score }}
+{% endfor %}
+{% endfor %}
+{% endfor %}
+{% endfor %}
diff --git a/tests/integration/templates/system-info.j2 b/tests/integration/templates/system-info.j2
new file mode 100644
index 00000000..305a2af2
--- /dev/null
+++ b/tests/integration/templates/system-info.j2
@@ -0,0 +1,16 @@
+System Information from inxi
+============================
+
+{% for host in groups['compute'] %}
+{{ hostvars[host].ansible_hostname }}
+-----------------------------
+
+{{ ('CPU Brand', hostvars[host].system_info.cpu[0])|justify }}
+{{ ('Disk', hostvars[host].system_info.disk[0])|justify }}
+{{ ('Host Name', hostvars[host].system_info.hostname[0])|justify }}
+{{ ('Kernel', hostvars[host].system_info.kernel[0])|justify }}
+{{ ('Memory', hostvars[host].system_info.memory[0])|justify }}
+{{ ('Operating System', hostvars[host].system_info.os[0])|justify }}
+{{ ('Product', hostvars[host].system_info.product[0])|justify }}
+
+{% endfor %}