diff options
30 files changed, 0 insertions, 1878 deletions
diff --git a/qtip/collector/calculator.py b/qtip/collector/calculator.py deleted file mode 100644 index c3d961b3..00000000 --- a/qtip/collector/calculator.py +++ /dev/null @@ -1,38 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from operator import add - -from qtip.util.logger import QtipLogger - -logger = QtipLogger('calculator').get - - -def dpi_calculator(samples): - try: - float_pps = map(lambda x: float(x), samples['pps']) - float_bps = map(lambda x: float(x), samples['bps']) - sum_dpi_pps = reduce(add, - map(lambda x: x / 1000 if x > 100 else x, float_pps)) - sum_dpi_bps = reduce(add, - map(lambda x: x / 1000 if x > 100 else x, float_bps)) - - return {'pps': round(sum_dpi_pps / 10, 3), 'bps': round(sum_dpi_bps / 10, 3)} - except Exception as error: - logger.error(error) - return {'pps': None, 'bps': None} - - -def calculate_cpu_usage(cpu_idle): - try: - cpu_usage = round((100.0 - float(cpu_idle)), 3) - return '{0}%'.format(str(cpu_usage)) - except Exception, error: - logger.error(error) - return None diff --git a/qtip/collector/parser/grep.py b/qtip/collector/parser/grep.py index d3a8210a..66b51b1e 100644 --- a/qtip/collector/parser/grep.py +++ b/qtip/collector/parser/grep.py @@ -7,21 +7,15 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from collections import defaultdict -from os import path import re -import yaml from qtip.base import BaseActor from qtip.base.constant import BaseProp -from qtip.collector import calculator from qtip.util.logger import QtipLogger logger = QtipLogger('grep').get -REGEX_FILE = path.join(path.dirname(__file__), 'regex.yaml') - class GrepProp(BaseProp): FILENAME = 'filename' @@ -40,63 +34,3 @@ def grep_in_file(filename, regex): with open(filename, 'r') as f: return filter(lambda x: x is not None, re.finditer(regex, f.read(), re.MULTILINE)) - - -def _parse_logfile(config, paths): - captured = {} - for regex_rules_by_file in config: - filename = \ - '{0}/{1}'.format(paths, regex_rules_by_file[GrepProp.FILENAME]) - for regex in regex_rules_by_file['grep']: - matches = grep_in_file(filename, regex) - if len(matches) > 1: - temp_dict = defaultdict(list) - for item in [match.groupdict() for match in matches]: - for key in item: - temp_dict[key].append(item[key]) - captured.update(temp_dict) - elif len(matches) == 1: - captured.update(matches[0].groupdict()) - else: - logger.error("Nothing is matched from {0}".format(filename)) - return captured - - -# TODO: Hardcord in Danube, it will be removed in the future. -def parse_sysinfo(config, result_dir): - sysinfo = _parse_logfile(config, result_dir) - if "cpu_idle" in sysinfo: - sysinfo['cpu_usage'] = \ - calculator.calculate_cpu_usage(sysinfo['cpu_idle']) - sysinfo.pop('cpu_idle') - return sysinfo - - -# TODO: Hardcord in Danube, it will be removed in the future. -def parse_test_result(benchmark, config, result_dir): - test_result = _parse_logfile(config, result_dir) - if benchmark == 'dpi': - return calculator.dpi_calculator(test_result) - if benchmark == 'dhrystone' or benchmark == 'whetstone': - return {'total_cpus': test_result['total_cpus'], - 'single_cpu': {'num': test_result['single_cpu'], - 'score': test_result['score'][0]}, - 'multi_cpus': {'num': test_result['multi_cpus'], - 'score': test_result['score'][1]}} - return test_result - - -# TODO: Hardcord in Danube, it will be removed in the future. -def parse_benchmark_result(result_dir): - regex_config = yaml.safe_load(file(REGEX_FILE)) - benchmark = result_dir.split('/')[-1] - result = {'name': benchmark} - - test_result = \ - parse_test_result(benchmark, regex_config[benchmark], result_dir) - result['results'] = test_result.copy() - - sysinfo = parse_sysinfo(regex_config['sysinfo'], result_dir) - result['sysinfo'] = sysinfo.copy() - - return result diff --git a/qtip/collector/parser/regex.yaml b/qtip/collector/parser/regex.yaml deleted file mode 100644 index e1512855..00000000 --- a/qtip/collector/parser/regex.yaml +++ /dev/null @@ -1,85 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -dhrystone: - - filename: dhrystone - grep: - - '^(?P<total_cpus>\d+)\sCPUs in system; running 1 parallel copy of tests$' - - '.+\srunning (?P<single_cpu>\d+) parallel copy of tests$' - - '.+\srunning (?P<multi_cpus>\d+) parallel copies of tests$' - - '^System Benchmarks Index Score \(Partial Only\)\s+(?P<score>\d+\.\d)$' -whetstone: - - filename: whetstone - grep: - - '^(?P<total_cpus>\d+)\sCPUs in system; running 1 parallel copy of tests$' - - '.+\srunning (?P<single_cpu>\d+) parallel copy of tests$' - - '.+\srunning (?P<multi_cpus>\d+) parallel copies of tests$' - - '^System Benchmarks Index Score \(Partial Only\)\s+(?P<score>\d+\.\d)$' -dpi: - - filename: dpi_dump.txt - grep: - - |- - ^\s+nDPI throughput:.+?(?P<pps>\d+.\d+)\s.+\spps.+ - ?(?P<bps>\d+.\d+)\s.+\/sec -ramspeed: - - filename: Intmem - grep: - - '^INTEGER\s+BatchRun\s+Copy:\s+?(?P<integer_copy>\d+\.\d+)\sMB/s$' - - '^INTEGER\s+BatchRun\s+Scale:\s+?(?P<integer_scale>\d+\.\d+)\sMB/s$' - - '^INTEGER\s+BatchRun\s+Add:\s+?(?P<integer_add>\d+\.\d+)\sMB/s$' - - '^INTEGER\s+BatchRun\s+Triad:\s+?(?P<integer_triad>\d+\.\d+)\sMB/s$' - - '^INTEGER\s+BatchRun\s+AVERAGE:\s+?(?P<integer_average>\d+\.\d+)\sMB/s$' - - filename: Floatmem - grep: - - '^FL-POINT\s+BatchRun\s+Copy:\s+?(?P<float_copy>\d+\.\d+)\sMB/s$' - - '^FL-POINT\s+BatchRun\s+Scale:\s+?(?P<float_scale>\d+\.\d+)\sMB/s$' - - '^FL-POINT\s+BatchRun\s+Add:\s+?(?P<float_add>\d+\.\d+)\sMB/s$' - - '^FL-POINT\s+BatchRun\s+Triad:\s+?(?P<float_triad>\d+\.\d+)\sMB/s$' - - '^FL-POINT\s+BatchRun\s+AVERAGE:\s+?(?P<float_average>\d+\.\d+)\sMB/s$' -ssl: - - filename: RSA_dump - grep: - - |- - ^rsa\s+512\sbits\s.+\s+ - ?(?P<rsa_sign_512>\d+\.\d)\s+ - ?(?P<rsa_verify_512>\d+\.\d)$ - - |- - ^rsa\s+1024\sbits\s.+\s+ - ?(?P<rsa_sign_1024>\d+\.\d)\s+ - ?(?P<rsa_verify_1024>\d+\.\d)$ - - |- - ^rsa\s+2048\sbits\s.+\s+ - ?(?P<rsa_sign_2048>\d+\.\d)\s+ - ?(?P<rsa_verify_2048>\d+\.\d)$ - - |- - ^rsa\s+4096\sbits\s.+\s+ - ?(?P<rsa_sign_4096>\d+\.\d)\s+ - ?(?P<rsa_verify_4096>\d+\.\d)$ - - filename: AES-128-CBC_dump - grep: - - |- - ^aes-128-cbc\s+ - ?(?P<aes_128_cbc_16_bytes>\d+\.\w+)\s+ - ?(?P<aes_128_cbc_64_bytes>\d+\.\w+)\s+ - ?(?P<aes_128_cbc_256_bytes>\d+\.\w+)\s+ - ?(?P<aes_128_cbc_1024_bytes>\d+\.\w+)\s+ - ?(?P<aes_128_cbc_8192_bytes>\d+\.\w+)$ -sysinfo: - - filename: top.log - grep: - - 'Cpu\(s\):.+?(?P<cpu_idle>\d+\.\d)\sid' - - filename: inxi.log - grep: - - '.+\s+Host:\s+(?P<hostname>.+)\sKernel' - - '.+\sMemory:\s+(?P<memory>.+MB)\s' - - '^CPU\(s\):\s+(?P<cpu>.+)' - - '.+\sDistro:\s+(?P<os>.+)' - - '.+\sKernel:\s+(?P<kernel>.+)\sConsole' - - '.+\s+HDD Total Size:\s+(?P<disk>.+)\s' - - '.+\sproduct:\s+(?P<product>.+)\sv'
\ No newline at end of file diff --git a/qtip/driver/__init__.py b/qtip/driver/__init__.py deleted file mode 100644 index e69de29b..00000000 --- a/qtip/driver/__init__.py +++ /dev/null diff --git a/qtip/driver/ansible_api.py b/qtip/driver/ansible_api.py deleted file mode 100644 index 5c5baffc..00000000 --- a/qtip/driver/ansible_api.py +++ /dev/null @@ -1,58 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from collections import namedtuple - -from ansible.executor.playbook_executor import PlaybookExecutor -from ansible.inventory import Inventory -from ansible.parsing.dataloader import DataLoader -from ansible.vars import VariableManager - - -class AnsibleApi(object): - - def __init__(self): - self.variable_manager = VariableManager() - self.loader = DataLoader() - self.passwords = {} - self.playbook_executor = None - - def execute_playbook(self, playbook_path, hosts_file=None, - key_file=None, extra_vars=None): - inventory = Inventory(loader=self.loader, - variable_manager=self.variable_manager, - host_list=hosts_file) - Options = namedtuple('Options', - ['listtags', 'listtasks', 'listhosts', 'syntax', - 'connection', 'module_path', 'forks', 'remote_user', - 'private_key_file', 'ssh_common_args', 'ssh_extra_args', - 'sftp_extra_args', 'scp_extra_args', 'become', - 'become_method', 'become_user', 'verbosity', 'check']) - options = Options(listtags=False, listtasks=False, listhosts=False, - syntax=False, connection='ssh', module_path=None, - forks=100, remote_user='root', private_key_file=key_file, - ssh_common_args=None, ssh_extra_args=None, sftp_extra_args=None, - scp_extra_args=None, become=None, become_method=None, - become_user='root', verbosity=None, check=False) - self.variable_manager.extra_vars = extra_vars - - self.playbook_executor = PlaybookExecutor(playbooks=[playbook_path], - inventory=inventory, - variable_manager=self.variable_manager, - loader=self.loader, - options=options, - passwords=self.passwords) - return self.playbook_executor.run() - - def get_detail_playbook_stats(self): - if self.playbook_executor: - stats = self.playbook_executor._tqm._stats - return map(lambda x: (x, stats.summarize(x)), stats.processed.keys()) - else: - return None diff --git a/qtip/driver/ansible_driver.py b/qtip/driver/ansible_driver.py deleted file mode 100644 index 34ef46ed..00000000 --- a/qtip/driver/ansible_driver.py +++ /dev/null @@ -1,102 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from collections import defaultdict -from os import path -from operator import add - -from qtip.driver.ansible_api import AnsibleApi -from qtip.util.env import AnsibleEnvSetup -from qtip.util.logger import QtipLogger - -logger = QtipLogger('ansible_driver').get -PLAYBOOK_DIR = path.join(path.dirname(__file__), 'playbook') - - -class AnsibleDriver(object): - """driver for running performance tests with Ansible""" - - def __init__(self, config={}): - self.config = config - self.env = AnsibleEnvSetup() - self.env_setup_flag = False - - @staticmethod - def merge_two_dicts(x, y): - ''' - It is from http://stackoverflow.com/questions/38987/ - how-can-i-merge-two-python-dictionaries-in-a-single-expression - ''' - z = x.copy() - z.update(y) - return z - - def pre_run(self): - if self.env_setup_flag: - logger.info("Already setup environment......") - else: - logger.info("Starting to setup test environment...") - self.env.setup(self.config) - self.env_setup_flag = True - logger.info("Setup test enviroment, Done!") - - def cleanup(self): - self.env.cleanup() - - def run(self, metric_list, **kwargs): - if 'args' in self.config: - extra_vars = self.merge_two_dicts(kwargs, self.config['args']) - else: - extra_vars = kwargs - logger.info("extra_var: {0}".format(extra_vars)) - - tool_to_metrics = defaultdict(list) - for metric in metric_list: - if metric == 'dhrystone' or metric == 'whetstone': - tool_to_metrics['unixbench'].append(metric) - extra_vars[metric] = True - elif metric == 'ssl': - tool_to_metrics['openssl'].append(metric) - else: - tool_to_metrics[metric].append(metric) - - result_list = map(lambda tool: self._run_metric(tool, - tool_to_metrics[tool], - extra_vars), - tool_to_metrics) - return False not in result_list - - def _run_metric(self, tool, metrics, extra_vars): - logger.info('Using {0} to measure metrics {1}'.format(tool, metrics)) - - setup_pbook = "{0}/{1}/setup.yaml".format(PLAYBOOK_DIR, tool) - run_pbook = "{0}/{1}/run.yaml".format(PLAYBOOK_DIR, tool) - clean_pbook = "{0}/{1}/clean.yaml".format(PLAYBOOK_DIR, tool) - - if self._run_ansible_playbook(setup_pbook, extra_vars): - self._run_ansible_playbook(run_pbook, extra_vars) - else: - logger.error("{0} is failed.".format(setup_pbook)) - - return self._run_ansible_playbook(clean_pbook, extra_vars) - - def _run_ansible_playbook(self, pbook, extra_vars): - ansible_api = AnsibleApi() - logger.debug("Run {0} with extra_vars: {1}".format(pbook, extra_vars)) - ansible_api.execute_playbook(pbook, self.env.hostfile, - self.env.keypair['private'], extra_vars) - playbook_stats = ansible_api.get_detail_playbook_stats() - logger.debug("playbook_stat: {0}".format(playbook_stats)) - return self.is_pass(playbook_stats) - - @staticmethod - def is_pass(stats): - return 0 == reduce(add, - map(lambda x: x[1]['failures'] + x[1]['unreachable'], - stats)) diff --git a/qtip/driver/base.py b/qtip/driver/base.py deleted file mode 100644 index 6f5cab3c..00000000 --- a/qtip/driver/base.py +++ /dev/null @@ -1,20 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -class BaseDriver(object): - """performance testing tool driver""" - def pre_run(self): - pass - - def run(self): - pass - - def post_run(self): - pass diff --git a/qtip/driver/playbook/dpi/clean.yaml b/qtip/driver/playbook/dpi/clean.yaml deleted file mode 100644 index 0b9f9291..00000000 --- a/qtip/driver/playbook/dpi/clean.yaml +++ /dev/null @@ -1,23 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Cleaning tempD - file: - path: '{{ ansible_env.HOME }}/tempD' - state: absent - - - name: Cleaning qtip_result - file: - path: '{{ ansible_env.HOME }}/qtip_result' - state: absent diff --git a/qtip/driver/playbook/dpi/dpi_average.sh b/qtip/driver/playbook/dpi/dpi_average.sh deleted file mode 100644 index 6f038053..00000000 --- a/qtip/driver/playbook/dpi/dpi_average.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -COUNTER=0 -WDIR=$PWD -while [ $COUNTER -lt 10 ]; do - echo $WDIR - $( ./ndpiReader -i test.pcap >> $WDIR/dpi_dump.txt ) - let COUNTER=COUNTER+1 - echo "Run number: $COUNTER" -done
\ No newline at end of file diff --git a/qtip/driver/playbook/dpi/run.yaml b/qtip/driver/playbook/dpi/run.yaml deleted file mode 100644 index f4c8c457..00000000 --- a/qtip/driver/playbook/dpi/run.yaml +++ /dev/null @@ -1,42 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Make some directories needed - file: - path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dpi/' - state: directory - - - include: ../inxi.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dpi/' - - - include: ../top.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dpi/' - - - name: Run nDPI benchmark - shell: ./dpi_average.sh - args: - chdir: '{{ ansible_env.HOME }}/tempD/nDPI/example/' - - - name: Copying result and system info to qtip result directory - command: cp $HOME/tempD/nDPI/example/dpi_dump.txt ./ - args: - chdir: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dpi/' - - - name: Fetch result files to local manchine - synchronize: - mode: pull - src: '{{ ansible_env.HOME }}/qtip_result/' - dest: '{{ result_dir }}/' diff --git a/qtip/driver/playbook/dpi/setup.yaml b/qtip/driver/playbook/dpi/setup.yaml deleted file mode 100644 index 76e62e6d..00000000 --- a/qtip/driver/playbook/dpi/setup.yaml +++ /dev/null @@ -1,93 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: localhost - connection: local - gather_facts: no - - tasks: - - name: Making Dpi directory - file: - path: '{{ result_dir }}/' - state: directory - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Cleaning tempD directory - file: - path: '{{ ansible_env.HOME }}/tempD' - state: absent - - - name: Cleaning qtip_result directory - file: - path: '{{ ansible_env.HOME }}/qtip_result' - state: absent - - - include: ../prepare_env.yaml - - - name: Installing nDPI dependencies if CentOS - yum: - name: '{{ item }}' - state: present - when: ansible_os_family == "RedHat" - with_items: - - git - - gcc - - patch - - perl-Time-HiRes - - autofconf - - automake - - libpcap-devel libtool - - - name: Installing nDPI dependencies if Ubuntu - apt: - name: '{{ item }}' - state: present - when: ansible_os_family == "Debian" - with_items: - - git - - gcc - - patch - - autoconf - - automake - - libpcap-dev - - libtool - - - name: Making nDPI temporary directory - file: - path: '{{ ansible_env.HOME }}/tempD' - state: directory - - - name: Clone nDPI - git: - repo: https://github.com/ntop/nDPI.git - dest: '{{ ansible_env.HOME }}/tempD/nDPI' - - - name: Run autogen && configure Dpi && make Dpi - command: '{{ item }}' - with_items: - - ./autogen.sh - - ./configure - - make - args: - chdir: '{{ ansible_env.HOME }}/tempD/nDPI/' - - - name: Fetching Test_pcap file - get_url: - url: http://build.opnfv.org/artifacts.opnfv.org/qtip/utilities/test.pcap - dest: '{{ ansible_env.HOME }}/tempD/nDPI/example/test.pcap' - - - name: Fetch Averaging script - copy: - src: ./dpi_average.sh - dest: '{{ ansible_env.HOME }}/tempD/nDPI/example' - mode: 777
\ No newline at end of file diff --git a/qtip/driver/playbook/inxi.yaml b/qtip/driver/playbook/inxi.yaml deleted file mode 100644 index 2a4d9b3f..00000000 --- a/qtip/driver/playbook/inxi.yaml +++ /dev/null @@ -1,25 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -- name: Install Inxi when CentOS - yum: - name: inxi - state: present - when: ansible_os_family == "RedHat" - -- name: Install Inxi when Ubuntu - apt: - name: inxi - state: present - update_cache: yes - when: ansible_os_family == "Debian" - -- name: Run inxi - shell: inxi -b -c0 -n > inxi.log - args: - chdir: '{{ dest_path }}' diff --git a/qtip/driver/playbook/openssl/clean.yaml b/qtip/driver/playbook/openssl/clean.yaml deleted file mode 100644 index 0139ba54..00000000 --- a/qtip/driver/playbook/openssl/clean.yaml +++ /dev/null @@ -1,23 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Cleaning Open_SSL - file: - path: '{{ ansible_env.HOME }}/Open_SSL' - state: absent - - - name: Cleaning qtip_result - file: - path: '{{ ansible_env.HOME }}/qtip_result' - state: absent diff --git a/qtip/driver/playbook/openssl/run.yaml b/qtip/driver/playbook/openssl/run.yaml deleted file mode 100644 index db923fed..00000000 --- a/qtip/driver/playbook/openssl/run.yaml +++ /dev/null @@ -1,45 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Make some directories needed - file: - path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ssl/' - state: directory - - - include: ../inxi.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ssl/' - - - include: ../top.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ssl/' - - - name: Benchmarking RSA signatures and AES-128-cbc cipher encryption throughput - shell: '{{ item }}' - with_items: - - ./openssl speed rsa >> RSA_dump - - ./openssl speed -evp aes-128-cbc >> AES-128-CBC_dump - args: - chdir: '{{ ansible_env.HOME }}/Open_SSL/openssl-1.0.2f/apps' - - - name: Copying result to qtip result directory - shell: cp ~/Open_SSL/openssl-1.0.2f/apps/*_dump ./ - args: - chdir: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ssl/' - - - name: Fetch result files to local manchine - synchronize: - mode: pull - src: '{{ ansible_env.HOME }}/qtip_result/' - dest: '{{ result_dir }}/' diff --git a/qtip/driver/playbook/openssl/setup.yaml b/qtip/driver/playbook/openssl/setup.yaml deleted file mode 100644 index 3a6f385a..00000000 --- a/qtip/driver/playbook/openssl/setup.yaml +++ /dev/null @@ -1,87 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: localhost - connection: local - gather_facts: no - - tasks: - - name: Making ssl directory - file: - path: '{{ result_dir }}/' - state: directory - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Cleaning Open_SSL directory - file: - path: '{{ ansible_env.HOME }}/Open_SSL' - state: absent - - - name: Cleaning qtip_result directory - file: - path: '{{ ansible_env.HOME }}/qtip_result' - state: absent - - - include: ../prepare_env.yaml - - - name: Installing UnixBench dependencies if CentOS - yum: - name: '{{ item }}' - state: present - when: ansible_os_family == "RedHat" - with_items: - - git - - gcc - - patch - - perl-Time-HiRes - - wget - - autofconf - - automake - - libpcap-devel - - libtool - - - name: Installing UnixBench dependencies if Ubuntu - apt: - name: '{{ item }}' - state: present - when: ansible_os_family == "Debian" - with_items: - - git - - gcc - - patch - - perl - - wget - - autoconf - - automake - - libpcap-dev - - libtool - - - name: Make Open_SSL directory - file: - path: '{{ ansible_env.HOME }}/Open_SSL' - state: directory - - - name: Untar OpenSSL - unarchive: - src: http://artifacts.opnfv.org/qtip/utilities/openssl-1.0.2f.tar.gz - dest: '{{ ansible_env.HOME }}/Open_SSL/' - remote_src: True - - - name: Configure && Make && Install OpenSSL - shell: "{{ item }}" - with_items: - - ./config - - make - - make install - args: - chdir: '{{ ansible_env.HOME }}/Open_SSL/openssl-1.0.2f'
\ No newline at end of file diff --git a/qtip/driver/playbook/prepare_env.yaml b/qtip/driver/playbook/prepare_env.yaml deleted file mode 100644 index 1ec71520..00000000 --- a/qtip/driver/playbook/prepare_env.yaml +++ /dev/null @@ -1,54 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -- name: Epel Release install when CentOS - yum: - name: epel-release - state: present - when: ansible_os_family == "RedHat" - -- name: Software Properties Common - apt: - name: software-properties-common - state: present - when: ansible_os_family == "Debian" - -- name: Adding ubuntu backport main repo - apt_repository: - repo: deb http://archive.ubuntu.com/ubuntu/ {{ansible_distribution_release}}-backports main restricted universe multiverse - state: present - when: ansible_os_family == "Debian" - -- name: Adding ubuntu main repo - apt_repository: - repo: deb http://archive.ubuntu.com/ubuntu/ {{ansible_distribution_release}} main restricted universe multiverse - when: ansible_os_family == "Debian" - -- name: Install ansible copy dependencies if remote host has selinux enabled - yum: - name: libselinux-python - state: present - when: ansible_os_family == "RedHat" - -- name: Install ansiblle copy dependencies if remote host has selinux enaled - apt: - name: python-selinux - state: present - when: ansible_os_family == "Debian" - -- name: Install rsync when CentOS - yum: - name: rsync - state: present - when: ansible_os_family == "RedHat" - -- name: Install rsync when Ubuntu - apt: - name: rsync - state: present - when: ansible_os_family == "Debian" diff --git a/qtip/driver/playbook/ramspeed/clean.yaml b/qtip/driver/playbook/ramspeed/clean.yaml deleted file mode 100644 index f0188159..00000000 --- a/qtip/driver/playbook/ramspeed/clean.yaml +++ /dev/null @@ -1,23 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Cleaning ramspeed - file: - path: '{{ ansible_env.HOME }}/ramspeed' - state: absent - - - name: Cleaning qtip_result - file: - path: '{{ ansible_env.HOME }}/qtip_result' - state: absent diff --git a/qtip/driver/playbook/ramspeed/run.yaml b/qtip/driver/playbook/ramspeed/run.yaml deleted file mode 100644 index 496cd5db..00000000 --- a/qtip/driver/playbook/ramspeed/run.yaml +++ /dev/null @@ -1,40 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Make some directories needed - file: - path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ramspeed/' - state: directory - - - include: ../inxi.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ramspeed/' - - - include: ../top.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ramspeed/' - - - name: Benchmarking IntMem Bandwidth and FloatMem Bandwidth - shell: '{{ item }}' - with_items: - - ~/ramspeed/ramsmp-3.5.0/ramsmp -b 3 -l 5 -p 1 >> Intmem - - ~/ramspeed/ramsmp-3.5.0/ramsmp -b 6 -l 5 -p 1 >> Floatmem - args: - chdir: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ramspeed/' - - - name: Fetch result files to local manchine - synchronize: - mode: pull - src: '{{ ansible_env.HOME }}/qtip_result/' - dest: '{{ result_dir }}/' diff --git a/qtip/driver/playbook/ramspeed/setup.yaml b/qtip/driver/playbook/ramspeed/setup.yaml deleted file mode 100644 index 842bbda2..00000000 --- a/qtip/driver/playbook/ramspeed/setup.yaml +++ /dev/null @@ -1,70 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: localhost - connection: local - gather_facts: no - - tasks: - - name: Making ramspeed directory - file: - path: '{{ result_dir }}/' - state: directory - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Cleaning ramspeed directory - file: - path: '{{ ansible_env.HOME }}/ramspeed' - state: absent - - - name: Cleaning qtip_result directory - file: - path: '{{ ansible_env.HOME }}/qtip_result' - state: absent - - - include: ../prepare_env.yaml - - - name: Installing RAM_Speed dependencies if CentOS - yum: - name: '{{ item }}' - state: present - when: ansible_os_family == "RedHat" - with_items: - - gcc - - wget - - - name: Installing RAM_Speed dependencies if Ubuntu - apt: - name: '{{ item }}' - state: present - when: ansible_os_family == "Debian" - with_items: - - gcc - - wget - - - name: Making ramspeed temporary directory - file: - path: '{{ ansible_env.HOME }}/ramspeed' - state: directory - - - name: Fetch and untar ramspeed.tar.gz - unarchive: - # TODO: Need to upload this file to http://artifacts.opnfv.org/qtip/utilities - src: https://docs.google.com/uc?id=0B92Bp5LZTM7gRFctalZLMktTNDQ - dest: '{{ ansible_env.HOME }}/ramspeed/' - remote_src: True - - - name: Build ramsmp - shell: ./build.sh - args: - chdir: '{{ ansible_env.HOME }}/ramspeed/ramsmp-3.5.0' diff --git a/qtip/driver/playbook/unixbench/clean.yaml b/qtip/driver/playbook/unixbench/clean.yaml deleted file mode 100644 index a7cb2540..00000000 --- a/qtip/driver/playbook/unixbench/clean.yaml +++ /dev/null @@ -1,25 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# zhihui.wu1@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Cleaning tempT - file: - path: '{{ ansible_env.HOME }}/tempT' - state: absent - - - name: Cleaning qtip_result - file: - path: '{{ ansible_env.HOME }}/qtip_result' - state: absent - - diff --git a/qtip/driver/playbook/unixbench/dhrystone.yaml b/qtip/driver/playbook/unixbench/dhrystone.yaml deleted file mode 100644 index a0ee89a3..00000000 --- a/qtip/driver/playbook/unixbench/dhrystone.yaml +++ /dev/null @@ -1,41 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# zhihui.wu1@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- name: Make dhrystone directories - file: - path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dhrystone/' - state: directory - -- include: ../inxi.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dhrystone/' - -- include: ../top.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dhrystone/' - -- name: Run dhrystone - shell: ./Run -v dhrystone - args: - chdir: '{{ ansible_env.HOME }}/tempT/UnixBench/' - -- name: Copying result to qtip result directory - shell: '{{ item }}' - with_items: - - mv ~/tempT/UnixBench/results/*.log ./ - - mv ~/tempT/UnixBench/results/*.html ./ - - mv ~/tempT/UnixBench/results/* ./dhrystone - args: - chdir: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dhrystone/' - -- name: Fetch dhrystone result files to local manchine - synchronize: - mode: pull - src: '{{ ansible_env.HOME }}/qtip_result/' - dest: '{{ result_dir }}/' diff --git a/qtip/driver/playbook/unixbench/run.yaml b/qtip/driver/playbook/unixbench/run.yaml deleted file mode 100644 index fbe4d4ad..00000000 --- a/qtip/driver/playbook/unixbench/run.yaml +++ /dev/null @@ -1,23 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# zhihui.wu1@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - set_fact: - is_dhrystone: "{{ dhrystone | default(False) }}" - is_whetstone: "{{ whetstone | default(False) }}" - - - include: ./dhrystone.yaml - when: "{{ is_dhrystone }}" - - - include: ./whetstone.yaml - when: "{{ is_whetstone }}"
\ No newline at end of file diff --git a/qtip/driver/playbook/unixbench/setup.yaml b/qtip/driver/playbook/unixbench/setup.yaml deleted file mode 100644 index 4dcdd2c4..00000000 --- a/qtip/driver/playbook/unixbench/setup.yaml +++ /dev/null @@ -1,65 +0,0 @@ -############################################################################# -# Copyright (c) 2017 ZTE Corporation and others. -# zhihui.wu1@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################# - -- hosts: localhost - connection: local - gather_facts: no - - tasks: - - name: Making dhrystone directory - file: - path: '{{ result_dir }}/' - state: directory - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Cleaning tempT directory - file: - path: '{{ ansible_env.HOME }}/tempT' - state: absent - - - name: Cleaning qtip_result directory - file: - path: '{{ ansible_env.HOME }}/qtip_result' - state: absent - - - include: ../prepare_env.yaml - - - name: Installing UnixBench dependencies if CentOS - yum: - name: '{{ item }}' - state: present - when: ansible_os_family == "RedHat" - with_items: - - git - - gcc - - patch - - perl-Time-HiRes - - - name: Installing UnixBench dependencies if Ubuntu - apt: - name: '{{ item }}' - state: present - when: ansible_os_family == "Debian" - with_items: - - git - - gcc - - patch - - perl - - - name: Clone UnixBench - git: - repo: https://github.com/kdlucas/byte-unixbench.git - dest: '{{ ansible_env.HOME }}/tempT/' - - - name: Make UnixBench1 - shell: make --directory $HOME/tempT/UnixBench/
\ No newline at end of file diff --git a/qtip/driver/playbook/unixbench/whetstone.yaml b/qtip/driver/playbook/unixbench/whetstone.yaml deleted file mode 100644 index c753779f..00000000 --- a/qtip/driver/playbook/unixbench/whetstone.yaml +++ /dev/null @@ -1,41 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# zhihui.wu1@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- name: Make whetstone directories - file: - path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/whetstone/' - state: directory - -- include: ../inxi.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/whetstone/' - -- include: ../top.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/whetstone/' - -- name: Run whetstone - shell: ./Run -v whetstone - args: - chdir: '{{ ansible_env.HOME }}/tempT/UnixBench/' - -- name: Copying result to qtip result directory - shell: '{{ item }}' - with_items: - - mv ~/tempT/UnixBench/results/*.log ./ - - mv ~/tempT/UnixBench/results/*.html ./ - - mv ~/tempT/UnixBench/results/* ./whetstone - args: - chdir: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/whetstone/' - -- name: Fetch whetstone result files to local manchine - synchronize: - mode: pull - src: '{{ ansible_env.HOME }}/qtip_result/' - dest: '{{ result_dir }}/' diff --git a/qtip/runner/runner.py b/qtip/runner/runner.py deleted file mode 100644 index 9b09f0f8..00000000 --- a/qtip/runner/runner.py +++ /dev/null @@ -1,107 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE corp. and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -import argparse -import json -import os -from os import path -import sys -import time - -from qtip.collector.parser import grep -from qtip.driver.ansible_driver import AnsibleDriver -from qtip.util.logger import QtipLogger - -logger = QtipLogger('runner').get - -ALL_BENCHMARKS = ['dpi', 'ramspeed', 'ssl', 'dhrystone', 'whetstone'] - - -def parse_args(args): - parser = argparse.ArgumentParser() - parser.add_argument('-d', '--dest', required=True, - help='the destination where results will be stored.') - parser.add_argument('-b', '--benchmark', required=True, action='append', - help='the benchmark you want to execute.') - return parser.parse_args(args) - - -def run_benchmark(result_dir, benchmarks): - if not path.isdir(result_dir): - os.makedirs(result_dir) - driver = AnsibleDriver({'args': {'result_dir': result_dir}}) - driver.pre_run() - result = driver.run(benchmarks) - driver.cleanup() - return result - - -def generate_report(result_dir, start_time, stop_time): - output = { - "plan_name": "compute_qpi", - "start_time": start_time, - "stop_time": stop_time, - "sut": [] - } - output.update(parse_result(result_dir)) - output.update({'stop_time': stop_time}) - with open('{0}/result.json'.format(result_dir), 'w+') as f: - json.dump(output, f, indent=4, sort_keys=True) - - -def parse_result(result_dir): - sut_template = {'sut': []} - nodes_list = os.listdir(result_dir) - for node in nodes_list: - node_output_template = { - 'name': node, - 'type': 'baremetal', - 'qpis': [] - } - qpi_result = {'name': 'compute_qpi', 'benchmarks': []} - for benchmark in os.listdir('{0}/{1}'.format(result_dir, node)): - benchmark_result = \ - grep.parse_benchmark_result( - '{0}/{1}/{2}'.format(result_dir, node, benchmark)) - qpi_result['benchmarks'].append(benchmark_result) - node_output_template['qpis'].append(qpi_result) - sut_template['sut'].append(node_output_template) - return sut_template - - -def main(args=sys.argv[1:]): - args = parse_args(args) - - if not path.isdir(str(args.dest)): - logger.error("The destination {0} you give doesn't exist. " - "Please check!".format(args.dest)) - sys.exit(1) - - if args.benchmark == ['all']: - args.benchmark = ALL_BENCHMARKS - elif len(set(args.benchmark).difference(ALL_BENCHMARKS)) != 0: - logger.error("Please check benchmarks name. The supported benchmarks are" - "{0}".format(ALL_BENCHMARKS)) - logger.info("Start to run benchmark test: {0}.".format(args.benchmark)) - - start_time = time.strftime("%Y-%m-%d-%H-%M") - logger.info("start_time: {0}".format(start_time)) - if not args.dest.endswith('/'): - args.dest += '/' - result_dir = args.dest + 'qtip-' + start_time - ansible_result = run_benchmark(result_dir, args.benchmark) - stop_time = time.strftime("%Y-%m-%d-%H-%M") - logger.info("stop_time: {0}".format(stop_time)) - if not ansible_result: - logger.error("Bechmarks run failed. Cann't generate any report.") - sys.exit(1) - generate_report(result_dir, start_time, stop_time) - - -if __name__ == "__main__": - main() diff --git a/qtip/scripts/cleanup_creds.sh b/qtip/scripts/cleanup_creds.sh deleted file mode 100755 index ad66ba95..00000000 --- a/qtip/scripts/cleanup_creds.sh +++ /dev/null @@ -1,20 +0,0 @@ -#! /bin/bash -############################################################################## -# Copyright (c) 2017 ZTE corp. and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -DEST_IP=$1 -PRIVATE_KEY=$2 -PUBLIC_KEY=$3 -sshoptions="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" - -case "$INSTALLER_TYPE" in - fuel) - ssh $sshoptions -i $PRIVATE_KEY root@$DEST_IP "sed -i '/$PUBLIC_KEY/d' /root/.ssh/authorized_keys" - ;; -esac diff --git a/qtip/scripts/generate_host_file.sh b/qtip/scripts/generate_host_file.sh deleted file mode 100755 index ecc4d55f..00000000 --- a/qtip/scripts/generate_host_file.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash -############################################################################## -#Copyright (c) 2016 Ericsson AB, ZTE and others. -#jose.lausuch@ericsson.com -#All rights reserved. This program and the accompanying materials -#are made available under the terms of the Apache License, Version 2.0 -#which accompanies this distribution, and is available at -#http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -usage(){ - echo "usage: $0 -t <installer_type> -i <installer_ipaddr> -d <dest_hostfile>" >&2 -} - -info() { - logger -s -t "generate_host_file.info" "$*" -} - -error() { - logger -s -t "generate_host_file.error" "$*" - exit 1 -} - -verify_connectivity(){ - local ip=$1 - info "Verifying connectivity to $ip..." - for i in $(seq 0 10); do - if ping -c 1 -W 1 $ip > /dev/null; then - info "$ip is reachable!" - return 0 - fi - sleep 1 - done - error "Can not talk to $ip." -} - -:${DEPLOY_TYPE:=''} - -#Getoptions -while getopts ":t:i:d:" optchar; do - case "${optchar}" in - t) installer_type=${OPTARG} ;; - i) installer_ipaddr=${OPTARG} ;; - d) dest_hostfile=${OPTARG} ;; - *) echo "Non-option argument: '-${OPTARG}'" >&2 - usage - exit 2 - ;; - esac -done - -#set vars from env if not provided by user as options -installer_type=${installer_type:-$INSTALLER_TYPE} -installer_ipaddr=${installer_ipaddr:-$INSTALLER_IP} - -if [ -z $installer_type ] || [ -z $installer_ipaddr ]; then - usage - exit 2 -fi - -ssh_options="-oUserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" - -#Start fetching compute ip -if [ "$installer_type" == "fuel" ]; then - verify_connectivity $installer_ipaddr - - env=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ipaddr} \ - 'fuel env'|grep operational|head -1|awk '{print $1}') &> /dev/null - if [ -z $env ]; then - error "No operational environment detected in Fuel" - fi - env_id="${FUEL_ENV:-$env}" - - # Check if compute is alive (online='True') - IPS=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ipaddr} \ - "fuel node --env ${env_id} | grep compute | grep 'True\| 1' | awk -F\| '{print \$5}' " | \ - sed 's/ //g') &> /dev/null - - -elif [ "$installer_type" == "apex" ]; then - echo "not implement now" - exit 1 - -elif [ "$installer_type" == "compass" ]; then - # need test - verify_connectivity $installer_ipaddr - IPS=$(sshpass -p'root' ssh 2>/dev/null $ssh_options root@${installer_ipaddr} \ - 'mysql -ucompass -pcompass -Dcompass -e"select * from cluster;"' \ - | awk -F"," '{for(i=1;i<NF;i++)if($i~/\"host[4-5]\"/) {print $(i+1);}}' \ - | grep -oP "\d+.\d+.\d+.\d+") - -elif [ "$installer_type" == "joid" ]; then - echo "not implement now" - exit 1 - -elif [ "$installer_type" == "foreman" ]; then - echo "not implement now" - exit 1 - -else - error "Installer $installer is not supported by this script" -fi - -if [ -z "$IPS" ]; then - error "The compute node $IPS are not up. Please check that the POD is correctly deployed." -else - echo "-------- all compute node ips: --------" - rm $dest_hostfile - touch $dest_hostfile - echo "[hosts]" >> $dest_hostfile - echo "$IPS" >> $dest_hostfile - cat $dest_hostfile -fi - -exit 0 diff --git a/qtip/util/env.py b/qtip/util/env.py deleted file mode 100644 index 9299f8c0..00000000 --- a/qtip/util/env.py +++ /dev/null @@ -1,213 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -from collections import defaultdict -import os -from os import path -import re -import socket -import sys -import time - -import paramiko - -from qtip.util.logger import QtipLogger - -logger = QtipLogger('env').get - -SCRIPT_DIR = path.join(path.dirname(__file__), path.pardir, 'scripts') -KEYNAME = 'QtipKey' -PRIVATE_KEY = '{0}/qtip/{1}'.format(os.environ['HOME'], KEYNAME) -PUBLIC_KEY = PRIVATE_KEY + '.pub' -HOST_FILE = '{0}/qtip/hosts'.format(os.environ['HOME']) - - -def all_files_exist(*files): - if len(files) == 0: - return False - flag = True - for f_item in files: - flag &= path.isfile(f_item) - logger.info("Is {0} existed: {1}".format(f_item, flag)) - return flag - - -def clean_file(*files): - if len(files) == 0: - logger.info('Nothing to clean') - return False - - def clean(f): - try: - if all_files_exist(f): - os.remove(f) - logger.info("Removed: {0}".format(f)) - else: - logger.info("Not exists: {0}".format(f)) - return True - except OSError as error: - logger.error("Not able to Remove: {0}".format(f), error) - return False - - results = map(clean, files) - return len(results) == len(files) and False not in results - - -class AnsibleEnvSetup(object): - def __init__(self): - self.keypair = defaultdict(str) - self.hostfile = None - self.host_ip_list = [] - - def setup(self, config={}): - try: - if 'hostfile' in config: - self.check_hostfile(config['hostfile']) - else: - self.generate_default_hostfile() - self.fetch_host_ip_from_hostfile() - if 'keypair' in config: - self.check_keypair(config['keypair']) - else: - self.generate_default_keypair() - self.pass_keypair_to_remote() - self.check_hosts_ssh_connectivity() - except Exception as error: - logger.info(error) - sys.exit(1) - - def check_keypair(self, keypair): - self.keypair = defaultdict(str) - if all_files_exist(keypair, '{0}.pub'.format(keypair)): - self.keypair['private'] = keypair - self.keypair['public'] = '{0}.pub'.format(keypair) - else: - raise RuntimeError("The keypairs you in the configuration file" - " is invalid or not existed.") - - def generate_default_keypair(self): - if not all_files_exist(PRIVATE_KEY, PUBLIC_KEY): - logger.info("Generate default keypair {0} under " - "{1}".format(KEYNAME, os.environ['HOME'])) - cmd = '''ssh-keygen -t rsa -N "" -f {0} -q -b 2048 - -C qtip@insecure'''.format(PRIVATE_KEY) - os.system(cmd) - self.keypair['private'] = PRIVATE_KEY - self.keypair['public'] = PUBLIC_KEY - - def pass_keypair_to_remote(self): - results = map(lambda ip: self._pass_keypair(ip, self.keypair['private']), - self.host_ip_list) - - if not (len(results) == len(self.host_ip_list) and False not in results): - raise RuntimeError("Failed on passing keypair to remote.") - - @staticmethod - def _pass_keypair(ip, private_key): - try: - os.system('ssh-keyscan %s >> /root/.ssh/known_hosts' % ip) - time.sleep(2) - ssh_cmd = '%s/qtip_creds.sh %s %s' % (SCRIPT_DIR, ip, private_key) - os.system(ssh_cmd) - logger.info('Pass keypair to remote hosts {0} successfully'.format(ip)) - return True - except Exception as error: - logger.error(error) - return False - - def check_hostfile(self, hostfile): - if all_files_exist(hostfile): - self.hostfile = hostfile - else: - raise RuntimeError( - "The hostfile {0} is invalid or not existed.".format(hostfile)) - - def generate_default_hostfile(self): - try: - # check whether the file is already existed - self.check_hostfile(HOST_FILE) - except Exception: - logger.info("Generate default hostfile {0} under " - "{1}".format(HOST_FILE, os.environ['HOME'])) - self._generate_hostfile_via_installer() - - def _generate_hostfile_via_installer(self): - self.hostfile = None - - installer_type = str(os.environ['INSTALLER_TYPE'].lower()) - installer_ip = str(os.environ['INSTALLER_IP']) - - if installer_type not in ["fuel"]: - raise ValueError("{0} is not supported".format(installer_type)) - if not installer_ip: - raise ValueError( - "The value of environment variable INSTALLER_IP is empty.") - - cmd = "bash %s/generate_host_file.sh -t %s -i %s -d %s" % \ - (SCRIPT_DIR, installer_type, installer_ip, HOST_FILE) - os.system(cmd) - - self.hostfile = HOST_FILE - - def fetch_host_ip_from_hostfile(self): - self.host_ip_list = [] - logger.info('Fetch host ips from hostfile...') - with open(self.hostfile, 'r') as f: - self.host_ip_list = re.findall('\d+.\d+.\d+.\d+', f.read()) - if self.host_ip_list: - logger.info("The remote compute nodes: {0}".format(self.host_ip_list)) - else: - raise ValueError("The hostfile doesn't include host ip addresses.") - - def check_hosts_ssh_connectivity(self): - results = map(lambda ip: self._ssh_is_ok(ip, self.keypair['private']), - self.host_ip_list) - if not (len(results) == len(self.host_ip_list) and False not in results): - raise RuntimeError("Failed on checking hosts ssh connectivity.") - - @staticmethod - def _ssh_is_ok(ip, private_key, attempts=100): - logger.info('Check hosts {0} ssh connectivity...'.format(ip)) - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(ip, key_filename=private_key) - - for attempt in range(attempts): - try: - stdin, stdout, stderr = ssh.exec_command('uname') - if not stderr.readlines(): - logger.info("{0}: SSH test successful.".format(ip)) - return True - except socket.error: - logger.debug("%s times ssh test......failed." % str(attempt + 1)) - if attempt == (attempts - 1): - return False - time.sleep(2) - return False - - def cleanup(self): - CI_DEBUG = os.getenv('CI_DEBUG') - - if CI_DEBUG is not None and CI_DEBUG.lower() == 'true': - logger.info("DEBUG Mode: please do cleanup by manual.") - else: - with open(self.keypair['public'], 'r') as f: - key = f.read().strip('\n').replace('/', '\/') - if key: - for ip in self.host_ip_list: - logger.info("Cleanup authorized_keys from {0}...".format(ip)) - cmd = '''bash {0}/cleanup_creds.sh {1} {2} "{3}"'''.format( - SCRIPT_DIR, ip, self.keypair['private'], key) - os.system(cmd) - else: - logger.error("Nothing in public key file.") - - logger.info("Cleanup hostfile and keypair.") - clean_file(self.hostfile, - self.keypair['private'], - self.keypair['public']) diff --git a/tests/unit/runner/runner_test.py b/tests/unit/runner/runner_test.py deleted file mode 100644 index b7da1611..00000000 --- a/tests/unit/runner/runner_test.py +++ /dev/null @@ -1,16 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from qtip.runner.base import BaseRunner - - -def test_constructor(): - runner = BaseRunner() - assert isinstance(runner, BaseRunner) diff --git a/tests/unit/util/env_test.py b/tests/unit/util/env_test.py deleted file mode 100644 index 793d1e4e..00000000 --- a/tests/unit/util/env_test.py +++ /dev/null @@ -1,307 +0,0 @@ -############################################################### -# Copyright (c) 2017 ZTE Corporation -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import time - -import pytest -import mock -from collections import defaultdict -import socket - -from qtip.util import env -from qtip.util.env import AnsibleEnvSetup - - -@pytest.fixture(scope='session') -def ansible_envsetup(): - return AnsibleEnvSetup() - - -@pytest.fixture() -def hostfile(tmpdir): - fake_hostfile = tmpdir.join('hosts') - fake_hostfile.write("[hosts]\n") - fake_hostfile.write("10.20.0.3") - return fake_hostfile - - -@pytest.fixture() -def private_key(tmpdir): - fake_private_key = tmpdir.join('QtipKey') - fake_private_key.write("fake keypair") - return fake_private_key - - -@pytest.fixture() -def public_key(tmpdir): - fake_public_key = tmpdir.join('QtipKey.pub') - fake_public_key.write("fake public key") - return fake_public_key - - -def test_all_files_exist(tmpdir): - exist_file = tmpdir.mkdir('qtip').join('hello.txt') - exist_file.write("hello") - non_exist_file = tmpdir.strpath + '/tmp.txt' - assert env.all_files_exist() is False - assert env.all_files_exist(str(exist_file)) - assert env.all_files_exist(non_exist_file) is False - assert env.all_files_exist(str(exist_file), non_exist_file) is False - - -def test_clean_file(tmpdir): - exist_file = tmpdir.mkdir('qtip').join('hello.txt') - exist_file.write("hello") - non_exist_file = tmpdir.strpath + '/tmp.txt' - - assert env.clean_file() is False - assert env.clean_file(str(exist_file)) - assert env.clean_file(non_exist_file) - - -def test_init(ansible_envsetup): - assert 'AnsibleEnvSetup' in str(type(ansible_envsetup)) - assert ansible_envsetup.keypair == defaultdict(str) - assert ansible_envsetup.hostfile is None - assert ansible_envsetup.host_ip_list == [] - - -def test_setup_exception(mocker, ansible_envsetup, hostfile): - with mock.patch.object(AnsibleEnvSetup, 'check_hostfile', side_effect=RuntimeError()): - mock_os = mocker.patch('sys.exit') - ansible_envsetup.setup({'hostfile': str(hostfile)}) - assert mock_os.call_count == 1 - - -# TODO(zhihui_wu) Need find a smart way to write this pytest -def test_setup(mocker, ansible_envsetup): - mock_check_hostfile = \ - mocker.patch.object(AnsibleEnvSetup, 'check_hostfile') - mock_generate_default_hostfile = \ - mocker.patch.object(AnsibleEnvSetup, 'generate_default_hostfile') - mock_fetch_ip = \ - mocker.patch.object(AnsibleEnvSetup, 'fetch_host_ip_from_hostfile') - mock_check_keypair = \ - mocker.patch.object(AnsibleEnvSetup, 'check_keypair') - mock_generate_default_keypair = \ - mocker.patch.object(AnsibleEnvSetup, 'generate_default_keypair') - mock_pass_keypair = \ - mocker.patch.object(AnsibleEnvSetup, 'pass_keypair_to_remote') - mock_check_ssh = \ - mocker.patch.object(AnsibleEnvSetup, 'check_hosts_ssh_connectivity') - - ansible_envsetup.setup({'keypair': str(private_key), - 'hostfile': str(hostfile)}) - mock_check_hostfile.assert_called_with(str(hostfile)) - mock_fetch_ip.assert_called_with() - mock_check_keypair.assert_called_with(str(private_key)) - mock_pass_keypair.assert_called_with() - mock_check_ssh.assert_called_with() - - ansible_envsetup.setup({'keypair': str(private_key)}) - mock_generate_default_hostfile.assert_called_with() - mock_fetch_ip.assert_called_with() - mock_check_keypair.assert_called_with(str(private_key)) - mock_pass_keypair.assert_called_with() - mock_check_ssh.assert_called_with() - - ansible_envsetup.setup({'hostfile': str(hostfile)}) - mock_check_hostfile.assert_called_with(str(hostfile)) - mock_fetch_ip.assert_called_with() - mock_generate_default_keypair.assert_called_with() - mock_pass_keypair.assert_called_with() - mock_check_ssh.assert_called_with() - - ansible_envsetup.setup() - mock_generate_default_hostfile.assert_called_with() - mock_fetch_ip.assert_called_with() - mock_generate_default_keypair.assert_called_with() - mock_pass_keypair.assert_called_with() - mock_check_ssh.assert_called_with() - - -def test_check_keypair(mocker, ansible_envsetup, private_key, public_key): - with mocker.patch.object(env, 'all_files_exist', return_value=True): - ansible_envsetup.check_keypair(str(private_key)) - assert ansible_envsetup.keypair['private'] == str(private_key) - assert ansible_envsetup.keypair['public'] == str(public_key) - - -def test_check_keypair_failed(mocker, ansible_envsetup): - mocker.patch.object(env, 'all_files_exist', return_value=False) - with pytest.raises(RuntimeError) as excinfo: - ansible_envsetup.check_keypair(str(private_key)) - assert 'The keypairs you in the configuration file ' \ - 'is invalid or not existed.' == str(excinfo.value) - assert ansible_envsetup.keypair['private'] == '' - assert ansible_envsetup.keypair['public'] == '' - - -@pytest.mark.parametrize("file_existence, expected", [ - (True, 0), - (False, 1) -]) -def test_generate_default_keypair(mocker, ansible_envsetup, file_existence, expected): - mock_os = mocker.patch('os.system') - mocker.patch.object(env, 'all_files_exist', return_value=file_existence) - ansible_envsetup.generate_default_keypair() - assert mock_os.call_count == expected - assert ansible_envsetup.keypair['private'] == env.PRIVATE_KEY - assert ansible_envsetup.keypair['public'] == env.PUBLIC_KEY - - -@pytest.mark.parametrize("ips, expected", [ - (['10.20.0.3'], 1), - (['10.20.0.3', '10.20.0.4'], 2) -]) -def test_pass_keypair_to_remote_successful(mocker, ansible_envsetup, ips, expected): - ansible_envsetup.host_ip_list = ips - mock_pass_keypair = \ - mocker.patch.object(AnsibleEnvSetup, '_pass_keypair', return_value=True) - ansible_envsetup.pass_keypair_to_remote() - assert mock_pass_keypair.call_count == expected - - -def test_pass_keypair_to_remote_failed(mocker, ansible_envsetup): - ansible_envsetup.host_ip_list = ['10.20.0.3'] - mocker.patch.object(AnsibleEnvSetup, '_pass_keypair', return_value=False) - with pytest.raises(RuntimeError) as excinfo: - ansible_envsetup.pass_keypair_to_remote() - assert "Failed on passing keypair to remote." in str(excinfo.value) - - -def test_pass_keypair(monkeypatch, mocker, ansible_envsetup): - monkeypatch.setattr(time, 'sleep', lambda s: None) - mock_os = mocker.patch('os.system') - ansible_envsetup._pass_keypair('10.20.0.3', str(private_key)) - assert mock_os.call_count == 2 - - -def test_pass_keypair_exception(ansible_envsetup): - with mock.patch('os.system', side_effect=Exception()) as mock_os: - result = ansible_envsetup._pass_keypair('10.20.0.3', str(private_key)) - assert result is False - assert mock_os.call_count == 1 - - -def test_check_hostfile(mocker, ansible_envsetup, hostfile): - ansible_envsetup.check_hostfile(str(hostfile)) - assert ansible_envsetup.hostfile == str(hostfile) - - with pytest.raises(RuntimeError) as excinfo: - mocker.patch.object(env, 'all_files_exist', return_value=False) - ansible_envsetup.check_hostfile(str(hostfile)) - assert str(excinfo.value) == 'The hostfile {0} is invalid or not ' \ - 'existed.'.format(str(hostfile)) - - -def test_default_hostfile_non_existed(mocker, ansible_envsetup): - with mocker.patch.object(env, 'all_files_exist', return_value=False): - mock_generate_hostfile_via_installer = \ - mocker.patch.object(AnsibleEnvSetup, - '_generate_hostfile_via_installer') - ansible_envsetup.generate_default_hostfile() - mock_generate_hostfile_via_installer.assert_called_once_with() - - -def test_default_hostfile_existed(mocker, ansible_envsetup): - with mocker.patch.object(env, 'all_files_exist', return_value=True): - mock_generate_hostfile_via_installer = \ - mocker.patch.object(AnsibleEnvSetup, - '_generate_hostfile_via_installer') - ansible_envsetup.generate_default_hostfile() - mock_generate_hostfile_via_installer.assert_not_called() - - -@pytest.mark.parametrize("test_input, expected", [ - (({}, KeyError), 'INSTALLER_TYPE'), - (({'INSTALLER_TYPE': 'fuel'}, KeyError), 'INSTALLER_IP'), - (({'INSTALLER_TYPE': 'fuel_1', 'INSTALLER_IP': '10.20.0.2'}, ValueError), - 'fuel_1 is not supported'), - (({'INSTALLER_TYPE': 'fuel', 'INSTALLER_IP': ''}, ValueError), - 'The value of environment variable INSTALLER_IP is empty') -]) -def test_generate_hostfile_via_installer_exception(monkeypatch, ansible_envsetup, test_input, expected): - if test_input[0]: - for key in test_input[0]: - monkeypatch.setenv(key, test_input[0][key]) - - with pytest.raises(test_input[1]) as excinfo: - ansible_envsetup._generate_hostfile_via_installer() - assert expected in str(excinfo.value) - - -def test_generate_hostfile_via_installer(monkeypatch, mocker, ansible_envsetup): - monkeypatch.setenv('INSTALLER_TYPE', 'fuel') - monkeypatch.setenv('INSTALLER_IP', '10.20.0.2') - mock_os = mocker.patch('os.system') - ansible_envsetup._generate_hostfile_via_installer() - assert mock_os.call_count == 1 - assert ansible_envsetup.hostfile == env.HOST_FILE - - -def test_fetch_host_ip_from_hostfile(ansible_envsetup, hostfile): - ansible_envsetup.hostfile = str(hostfile) - ansible_envsetup.fetch_host_ip_from_hostfile() - assert ansible_envsetup.host_ip_list == ['10.20.0.3'] - - -def test_fetch_host_ip_from_empty_hostfile(ansible_envsetup, tmpdir): - empty_hostfile = tmpdir.join('empty_hostfile') - empty_hostfile.write("") - ansible_envsetup.hostfile = str(empty_hostfile) - with pytest.raises(ValueError) as excinfo: - ansible_envsetup.fetch_host_ip_from_hostfile() - assert str(excinfo.value) == "The hostfile doesn't include host ip addresses." - - -@pytest.mark.parametrize("ips, expected", [ - (['10.20.0.3'], 1), - (['10.20.0.3', '10.20.0.4'], 2) -]) -def test_check_hosts_ssh_connectivity(mocker, ansible_envsetup, ips, expected): - ansible_envsetup.host_ip_list = ips - mock_ssh_is_ok = \ - mocker.patch.object(AnsibleEnvSetup, '_ssh_is_ok', return_value=True) - ansible_envsetup.check_hosts_ssh_connectivity() - assert mock_ssh_is_ok.call_count == expected - - -def test_check_hosts_ssh_connectivity_failed(mocker, ansible_envsetup): - ansible_envsetup.host_ip_list = ['10.20.0.3'] - mocker.patch.object(AnsibleEnvSetup, '_ssh_is_ok', return_value=False) - with pytest.raises(RuntimeError) as excinfo: - ansible_envsetup.check_hosts_ssh_connectivity() - assert "Failed on checking hosts ssh connectivity." == str(excinfo.value) - - -@pytest.mark.parametrize("stderrinfo, expected", [ - ('', True), - ('sorry', False) -]) -def test_ssh_is_ok(mocker, ansible_envsetup, private_key, stderrinfo, expected): - stderr = mock.MagicMock() - stderr.readlines.return_value = stderrinfo - mock_sshclient = mocker.patch('paramiko.SSHClient') - test_ssh_client = mock_sshclient.return_value - test_ssh_client.exec_command.return_value = ('', '', stderr) - result = ansible_envsetup._ssh_is_ok('10.20.0.3', str(private_key)) - assert result == expected - test_ssh_client.connect.assert_called_once_with( - '10.20.0.3', key_filename=str(private_key)) - test_ssh_client.exec_command.assert_called_with('uname') - - -def test_ssh_exception(monkeypatch, mocker, ansible_envsetup): - monkeypatch.setattr(time, 'sleep', lambda s: None) - mock_sshclient = mocker.patch('paramiko.SSHClient') - test_ssh_client = mock_sshclient.return_value - test_ssh_client.exec_command.side_effect = socket.error() - result = ansible_envsetup._ssh_is_ok('10.20.0.3', str(private_key), attempts=1) - assert result is False |