From 740cbf36beff04167473cc293a72b0442b74e701 Mon Sep 17 00:00:00 2001 From: zhihui wu Date: Tue, 16 May 2017 13:56:28 +0800 Subject: delete unuse code in directory /qtip Change-Id: I6732c7cf572d5b744c3a176ce266b9739e9233fd Signed-off-by: zhihui wu --- qtip/collector/calculator.py | 38 ----- qtip/collector/parser/grep.py | 66 -------- qtip/collector/parser/regex.yaml | 85 ---------- qtip/driver/__init__.py | 0 qtip/driver/ansible_api.py | 58 ------- qtip/driver/ansible_driver.py | 102 ------------ qtip/driver/base.py | 20 --- qtip/driver/playbook/dpi/clean.yaml | 23 --- qtip/driver/playbook/dpi/dpi_average.sh | 10 -- qtip/driver/playbook/dpi/run.yaml | 42 ----- qtip/driver/playbook/dpi/setup.yaml | 93 ----------- qtip/driver/playbook/inxi.yaml | 25 --- qtip/driver/playbook/openssl/clean.yaml | 23 --- qtip/driver/playbook/openssl/run.yaml | 45 ------ qtip/driver/playbook/openssl/setup.yaml | 87 ----------- qtip/driver/playbook/prepare_env.yaml | 54 ------- qtip/driver/playbook/ramspeed/clean.yaml | 23 --- qtip/driver/playbook/ramspeed/run.yaml | 40 ----- qtip/driver/playbook/ramspeed/setup.yaml | 70 --------- qtip/driver/playbook/unixbench/clean.yaml | 25 --- qtip/driver/playbook/unixbench/dhrystone.yaml | 41 ----- qtip/driver/playbook/unixbench/run.yaml | 23 --- qtip/driver/playbook/unixbench/setup.yaml | 65 -------- qtip/driver/playbook/unixbench/whetstone.yaml | 41 ----- qtip/runner/runner.py | 107 ------------- qtip/scripts/cleanup_creds.sh | 20 --- qtip/scripts/generate_host_file.sh | 116 -------------- qtip/util/env.py | 213 -------------------------- 28 files changed, 1555 deletions(-) delete mode 100644 qtip/collector/calculator.py delete mode 100644 qtip/collector/parser/regex.yaml delete mode 100644 qtip/driver/__init__.py delete mode 100644 qtip/driver/ansible_api.py delete mode 100644 qtip/driver/ansible_driver.py delete mode 100644 qtip/driver/base.py delete mode 100644 qtip/driver/playbook/dpi/clean.yaml delete mode 100644 qtip/driver/playbook/dpi/dpi_average.sh delete mode 100644 qtip/driver/playbook/dpi/run.yaml delete mode 100644 qtip/driver/playbook/dpi/setup.yaml delete mode 100644 qtip/driver/playbook/inxi.yaml delete mode 100644 qtip/driver/playbook/openssl/clean.yaml delete mode 100644 qtip/driver/playbook/openssl/run.yaml delete mode 100644 qtip/driver/playbook/openssl/setup.yaml delete mode 100644 qtip/driver/playbook/prepare_env.yaml delete mode 100644 qtip/driver/playbook/ramspeed/clean.yaml delete mode 100644 qtip/driver/playbook/ramspeed/run.yaml delete mode 100644 qtip/driver/playbook/ramspeed/setup.yaml delete mode 100644 qtip/driver/playbook/unixbench/clean.yaml delete mode 100644 qtip/driver/playbook/unixbench/dhrystone.yaml delete mode 100644 qtip/driver/playbook/unixbench/run.yaml delete mode 100644 qtip/driver/playbook/unixbench/setup.yaml delete mode 100644 qtip/driver/playbook/unixbench/whetstone.yaml delete mode 100644 qtip/runner/runner.py delete mode 100755 qtip/scripts/cleanup_creds.sh delete mode 100755 qtip/scripts/generate_host_file.sh delete mode 100644 qtip/util/env.py (limited to 'qtip') diff --git a/qtip/collector/calculator.py b/qtip/collector/calculator.py deleted file mode 100644 index c3d961b3..00000000 --- a/qtip/collector/calculator.py +++ /dev/null @@ -1,38 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from operator import add - -from qtip.util.logger import QtipLogger - -logger = QtipLogger('calculator').get - - -def dpi_calculator(samples): - try: - float_pps = map(lambda x: float(x), samples['pps']) - float_bps = map(lambda x: float(x), samples['bps']) - sum_dpi_pps = reduce(add, - map(lambda x: x / 1000 if x > 100 else x, float_pps)) - sum_dpi_bps = reduce(add, - map(lambda x: x / 1000 if x > 100 else x, float_bps)) - - return {'pps': round(sum_dpi_pps / 10, 3), 'bps': round(sum_dpi_bps / 10, 3)} - except Exception as error: - logger.error(error) - return {'pps': None, 'bps': None} - - -def calculate_cpu_usage(cpu_idle): - try: - cpu_usage = round((100.0 - float(cpu_idle)), 3) - return '{0}%'.format(str(cpu_usage)) - except Exception, error: - logger.error(error) - return None diff --git a/qtip/collector/parser/grep.py b/qtip/collector/parser/grep.py index d3a8210a..66b51b1e 100644 --- a/qtip/collector/parser/grep.py +++ b/qtip/collector/parser/grep.py @@ -7,21 +7,15 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from collections import defaultdict -from os import path import re -import yaml from qtip.base import BaseActor from qtip.base.constant import BaseProp -from qtip.collector import calculator from qtip.util.logger import QtipLogger logger = QtipLogger('grep').get -REGEX_FILE = path.join(path.dirname(__file__), 'regex.yaml') - class GrepProp(BaseProp): FILENAME = 'filename' @@ -40,63 +34,3 @@ def grep_in_file(filename, regex): with open(filename, 'r') as f: return filter(lambda x: x is not None, re.finditer(regex, f.read(), re.MULTILINE)) - - -def _parse_logfile(config, paths): - captured = {} - for regex_rules_by_file in config: - filename = \ - '{0}/{1}'.format(paths, regex_rules_by_file[GrepProp.FILENAME]) - for regex in regex_rules_by_file['grep']: - matches = grep_in_file(filename, regex) - if len(matches) > 1: - temp_dict = defaultdict(list) - for item in [match.groupdict() for match in matches]: - for key in item: - temp_dict[key].append(item[key]) - captured.update(temp_dict) - elif len(matches) == 1: - captured.update(matches[0].groupdict()) - else: - logger.error("Nothing is matched from {0}".format(filename)) - return captured - - -# TODO: Hardcord in Danube, it will be removed in the future. -def parse_sysinfo(config, result_dir): - sysinfo = _parse_logfile(config, result_dir) - if "cpu_idle" in sysinfo: - sysinfo['cpu_usage'] = \ - calculator.calculate_cpu_usage(sysinfo['cpu_idle']) - sysinfo.pop('cpu_idle') - return sysinfo - - -# TODO: Hardcord in Danube, it will be removed in the future. -def parse_test_result(benchmark, config, result_dir): - test_result = _parse_logfile(config, result_dir) - if benchmark == 'dpi': - return calculator.dpi_calculator(test_result) - if benchmark == 'dhrystone' or benchmark == 'whetstone': - return {'total_cpus': test_result['total_cpus'], - 'single_cpu': {'num': test_result['single_cpu'], - 'score': test_result['score'][0]}, - 'multi_cpus': {'num': test_result['multi_cpus'], - 'score': test_result['score'][1]}} - return test_result - - -# TODO: Hardcord in Danube, it will be removed in the future. -def parse_benchmark_result(result_dir): - regex_config = yaml.safe_load(file(REGEX_FILE)) - benchmark = result_dir.split('/')[-1] - result = {'name': benchmark} - - test_result = \ - parse_test_result(benchmark, regex_config[benchmark], result_dir) - result['results'] = test_result.copy() - - sysinfo = parse_sysinfo(regex_config['sysinfo'], result_dir) - result['sysinfo'] = sysinfo.copy() - - return result diff --git a/qtip/collector/parser/regex.yaml b/qtip/collector/parser/regex.yaml deleted file mode 100644 index e1512855..00000000 --- a/qtip/collector/parser/regex.yaml +++ /dev/null @@ -1,85 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -dhrystone: - - filename: dhrystone - grep: - - '^(?P\d+)\sCPUs in system; running 1 parallel copy of tests$' - - '.+\srunning (?P\d+) parallel copy of tests$' - - '.+\srunning (?P\d+) parallel copies of tests$' - - '^System Benchmarks Index Score \(Partial Only\)\s+(?P\d+\.\d)$' -whetstone: - - filename: whetstone - grep: - - '^(?P\d+)\sCPUs in system; running 1 parallel copy of tests$' - - '.+\srunning (?P\d+) parallel copy of tests$' - - '.+\srunning (?P\d+) parallel copies of tests$' - - '^System Benchmarks Index Score \(Partial Only\)\s+(?P\d+\.\d)$' -dpi: - - filename: dpi_dump.txt - grep: - - |- - ^\s+nDPI throughput:.+?(?P\d+.\d+)\s.+\spps.+ - ?(?P\d+.\d+)\s.+\/sec -ramspeed: - - filename: Intmem - grep: - - '^INTEGER\s+BatchRun\s+Copy:\s+?(?P\d+\.\d+)\sMB/s$' - - '^INTEGER\s+BatchRun\s+Scale:\s+?(?P\d+\.\d+)\sMB/s$' - - '^INTEGER\s+BatchRun\s+Add:\s+?(?P\d+\.\d+)\sMB/s$' - - '^INTEGER\s+BatchRun\s+Triad:\s+?(?P\d+\.\d+)\sMB/s$' - - '^INTEGER\s+BatchRun\s+AVERAGE:\s+?(?P\d+\.\d+)\sMB/s$' - - filename: Floatmem - grep: - - '^FL-POINT\s+BatchRun\s+Copy:\s+?(?P\d+\.\d+)\sMB/s$' - - '^FL-POINT\s+BatchRun\s+Scale:\s+?(?P\d+\.\d+)\sMB/s$' - - '^FL-POINT\s+BatchRun\s+Add:\s+?(?P\d+\.\d+)\sMB/s$' - - '^FL-POINT\s+BatchRun\s+Triad:\s+?(?P\d+\.\d+)\sMB/s$' - - '^FL-POINT\s+BatchRun\s+AVERAGE:\s+?(?P\d+\.\d+)\sMB/s$' -ssl: - - filename: RSA_dump - grep: - - |- - ^rsa\s+512\sbits\s.+\s+ - ?(?P\d+\.\d)\s+ - ?(?P\d+\.\d)$ - - |- - ^rsa\s+1024\sbits\s.+\s+ - ?(?P\d+\.\d)\s+ - ?(?P\d+\.\d)$ - - |- - ^rsa\s+2048\sbits\s.+\s+ - ?(?P\d+\.\d)\s+ - ?(?P\d+\.\d)$ - - |- - ^rsa\s+4096\sbits\s.+\s+ - ?(?P\d+\.\d)\s+ - ?(?P\d+\.\d)$ - - filename: AES-128-CBC_dump - grep: - - |- - ^aes-128-cbc\s+ - ?(?P\d+\.\w+)\s+ - ?(?P\d+\.\w+)\s+ - ?(?P\d+\.\w+)\s+ - ?(?P\d+\.\w+)\s+ - ?(?P\d+\.\w+)$ -sysinfo: - - filename: top.log - grep: - - 'Cpu\(s\):.+?(?P\d+\.\d)\sid' - - filename: inxi.log - grep: - - '.+\s+Host:\s+(?P.+)\sKernel' - - '.+\sMemory:\s+(?P.+MB)\s' - - '^CPU\(s\):\s+(?P.+)' - - '.+\sDistro:\s+(?P.+)' - - '.+\sKernel:\s+(?P.+)\sConsole' - - '.+\s+HDD Total Size:\s+(?P.+)\s' - - '.+\sproduct:\s+(?P.+)\sv' \ No newline at end of file diff --git a/qtip/driver/__init__.py b/qtip/driver/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/qtip/driver/ansible_api.py b/qtip/driver/ansible_api.py deleted file mode 100644 index 5c5baffc..00000000 --- a/qtip/driver/ansible_api.py +++ /dev/null @@ -1,58 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from collections import namedtuple - -from ansible.executor.playbook_executor import PlaybookExecutor -from ansible.inventory import Inventory -from ansible.parsing.dataloader import DataLoader -from ansible.vars import VariableManager - - -class AnsibleApi(object): - - def __init__(self): - self.variable_manager = VariableManager() - self.loader = DataLoader() - self.passwords = {} - self.playbook_executor = None - - def execute_playbook(self, playbook_path, hosts_file=None, - key_file=None, extra_vars=None): - inventory = Inventory(loader=self.loader, - variable_manager=self.variable_manager, - host_list=hosts_file) - Options = namedtuple('Options', - ['listtags', 'listtasks', 'listhosts', 'syntax', - 'connection', 'module_path', 'forks', 'remote_user', - 'private_key_file', 'ssh_common_args', 'ssh_extra_args', - 'sftp_extra_args', 'scp_extra_args', 'become', - 'become_method', 'become_user', 'verbosity', 'check']) - options = Options(listtags=False, listtasks=False, listhosts=False, - syntax=False, connection='ssh', module_path=None, - forks=100, remote_user='root', private_key_file=key_file, - ssh_common_args=None, ssh_extra_args=None, sftp_extra_args=None, - scp_extra_args=None, become=None, become_method=None, - become_user='root', verbosity=None, check=False) - self.variable_manager.extra_vars = extra_vars - - self.playbook_executor = PlaybookExecutor(playbooks=[playbook_path], - inventory=inventory, - variable_manager=self.variable_manager, - loader=self.loader, - options=options, - passwords=self.passwords) - return self.playbook_executor.run() - - def get_detail_playbook_stats(self): - if self.playbook_executor: - stats = self.playbook_executor._tqm._stats - return map(lambda x: (x, stats.summarize(x)), stats.processed.keys()) - else: - return None diff --git a/qtip/driver/ansible_driver.py b/qtip/driver/ansible_driver.py deleted file mode 100644 index 34ef46ed..00000000 --- a/qtip/driver/ansible_driver.py +++ /dev/null @@ -1,102 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from collections import defaultdict -from os import path -from operator import add - -from qtip.driver.ansible_api import AnsibleApi -from qtip.util.env import AnsibleEnvSetup -from qtip.util.logger import QtipLogger - -logger = QtipLogger('ansible_driver').get -PLAYBOOK_DIR = path.join(path.dirname(__file__), 'playbook') - - -class AnsibleDriver(object): - """driver for running performance tests with Ansible""" - - def __init__(self, config={}): - self.config = config - self.env = AnsibleEnvSetup() - self.env_setup_flag = False - - @staticmethod - def merge_two_dicts(x, y): - ''' - It is from http://stackoverflow.com/questions/38987/ - how-can-i-merge-two-python-dictionaries-in-a-single-expression - ''' - z = x.copy() - z.update(y) - return z - - def pre_run(self): - if self.env_setup_flag: - logger.info("Already setup environment......") - else: - logger.info("Starting to setup test environment...") - self.env.setup(self.config) - self.env_setup_flag = True - logger.info("Setup test enviroment, Done!") - - def cleanup(self): - self.env.cleanup() - - def run(self, metric_list, **kwargs): - if 'args' in self.config: - extra_vars = self.merge_two_dicts(kwargs, self.config['args']) - else: - extra_vars = kwargs - logger.info("extra_var: {0}".format(extra_vars)) - - tool_to_metrics = defaultdict(list) - for metric in metric_list: - if metric == 'dhrystone' or metric == 'whetstone': - tool_to_metrics['unixbench'].append(metric) - extra_vars[metric] = True - elif metric == 'ssl': - tool_to_metrics['openssl'].append(metric) - else: - tool_to_metrics[metric].append(metric) - - result_list = map(lambda tool: self._run_metric(tool, - tool_to_metrics[tool], - extra_vars), - tool_to_metrics) - return False not in result_list - - def _run_metric(self, tool, metrics, extra_vars): - logger.info('Using {0} to measure metrics {1}'.format(tool, metrics)) - - setup_pbook = "{0}/{1}/setup.yaml".format(PLAYBOOK_DIR, tool) - run_pbook = "{0}/{1}/run.yaml".format(PLAYBOOK_DIR, tool) - clean_pbook = "{0}/{1}/clean.yaml".format(PLAYBOOK_DIR, tool) - - if self._run_ansible_playbook(setup_pbook, extra_vars): - self._run_ansible_playbook(run_pbook, extra_vars) - else: - logger.error("{0} is failed.".format(setup_pbook)) - - return self._run_ansible_playbook(clean_pbook, extra_vars) - - def _run_ansible_playbook(self, pbook, extra_vars): - ansible_api = AnsibleApi() - logger.debug("Run {0} with extra_vars: {1}".format(pbook, extra_vars)) - ansible_api.execute_playbook(pbook, self.env.hostfile, - self.env.keypair['private'], extra_vars) - playbook_stats = ansible_api.get_detail_playbook_stats() - logger.debug("playbook_stat: {0}".format(playbook_stats)) - return self.is_pass(playbook_stats) - - @staticmethod - def is_pass(stats): - return 0 == reduce(add, - map(lambda x: x[1]['failures'] + x[1]['unreachable'], - stats)) diff --git a/qtip/driver/base.py b/qtip/driver/base.py deleted file mode 100644 index 6f5cab3c..00000000 --- a/qtip/driver/base.py +++ /dev/null @@ -1,20 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -class BaseDriver(object): - """performance testing tool driver""" - def pre_run(self): - pass - - def run(self): - pass - - def post_run(self): - pass diff --git a/qtip/driver/playbook/dpi/clean.yaml b/qtip/driver/playbook/dpi/clean.yaml deleted file mode 100644 index 0b9f9291..00000000 --- a/qtip/driver/playbook/dpi/clean.yaml +++ /dev/null @@ -1,23 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Cleaning tempD - file: - path: '{{ ansible_env.HOME }}/tempD' - state: absent - - - name: Cleaning qtip_result - file: - path: '{{ ansible_env.HOME }}/qtip_result' - state: absent diff --git a/qtip/driver/playbook/dpi/dpi_average.sh b/qtip/driver/playbook/dpi/dpi_average.sh deleted file mode 100644 index 6f038053..00000000 --- a/qtip/driver/playbook/dpi/dpi_average.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -COUNTER=0 -WDIR=$PWD -while [ $COUNTER -lt 10 ]; do - echo $WDIR - $( ./ndpiReader -i test.pcap >> $WDIR/dpi_dump.txt ) - let COUNTER=COUNTER+1 - echo "Run number: $COUNTER" -done \ No newline at end of file diff --git a/qtip/driver/playbook/dpi/run.yaml b/qtip/driver/playbook/dpi/run.yaml deleted file mode 100644 index f4c8c457..00000000 --- a/qtip/driver/playbook/dpi/run.yaml +++ /dev/null @@ -1,42 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Make some directories needed - file: - path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dpi/' - state: directory - - - include: ../inxi.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dpi/' - - - include: ../top.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dpi/' - - - name: Run nDPI benchmark - shell: ./dpi_average.sh - args: - chdir: '{{ ansible_env.HOME }}/tempD/nDPI/example/' - - - name: Copying result and system info to qtip result directory - command: cp $HOME/tempD/nDPI/example/dpi_dump.txt ./ - args: - chdir: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dpi/' - - - name: Fetch result files to local manchine - synchronize: - mode: pull - src: '{{ ansible_env.HOME }}/qtip_result/' - dest: '{{ result_dir }}/' diff --git a/qtip/driver/playbook/dpi/setup.yaml b/qtip/driver/playbook/dpi/setup.yaml deleted file mode 100644 index 76e62e6d..00000000 --- a/qtip/driver/playbook/dpi/setup.yaml +++ /dev/null @@ -1,93 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: localhost - connection: local - gather_facts: no - - tasks: - - name: Making Dpi directory - file: - path: '{{ result_dir }}/' - state: directory - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Cleaning tempD directory - file: - path: '{{ ansible_env.HOME }}/tempD' - state: absent - - - name: Cleaning qtip_result directory - file: - path: '{{ ansible_env.HOME }}/qtip_result' - state: absent - - - include: ../prepare_env.yaml - - - name: Installing nDPI dependencies if CentOS - yum: - name: '{{ item }}' - state: present - when: ansible_os_family == "RedHat" - with_items: - - git - - gcc - - patch - - perl-Time-HiRes - - autofconf - - automake - - libpcap-devel libtool - - - name: Installing nDPI dependencies if Ubuntu - apt: - name: '{{ item }}' - state: present - when: ansible_os_family == "Debian" - with_items: - - git - - gcc - - patch - - autoconf - - automake - - libpcap-dev - - libtool - - - name: Making nDPI temporary directory - file: - path: '{{ ansible_env.HOME }}/tempD' - state: directory - - - name: Clone nDPI - git: - repo: https://github.com/ntop/nDPI.git - dest: '{{ ansible_env.HOME }}/tempD/nDPI' - - - name: Run autogen && configure Dpi && make Dpi - command: '{{ item }}' - with_items: - - ./autogen.sh - - ./configure - - make - args: - chdir: '{{ ansible_env.HOME }}/tempD/nDPI/' - - - name: Fetching Test_pcap file - get_url: - url: http://build.opnfv.org/artifacts.opnfv.org/qtip/utilities/test.pcap - dest: '{{ ansible_env.HOME }}/tempD/nDPI/example/test.pcap' - - - name: Fetch Averaging script - copy: - src: ./dpi_average.sh - dest: '{{ ansible_env.HOME }}/tempD/nDPI/example' - mode: 777 \ No newline at end of file diff --git a/qtip/driver/playbook/inxi.yaml b/qtip/driver/playbook/inxi.yaml deleted file mode 100644 index 2a4d9b3f..00000000 --- a/qtip/driver/playbook/inxi.yaml +++ /dev/null @@ -1,25 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -- name: Install Inxi when CentOS - yum: - name: inxi - state: present - when: ansible_os_family == "RedHat" - -- name: Install Inxi when Ubuntu - apt: - name: inxi - state: present - update_cache: yes - when: ansible_os_family == "Debian" - -- name: Run inxi - shell: inxi -b -c0 -n > inxi.log - args: - chdir: '{{ dest_path }}' diff --git a/qtip/driver/playbook/openssl/clean.yaml b/qtip/driver/playbook/openssl/clean.yaml deleted file mode 100644 index 0139ba54..00000000 --- a/qtip/driver/playbook/openssl/clean.yaml +++ /dev/null @@ -1,23 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Cleaning Open_SSL - file: - path: '{{ ansible_env.HOME }}/Open_SSL' - state: absent - - - name: Cleaning qtip_result - file: - path: '{{ ansible_env.HOME }}/qtip_result' - state: absent diff --git a/qtip/driver/playbook/openssl/run.yaml b/qtip/driver/playbook/openssl/run.yaml deleted file mode 100644 index db923fed..00000000 --- a/qtip/driver/playbook/openssl/run.yaml +++ /dev/null @@ -1,45 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Make some directories needed - file: - path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ssl/' - state: directory - - - include: ../inxi.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ssl/' - - - include: ../top.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ssl/' - - - name: Benchmarking RSA signatures and AES-128-cbc cipher encryption throughput - shell: '{{ item }}' - with_items: - - ./openssl speed rsa >> RSA_dump - - ./openssl speed -evp aes-128-cbc >> AES-128-CBC_dump - args: - chdir: '{{ ansible_env.HOME }}/Open_SSL/openssl-1.0.2f/apps' - - - name: Copying result to qtip result directory - shell: cp ~/Open_SSL/openssl-1.0.2f/apps/*_dump ./ - args: - chdir: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ssl/' - - - name: Fetch result files to local manchine - synchronize: - mode: pull - src: '{{ ansible_env.HOME }}/qtip_result/' - dest: '{{ result_dir }}/' diff --git a/qtip/driver/playbook/openssl/setup.yaml b/qtip/driver/playbook/openssl/setup.yaml deleted file mode 100644 index 3a6f385a..00000000 --- a/qtip/driver/playbook/openssl/setup.yaml +++ /dev/null @@ -1,87 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: localhost - connection: local - gather_facts: no - - tasks: - - name: Making ssl directory - file: - path: '{{ result_dir }}/' - state: directory - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Cleaning Open_SSL directory - file: - path: '{{ ansible_env.HOME }}/Open_SSL' - state: absent - - - name: Cleaning qtip_result directory - file: - path: '{{ ansible_env.HOME }}/qtip_result' - state: absent - - - include: ../prepare_env.yaml - - - name: Installing UnixBench dependencies if CentOS - yum: - name: '{{ item }}' - state: present - when: ansible_os_family == "RedHat" - with_items: - - git - - gcc - - patch - - perl-Time-HiRes - - wget - - autofconf - - automake - - libpcap-devel - - libtool - - - name: Installing UnixBench dependencies if Ubuntu - apt: - name: '{{ item }}' - state: present - when: ansible_os_family == "Debian" - with_items: - - git - - gcc - - patch - - perl - - wget - - autoconf - - automake - - libpcap-dev - - libtool - - - name: Make Open_SSL directory - file: - path: '{{ ansible_env.HOME }}/Open_SSL' - state: directory - - - name: Untar OpenSSL - unarchive: - src: http://artifacts.opnfv.org/qtip/utilities/openssl-1.0.2f.tar.gz - dest: '{{ ansible_env.HOME }}/Open_SSL/' - remote_src: True - - - name: Configure && Make && Install OpenSSL - shell: "{{ item }}" - with_items: - - ./config - - make - - make install - args: - chdir: '{{ ansible_env.HOME }}/Open_SSL/openssl-1.0.2f' \ No newline at end of file diff --git a/qtip/driver/playbook/prepare_env.yaml b/qtip/driver/playbook/prepare_env.yaml deleted file mode 100644 index 1ec71520..00000000 --- a/qtip/driver/playbook/prepare_env.yaml +++ /dev/null @@ -1,54 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -- name: Epel Release install when CentOS - yum: - name: epel-release - state: present - when: ansible_os_family == "RedHat" - -- name: Software Properties Common - apt: - name: software-properties-common - state: present - when: ansible_os_family == "Debian" - -- name: Adding ubuntu backport main repo - apt_repository: - repo: deb http://archive.ubuntu.com/ubuntu/ {{ansible_distribution_release}}-backports main restricted universe multiverse - state: present - when: ansible_os_family == "Debian" - -- name: Adding ubuntu main repo - apt_repository: - repo: deb http://archive.ubuntu.com/ubuntu/ {{ansible_distribution_release}} main restricted universe multiverse - when: ansible_os_family == "Debian" - -- name: Install ansible copy dependencies if remote host has selinux enabled - yum: - name: libselinux-python - state: present - when: ansible_os_family == "RedHat" - -- name: Install ansiblle copy dependencies if remote host has selinux enaled - apt: - name: python-selinux - state: present - when: ansible_os_family == "Debian" - -- name: Install rsync when CentOS - yum: - name: rsync - state: present - when: ansible_os_family == "RedHat" - -- name: Install rsync when Ubuntu - apt: - name: rsync - state: present - when: ansible_os_family == "Debian" diff --git a/qtip/driver/playbook/ramspeed/clean.yaml b/qtip/driver/playbook/ramspeed/clean.yaml deleted file mode 100644 index f0188159..00000000 --- a/qtip/driver/playbook/ramspeed/clean.yaml +++ /dev/null @@ -1,23 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Cleaning ramspeed - file: - path: '{{ ansible_env.HOME }}/ramspeed' - state: absent - - - name: Cleaning qtip_result - file: - path: '{{ ansible_env.HOME }}/qtip_result' - state: absent diff --git a/qtip/driver/playbook/ramspeed/run.yaml b/qtip/driver/playbook/ramspeed/run.yaml deleted file mode 100644 index 496cd5db..00000000 --- a/qtip/driver/playbook/ramspeed/run.yaml +++ /dev/null @@ -1,40 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Make some directories needed - file: - path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ramspeed/' - state: directory - - - include: ../inxi.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ramspeed/' - - - include: ../top.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ramspeed/' - - - name: Benchmarking IntMem Bandwidth and FloatMem Bandwidth - shell: '{{ item }}' - with_items: - - ~/ramspeed/ramsmp-3.5.0/ramsmp -b 3 -l 5 -p 1 >> Intmem - - ~/ramspeed/ramsmp-3.5.0/ramsmp -b 6 -l 5 -p 1 >> Floatmem - args: - chdir: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/ramspeed/' - - - name: Fetch result files to local manchine - synchronize: - mode: pull - src: '{{ ansible_env.HOME }}/qtip_result/' - dest: '{{ result_dir }}/' diff --git a/qtip/driver/playbook/ramspeed/setup.yaml b/qtip/driver/playbook/ramspeed/setup.yaml deleted file mode 100644 index 842bbda2..00000000 --- a/qtip/driver/playbook/ramspeed/setup.yaml +++ /dev/null @@ -1,70 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: localhost - connection: local - gather_facts: no - - tasks: - - name: Making ramspeed directory - file: - path: '{{ result_dir }}/' - state: directory - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Cleaning ramspeed directory - file: - path: '{{ ansible_env.HOME }}/ramspeed' - state: absent - - - name: Cleaning qtip_result directory - file: - path: '{{ ansible_env.HOME }}/qtip_result' - state: absent - - - include: ../prepare_env.yaml - - - name: Installing RAM_Speed dependencies if CentOS - yum: - name: '{{ item }}' - state: present - when: ansible_os_family == "RedHat" - with_items: - - gcc - - wget - - - name: Installing RAM_Speed dependencies if Ubuntu - apt: - name: '{{ item }}' - state: present - when: ansible_os_family == "Debian" - with_items: - - gcc - - wget - - - name: Making ramspeed temporary directory - file: - path: '{{ ansible_env.HOME }}/ramspeed' - state: directory - - - name: Fetch and untar ramspeed.tar.gz - unarchive: - # TODO: Need to upload this file to http://artifacts.opnfv.org/qtip/utilities - src: https://docs.google.com/uc?id=0B92Bp5LZTM7gRFctalZLMktTNDQ - dest: '{{ ansible_env.HOME }}/ramspeed/' - remote_src: True - - - name: Build ramsmp - shell: ./build.sh - args: - chdir: '{{ ansible_env.HOME }}/ramspeed/ramsmp-3.5.0' diff --git a/qtip/driver/playbook/unixbench/clean.yaml b/qtip/driver/playbook/unixbench/clean.yaml deleted file mode 100644 index a7cb2540..00000000 --- a/qtip/driver/playbook/unixbench/clean.yaml +++ /dev/null @@ -1,25 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# zhihui.wu1@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Cleaning tempT - file: - path: '{{ ansible_env.HOME }}/tempT' - state: absent - - - name: Cleaning qtip_result - file: - path: '{{ ansible_env.HOME }}/qtip_result' - state: absent - - diff --git a/qtip/driver/playbook/unixbench/dhrystone.yaml b/qtip/driver/playbook/unixbench/dhrystone.yaml deleted file mode 100644 index a0ee89a3..00000000 --- a/qtip/driver/playbook/unixbench/dhrystone.yaml +++ /dev/null @@ -1,41 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# zhihui.wu1@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- name: Make dhrystone directories - file: - path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dhrystone/' - state: directory - -- include: ../inxi.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dhrystone/' - -- include: ../top.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dhrystone/' - -- name: Run dhrystone - shell: ./Run -v dhrystone - args: - chdir: '{{ ansible_env.HOME }}/tempT/UnixBench/' - -- name: Copying result to qtip result directory - shell: '{{ item }}' - with_items: - - mv ~/tempT/UnixBench/results/*.log ./ - - mv ~/tempT/UnixBench/results/*.html ./ - - mv ~/tempT/UnixBench/results/* ./dhrystone - args: - chdir: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/dhrystone/' - -- name: Fetch dhrystone result files to local manchine - synchronize: - mode: pull - src: '{{ ansible_env.HOME }}/qtip_result/' - dest: '{{ result_dir }}/' diff --git a/qtip/driver/playbook/unixbench/run.yaml b/qtip/driver/playbook/unixbench/run.yaml deleted file mode 100644 index fbe4d4ad..00000000 --- a/qtip/driver/playbook/unixbench/run.yaml +++ /dev/null @@ -1,23 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# zhihui.wu1@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - set_fact: - is_dhrystone: "{{ dhrystone | default(False) }}" - is_whetstone: "{{ whetstone | default(False) }}" - - - include: ./dhrystone.yaml - when: "{{ is_dhrystone }}" - - - include: ./whetstone.yaml - when: "{{ is_whetstone }}" \ No newline at end of file diff --git a/qtip/driver/playbook/unixbench/setup.yaml b/qtip/driver/playbook/unixbench/setup.yaml deleted file mode 100644 index 4dcdd2c4..00000000 --- a/qtip/driver/playbook/unixbench/setup.yaml +++ /dev/null @@ -1,65 +0,0 @@ -############################################################################# -# Copyright (c) 2017 ZTE Corporation and others. -# zhihui.wu1@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################# - -- hosts: localhost - connection: local - gather_facts: no - - tasks: - - name: Making dhrystone directory - file: - path: '{{ result_dir }}/' - state: directory - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Cleaning tempT directory - file: - path: '{{ ansible_env.HOME }}/tempT' - state: absent - - - name: Cleaning qtip_result directory - file: - path: '{{ ansible_env.HOME }}/qtip_result' - state: absent - - - include: ../prepare_env.yaml - - - name: Installing UnixBench dependencies if CentOS - yum: - name: '{{ item }}' - state: present - when: ansible_os_family == "RedHat" - with_items: - - git - - gcc - - patch - - perl-Time-HiRes - - - name: Installing UnixBench dependencies if Ubuntu - apt: - name: '{{ item }}' - state: present - when: ansible_os_family == "Debian" - with_items: - - git - - gcc - - patch - - perl - - - name: Clone UnixBench - git: - repo: https://github.com/kdlucas/byte-unixbench.git - dest: '{{ ansible_env.HOME }}/tempT/' - - - name: Make UnixBench1 - shell: make --directory $HOME/tempT/UnixBench/ \ No newline at end of file diff --git a/qtip/driver/playbook/unixbench/whetstone.yaml b/qtip/driver/playbook/unixbench/whetstone.yaml deleted file mode 100644 index c753779f..00000000 --- a/qtip/driver/playbook/unixbench/whetstone.yaml +++ /dev/null @@ -1,41 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# zhihui.wu1@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- name: Make whetstone directories - file: - path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/whetstone/' - state: directory - -- include: ../inxi.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/whetstone/' - -- include: ../top.yaml - vars: - dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/whetstone/' - -- name: Run whetstone - shell: ./Run -v whetstone - args: - chdir: '{{ ansible_env.HOME }}/tempT/UnixBench/' - -- name: Copying result to qtip result directory - shell: '{{ item }}' - with_items: - - mv ~/tempT/UnixBench/results/*.log ./ - - mv ~/tempT/UnixBench/results/*.html ./ - - mv ~/tempT/UnixBench/results/* ./whetstone - args: - chdir: '{{ ansible_env.HOME }}/qtip_result/{{ ansible_hostname }}/whetstone/' - -- name: Fetch whetstone result files to local manchine - synchronize: - mode: pull - src: '{{ ansible_env.HOME }}/qtip_result/' - dest: '{{ result_dir }}/' diff --git a/qtip/runner/runner.py b/qtip/runner/runner.py deleted file mode 100644 index 9b09f0f8..00000000 --- a/qtip/runner/runner.py +++ /dev/null @@ -1,107 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE corp. and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -import argparse -import json -import os -from os import path -import sys -import time - -from qtip.collector.parser import grep -from qtip.driver.ansible_driver import AnsibleDriver -from qtip.util.logger import QtipLogger - -logger = QtipLogger('runner').get - -ALL_BENCHMARKS = ['dpi', 'ramspeed', 'ssl', 'dhrystone', 'whetstone'] - - -def parse_args(args): - parser = argparse.ArgumentParser() - parser.add_argument('-d', '--dest', required=True, - help='the destination where results will be stored.') - parser.add_argument('-b', '--benchmark', required=True, action='append', - help='the benchmark you want to execute.') - return parser.parse_args(args) - - -def run_benchmark(result_dir, benchmarks): - if not path.isdir(result_dir): - os.makedirs(result_dir) - driver = AnsibleDriver({'args': {'result_dir': result_dir}}) - driver.pre_run() - result = driver.run(benchmarks) - driver.cleanup() - return result - - -def generate_report(result_dir, start_time, stop_time): - output = { - "plan_name": "compute_qpi", - "start_time": start_time, - "stop_time": stop_time, - "sut": [] - } - output.update(parse_result(result_dir)) - output.update({'stop_time': stop_time}) - with open('{0}/result.json'.format(result_dir), 'w+') as f: - json.dump(output, f, indent=4, sort_keys=True) - - -def parse_result(result_dir): - sut_template = {'sut': []} - nodes_list = os.listdir(result_dir) - for node in nodes_list: - node_output_template = { - 'name': node, - 'type': 'baremetal', - 'qpis': [] - } - qpi_result = {'name': 'compute_qpi', 'benchmarks': []} - for benchmark in os.listdir('{0}/{1}'.format(result_dir, node)): - benchmark_result = \ - grep.parse_benchmark_result( - '{0}/{1}/{2}'.format(result_dir, node, benchmark)) - qpi_result['benchmarks'].append(benchmark_result) - node_output_template['qpis'].append(qpi_result) - sut_template['sut'].append(node_output_template) - return sut_template - - -def main(args=sys.argv[1:]): - args = parse_args(args) - - if not path.isdir(str(args.dest)): - logger.error("The destination {0} you give doesn't exist. " - "Please check!".format(args.dest)) - sys.exit(1) - - if args.benchmark == ['all']: - args.benchmark = ALL_BENCHMARKS - elif len(set(args.benchmark).difference(ALL_BENCHMARKS)) != 0: - logger.error("Please check benchmarks name. The supported benchmarks are" - "{0}".format(ALL_BENCHMARKS)) - logger.info("Start to run benchmark test: {0}.".format(args.benchmark)) - - start_time = time.strftime("%Y-%m-%d-%H-%M") - logger.info("start_time: {0}".format(start_time)) - if not args.dest.endswith('/'): - args.dest += '/' - result_dir = args.dest + 'qtip-' + start_time - ansible_result = run_benchmark(result_dir, args.benchmark) - stop_time = time.strftime("%Y-%m-%d-%H-%M") - logger.info("stop_time: {0}".format(stop_time)) - if not ansible_result: - logger.error("Bechmarks run failed. Cann't generate any report.") - sys.exit(1) - generate_report(result_dir, start_time, stop_time) - - -if __name__ == "__main__": - main() diff --git a/qtip/scripts/cleanup_creds.sh b/qtip/scripts/cleanup_creds.sh deleted file mode 100755 index ad66ba95..00000000 --- a/qtip/scripts/cleanup_creds.sh +++ /dev/null @@ -1,20 +0,0 @@ -#! /bin/bash -############################################################################## -# Copyright (c) 2017 ZTE corp. and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -DEST_IP=$1 -PRIVATE_KEY=$2 -PUBLIC_KEY=$3 -sshoptions="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" - -case "$INSTALLER_TYPE" in - fuel) - ssh $sshoptions -i $PRIVATE_KEY root@$DEST_IP "sed -i '/$PUBLIC_KEY/d' /root/.ssh/authorized_keys" - ;; -esac diff --git a/qtip/scripts/generate_host_file.sh b/qtip/scripts/generate_host_file.sh deleted file mode 100755 index ecc4d55f..00000000 --- a/qtip/scripts/generate_host_file.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash -############################################################################## -#Copyright (c) 2016 Ericsson AB, ZTE and others. -#jose.lausuch@ericsson.com -#All rights reserved. This program and the accompanying materials -#are made available under the terms of the Apache License, Version 2.0 -#which accompanies this distribution, and is available at -#http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -usage(){ - echo "usage: $0 -t -i -d " >&2 -} - -info() { - logger -s -t "generate_host_file.info" "$*" -} - -error() { - logger -s -t "generate_host_file.error" "$*" - exit 1 -} - -verify_connectivity(){ - local ip=$1 - info "Verifying connectivity to $ip..." - for i in $(seq 0 10); do - if ping -c 1 -W 1 $ip > /dev/null; then - info "$ip is reachable!" - return 0 - fi - sleep 1 - done - error "Can not talk to $ip." -} - -:${DEPLOY_TYPE:=''} - -#Getoptions -while getopts ":t:i:d:" optchar; do - case "${optchar}" in - t) installer_type=${OPTARG} ;; - i) installer_ipaddr=${OPTARG} ;; - d) dest_hostfile=${OPTARG} ;; - *) echo "Non-option argument: '-${OPTARG}'" >&2 - usage - exit 2 - ;; - esac -done - -#set vars from env if not provided by user as options -installer_type=${installer_type:-$INSTALLER_TYPE} -installer_ipaddr=${installer_ipaddr:-$INSTALLER_IP} - -if [ -z $installer_type ] || [ -z $installer_ipaddr ]; then - usage - exit 2 -fi - -ssh_options="-oUserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" - -#Start fetching compute ip -if [ "$installer_type" == "fuel" ]; then - verify_connectivity $installer_ipaddr - - env=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ipaddr} \ - 'fuel env'|grep operational|head -1|awk '{print $1}') &> /dev/null - if [ -z $env ]; then - error "No operational environment detected in Fuel" - fi - env_id="${FUEL_ENV:-$env}" - - # Check if compute is alive (online='True') - IPS=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ipaddr} \ - "fuel node --env ${env_id} | grep compute | grep 'True\| 1' | awk -F\| '{print \$5}' " | \ - sed 's/ //g') &> /dev/null - - -elif [ "$installer_type" == "apex" ]; then - echo "not implement now" - exit 1 - -elif [ "$installer_type" == "compass" ]; then - # need test - verify_connectivity $installer_ipaddr - IPS=$(sshpass -p'root' ssh 2>/dev/null $ssh_options root@${installer_ipaddr} \ - 'mysql -ucompass -pcompass -Dcompass -e"select * from cluster;"' \ - | awk -F"," '{for(i=1;i> $dest_hostfile - echo "$IPS" >> $dest_hostfile - cat $dest_hostfile -fi - -exit 0 diff --git a/qtip/util/env.py b/qtip/util/env.py deleted file mode 100644 index 9299f8c0..00000000 --- a/qtip/util/env.py +++ /dev/null @@ -1,213 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -from collections import defaultdict -import os -from os import path -import re -import socket -import sys -import time - -import paramiko - -from qtip.util.logger import QtipLogger - -logger = QtipLogger('env').get - -SCRIPT_DIR = path.join(path.dirname(__file__), path.pardir, 'scripts') -KEYNAME = 'QtipKey' -PRIVATE_KEY = '{0}/qtip/{1}'.format(os.environ['HOME'], KEYNAME) -PUBLIC_KEY = PRIVATE_KEY + '.pub' -HOST_FILE = '{0}/qtip/hosts'.format(os.environ['HOME']) - - -def all_files_exist(*files): - if len(files) == 0: - return False - flag = True - for f_item in files: - flag &= path.isfile(f_item) - logger.info("Is {0} existed: {1}".format(f_item, flag)) - return flag - - -def clean_file(*files): - if len(files) == 0: - logger.info('Nothing to clean') - return False - - def clean(f): - try: - if all_files_exist(f): - os.remove(f) - logger.info("Removed: {0}".format(f)) - else: - logger.info("Not exists: {0}".format(f)) - return True - except OSError as error: - logger.error("Not able to Remove: {0}".format(f), error) - return False - - results = map(clean, files) - return len(results) == len(files) and False not in results - - -class AnsibleEnvSetup(object): - def __init__(self): - self.keypair = defaultdict(str) - self.hostfile = None - self.host_ip_list = [] - - def setup(self, config={}): - try: - if 'hostfile' in config: - self.check_hostfile(config['hostfile']) - else: - self.generate_default_hostfile() - self.fetch_host_ip_from_hostfile() - if 'keypair' in config: - self.check_keypair(config['keypair']) - else: - self.generate_default_keypair() - self.pass_keypair_to_remote() - self.check_hosts_ssh_connectivity() - except Exception as error: - logger.info(error) - sys.exit(1) - - def check_keypair(self, keypair): - self.keypair = defaultdict(str) - if all_files_exist(keypair, '{0}.pub'.format(keypair)): - self.keypair['private'] = keypair - self.keypair['public'] = '{0}.pub'.format(keypair) - else: - raise RuntimeError("The keypairs you in the configuration file" - " is invalid or not existed.") - - def generate_default_keypair(self): - if not all_files_exist(PRIVATE_KEY, PUBLIC_KEY): - logger.info("Generate default keypair {0} under " - "{1}".format(KEYNAME, os.environ['HOME'])) - cmd = '''ssh-keygen -t rsa -N "" -f {0} -q -b 2048 - -C qtip@insecure'''.format(PRIVATE_KEY) - os.system(cmd) - self.keypair['private'] = PRIVATE_KEY - self.keypair['public'] = PUBLIC_KEY - - def pass_keypair_to_remote(self): - results = map(lambda ip: self._pass_keypair(ip, self.keypair['private']), - self.host_ip_list) - - if not (len(results) == len(self.host_ip_list) and False not in results): - raise RuntimeError("Failed on passing keypair to remote.") - - @staticmethod - def _pass_keypair(ip, private_key): - try: - os.system('ssh-keyscan %s >> /root/.ssh/known_hosts' % ip) - time.sleep(2) - ssh_cmd = '%s/qtip_creds.sh %s %s' % (SCRIPT_DIR, ip, private_key) - os.system(ssh_cmd) - logger.info('Pass keypair to remote hosts {0} successfully'.format(ip)) - return True - except Exception as error: - logger.error(error) - return False - - def check_hostfile(self, hostfile): - if all_files_exist(hostfile): - self.hostfile = hostfile - else: - raise RuntimeError( - "The hostfile {0} is invalid or not existed.".format(hostfile)) - - def generate_default_hostfile(self): - try: - # check whether the file is already existed - self.check_hostfile(HOST_FILE) - except Exception: - logger.info("Generate default hostfile {0} under " - "{1}".format(HOST_FILE, os.environ['HOME'])) - self._generate_hostfile_via_installer() - - def _generate_hostfile_via_installer(self): - self.hostfile = None - - installer_type = str(os.environ['INSTALLER_TYPE'].lower()) - installer_ip = str(os.environ['INSTALLER_IP']) - - if installer_type not in ["fuel"]: - raise ValueError("{0} is not supported".format(installer_type)) - if not installer_ip: - raise ValueError( - "The value of environment variable INSTALLER_IP is empty.") - - cmd = "bash %s/generate_host_file.sh -t %s -i %s -d %s" % \ - (SCRIPT_DIR, installer_type, installer_ip, HOST_FILE) - os.system(cmd) - - self.hostfile = HOST_FILE - - def fetch_host_ip_from_hostfile(self): - self.host_ip_list = [] - logger.info('Fetch host ips from hostfile...') - with open(self.hostfile, 'r') as f: - self.host_ip_list = re.findall('\d+.\d+.\d+.\d+', f.read()) - if self.host_ip_list: - logger.info("The remote compute nodes: {0}".format(self.host_ip_list)) - else: - raise ValueError("The hostfile doesn't include host ip addresses.") - - def check_hosts_ssh_connectivity(self): - results = map(lambda ip: self._ssh_is_ok(ip, self.keypair['private']), - self.host_ip_list) - if not (len(results) == len(self.host_ip_list) and False not in results): - raise RuntimeError("Failed on checking hosts ssh connectivity.") - - @staticmethod - def _ssh_is_ok(ip, private_key, attempts=100): - logger.info('Check hosts {0} ssh connectivity...'.format(ip)) - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(ip, key_filename=private_key) - - for attempt in range(attempts): - try: - stdin, stdout, stderr = ssh.exec_command('uname') - if not stderr.readlines(): - logger.info("{0}: SSH test successful.".format(ip)) - return True - except socket.error: - logger.debug("%s times ssh test......failed." % str(attempt + 1)) - if attempt == (attempts - 1): - return False - time.sleep(2) - return False - - def cleanup(self): - CI_DEBUG = os.getenv('CI_DEBUG') - - if CI_DEBUG is not None and CI_DEBUG.lower() == 'true': - logger.info("DEBUG Mode: please do cleanup by manual.") - else: - with open(self.keypair['public'], 'r') as f: - key = f.read().strip('\n').replace('/', '\/') - if key: - for ip in self.host_ip_list: - logger.info("Cleanup authorized_keys from {0}...".format(ip)) - cmd = '''bash {0}/cleanup_creds.sh {1} {2} "{3}"'''.format( - SCRIPT_DIR, ip, self.keypair['private'], key) - os.system(cmd) - else: - logger.error("Nothing in public key file.") - - logger.info("Cleanup hostfile and keypair.") - clean_file(self.hostfile, - self.keypair['private'], - self.keypair['public']) -- cgit 1.2.3-korg