diff options
41 files changed, 2190 insertions, 219 deletions
@@ -46,6 +46,63 @@ It is for macOS uses TLS instead of OpenSSL and no header files supported. The s # export LDFLAGS='-L $openssl_install_path/lib' ``` +### Third Party Code + +QTIP includes a few third party code via [subrepo](https://github.com/ingydotnet/git-subrepo). +All third party code are stored in `/third-party`. + +To pull the changes from remote repository, use + +``` +git subrepo pull third-party/<subdir> +``` + +It will create a new commit in parent repo, i.e. `qtip`. However, the +auto generated commit message does not include mandatory tags such as +`Change-Id` required by gerrit. You need to manually amend the commit to +append those. + +``` +git commit --amend -s +``` + +Example of final commit message + +``` +Include third party script for license checking and amending +The following commit message are generated automatically by git-subrepo +----------------------------------------------------------------------------- +git subrepo clone git@github.com:openzero-zte/License.git third-party/License + +subrepo: + subdir: "third-party/License" + merged: "61489da" +upstream: + origin: "git@github.com:openzero-zte/License.git" + branch: "master" + commit: "61489da" +git-subrepo: + version: "0.3.0" + origin: "https://github.com/ingydotnet/git-subrepo" + commit: "988f8c8" +----------------------------------------------------------------------------- +Change-Id: I8eab86a8ce3f26995af3e3535f31f361b4826a8b +Signed-off-by: Yujun Zhang <zhang.yujunz@zte.com.cn> +``` + +Sometimes you may modify the third-party code to adapt it in `qtip`. +To push the changes to remote repository, run + +``` +git subrepo push third-party/<subdir> +``` + +If you want to include a new repository of third party code. Use + +``` +git subrepo clone <remote-url> [<subdir>] +``` + ## Architecture **TODO**: move to design spec @@ -94,7 +151,23 @@ TBD ### CLI -TBD +Click currently supports Bash completion. The prerequisite for this is that the program +needs to be installed correctly. To install Qtip, execute the following command in root +folder of Qtip: + +``` +cd <project root> +pip install -e . +``` + +Once the installation has been completed successfully, the following needs to be added to +the `.bashrc` file: + +``` +eval "$(_QTIP_COMPLETE=source qtip)" +``` + +The above would activate command completion for Qtip. ### API diff --git a/legacy/assets/perftest/etc/dpi_average.sh b/legacy/assets/perftest/etc/dpi_average.sh deleted file mode 100644 index 405d3ff6..00000000 --- a/legacy/assets/perftest/etc/dpi_average.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -COUNTER=0 -WDIR=$PWD -while [ $COUNTER -lt 10 ]; do - - echo $WDIR - $( ./ndpiReader -i test.pcap >> $WDIR/dpi_dump.txt ) - let COUNTER=COUNTER+1 - echo "Run number: $COUNTER" - -done - - diff --git a/qtip/cli/commands/cmd_report.py b/qtip/cli/commands/cmd_report.py new file mode 100644 index 00000000..c780e847 --- /dev/null +++ b/qtip/cli/commands/cmd_report.py @@ -0,0 +1,27 @@ +############################################################################## +# Copyright (c) 2017 taseer94@gmail.com and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import click + +from qtip.cli.entry import Context + +pass_context = click.make_pass_decorator(Context, ensure=False) + + +@click.group() +@pass_context +def cli(ctx): + """ View Qtip results""" + pass + + +@cli.command('show') +@pass_context +def show(ctx): + pass diff --git a/qtip/cli/entry.py b/qtip/cli/entry.py index 9f51f25e..6cf78b58 100644 --- a/qtip/cli/entry.py +++ b/qtip/cli/entry.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2016 ZTE Corp and others. +# Copyright (c) 2017 taseer94@gmail.com and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -7,33 +7,19 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +import click import os import sys -import click CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) +# TODO (taseer) define user friendly error messages +sys.tracebacklimit = 0 -class Context(object): - - ''' TODO(taseer) implementation''' - - def __init__(self): - self.verbose = False - self.debug = False - - def log(self, msg, *args): - ''' Log message to stderr ''' - pass - def verbose(self, msg, *args): - ''' Log message to stderr when verbose ''' - pass - - def debug(self, msg, *args): - ''' Log message to debug ''' - pass +class Context(object): + """ Load configuration and pass to subcommands """ pass_context = click.make_pass_decorator(Context, ensure=True) @@ -70,4 +56,5 @@ class QtipCli(click.MultiCommand): @click.version_option('dev') @pass_context def cli(ctx, verbose, debug): - pass + if debug: + sys.tracebacklimit = 8 diff --git a/qtip/driver/playbook/bwn_ng.yaml b/qtip/driver/playbook/bwn_ng.yaml index 99477856..f79bb04e 100644 --- a/qtip/driver/playbook/bwn_ng.yaml +++ b/qtip/driver/playbook/bwn_ng.yaml @@ -20,4 +20,6 @@ when: ansible_os_family == "Debian" - name: Run bwm-ng - shell: bwm-ng -o plain -c 1 > $HOME/qtip_result/bwm-dump.log
\ No newline at end of file + shell: bwm-ng -o plain -c 1 > bwm-dump.log + args: + chdir: '{{ dest_path }}/'
\ No newline at end of file diff --git a/qtip/driver/playbook/dhrystone/run.yaml b/qtip/driver/playbook/dhrystone/run.yaml deleted file mode 100644 index 55de6597..00000000 --- a/qtip/driver/playbook/dhrystone/run.yaml +++ /dev/null @@ -1,63 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -- hosts: hosts - become: yes - remote_user: root - - tasks: - - name: Get current timestamp - set_fact: - timestamp: "{{ lookup('pipe', 'date +%Y-%m-%d-%H-%M') }}" - - - name: Checking home directory - shell: echo $HOME - register: home_dir - - - name: Fetch hostname - shell: hostname - register: host_name - - - name: Make UnixBench - shell: make --directory $HOME/tempT/UnixBench/ - - - name: Make some directories needed - file: - path: '{{ home_dir.stdout }}/qtip_result/{{ timestamp }}/{{ host_name.stdout }}' - state: directory - - - include: ../inxi.yaml - - - include: ../top.yaml - - - name: Run dhrystone - shell: ./Run -v dhrystone - args: - chdir: '{{ home_dir.stdout }}/tempT/UnixBench/' - - - name: Copying result to qtip result directory - shell: cp -r $HOME/tempT/UnixBench/results/* ./ - args: - chdir: '{{ home_dir.stdout }}/qtip_result/{{ timestamp }}/{{ host_name.stdout }}' - - - name: Copy top log to qtip result directory - shell: mv $HOME/qtip_result/top.log ./ - args: - chdir: '{{ home_dir.stdout }}/qtip_result/{{ timestamp }}/{{ host_name.stdout }}' - - - name: Copy inxi log to qtip result directory - shell: mv $HOME/qtip_result/inxi.log ./ - args: - chdir: '{{ home_dir.stdout }}/qtip_result/{{ timestamp }}/{{ host_name.stdout }}' - - - name: Fetch result files to local manchine - synchronize: - mode: pull - src: '{{ home_dir.stdout }}/qtip_result/' - dest: '{{ result_dir }}/dhrystone/logs/' diff --git a/qtip/driver/playbook/dpi/clean.yaml b/qtip/driver/playbook/dpi/clean.yaml new file mode 100644 index 00000000..0b9f9291 --- /dev/null +++ b/qtip/driver/playbook/dpi/clean.yaml @@ -0,0 +1,23 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: hosts + become: yes + remote_user: root + + tasks: + - name: Cleaning tempD + file: + path: '{{ ansible_env.HOME }}/tempD' + state: absent + + - name: Cleaning qtip_result + file: + path: '{{ ansible_env.HOME }}/qtip_result' + state: absent diff --git a/qtip/driver/playbook/dpi/dpi_average.sh b/qtip/driver/playbook/dpi/dpi_average.sh new file mode 100644 index 00000000..6f038053 --- /dev/null +++ b/qtip/driver/playbook/dpi/dpi_average.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +COUNTER=0 +WDIR=$PWD +while [ $COUNTER -lt 10 ]; do + echo $WDIR + $( ./ndpiReader -i test.pcap >> $WDIR/dpi_dump.txt ) + let COUNTER=COUNTER+1 + echo "Run number: $COUNTER" +done
\ No newline at end of file diff --git a/qtip/driver/playbook/dpi/run.yaml b/qtip/driver/playbook/dpi/run.yaml new file mode 100644 index 00000000..58f7eb2f --- /dev/null +++ b/qtip/driver/playbook/dpi/run.yaml @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: hosts + become: yes + remote_user: root + + tasks: + - name: Get current timestamp + set_fact: + timestamp: "{{ lookup('pipe', 'date +%Y-%m-%d-%H-%M') }}" + + - name: Make some directories needed + file: + path: '{{ ansible_env.HOME }}/qtip_result/{{ timestamp }}/{{ ansible_hostname }}' + state: directory + + - include: ../inxi.yaml + vars: + dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ timestamp }}/{{ ansible_hostname }}' + + - include: ../top.yaml + vars: + dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ timestamp }}/{{ ansible_hostname }}' + + - name: Run nDPI benchmark + shell: ./dpi_average.sh + args: + chdir: '{{ ansible_env.HOME }}/tempD/nDPI/example/' + + - name: Copying result and system info to qtip result directory + command: cp $HOME/tempD/nDPI/example/dpi_dump.txt ./ + args: + chdir: '{{ ansible_env.HOME }}/qtip_result/{{ timestamp }}/{{ ansible_hostname }}' + + - name: Fetch result files to local manchine + synchronize: + mode: pull + src: '{{ ansible_env.HOME }}/qtip_result/' + dest: '{{ result_dir }}/dpi/logs/' diff --git a/qtip/driver/playbook/dpi/setup.yaml b/qtip/driver/playbook/dpi/setup.yaml new file mode 100644 index 00000000..c1b45450 --- /dev/null +++ b/qtip/driver/playbook/dpi/setup.yaml @@ -0,0 +1,93 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: localhost + connection: local + gather_facts: no + + tasks: + - name: Making Dpi directory + file: + path: '{{ result_dir }}/dpi/logs/' + state: directory + +- hosts: hosts + become: yes + remote_user: root + + tasks: + - name: Cleaning tempD directory + file: + path: '{{ ansible_env.HOME }}/tempD' + state: absent + + - name: Cleaning qtip_result directory + file: + path: '{{ ansible_env.HOME }}/qtip_result' + state: absent + + - include: ../prepare_env.yaml + + - name: Installing nDPI dependencies if CentOS + yum: + name: '{{ item }}' + state: present + when: ansible_os_family == "RedHat" + with_items: + - git + - gcc + - patch + - perl-Time-HiRes + - autofconf + - automake + - libpcap-devel libtool + + - name: Installing nDPI dependencies if Ubuntu + apt: + name: '{{ item }}' + state: present + when: ansible_os_family == "Debian" + with_items: + - git + - gcc + - patch + - autoconf + - automake + - libpcap-dev + - libtool + + - name: Making nDPI temporary directory + file: + path: '{{ ansible_env.HOME }}/tempD' + state: directory + + - name: Clone nDPI + git: + repo: https://github.com/ntop/nDPI.git + dest: '{{ ansible_env.HOME }}/tempD/nDPI' + + - name: Run autogen && configure Dpi && make Dpi + command: '{{ item }}' + with_items: + - ./autogen.sh + - ./configure + - make + args: + chdir: '{{ ansible_env.HOME }}/tempD/nDPI/' + + - name: Fetching Test_pcap file + get_url: + url: http://build.opnfv.org/artifacts.opnfv.org/qtip/utilities/test.pcap + dest: '{{ ansible_env.HOME }}/tempD/nDPI/example/test.pcap' + + - name: Fetch Averaging script + copy: + src: ./dpi_average.sh + dest: '{{ ansible_env.HOME }}/tempD/nDPI/example' + mode: 777
\ No newline at end of file diff --git a/qtip/driver/playbook/inxi.yaml b/qtip/driver/playbook/inxi.yaml index f6a0311d..a06da042 100644 --- a/qtip/driver/playbook/inxi.yaml +++ b/qtip/driver/playbook/inxi.yaml @@ -20,4 +20,6 @@ when: ansible_os_family == "Debian" - name: Run inxi - shell: inxi -b -c0 -n > $HOME/qtip_result/inxi.log + shell: inxi -b -c0 -n > inxi.log + args: + chdir: '{{ dest_path }}/' diff --git a/qtip/driver/playbook/openssl/clean.yaml b/qtip/driver/playbook/openssl/clean.yaml new file mode 100644 index 00000000..0139ba54 --- /dev/null +++ b/qtip/driver/playbook/openssl/clean.yaml @@ -0,0 +1,23 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: hosts + become: yes + remote_user: root + + tasks: + - name: Cleaning Open_SSL + file: + path: '{{ ansible_env.HOME }}/Open_SSL' + state: absent + + - name: Cleaning qtip_result + file: + path: '{{ ansible_env.HOME }}/qtip_result' + state: absent diff --git a/qtip/driver/playbook/openssl/run.yaml b/qtip/driver/playbook/openssl/run.yaml new file mode 100644 index 00000000..241cac61 --- /dev/null +++ b/qtip/driver/playbook/openssl/run.yaml @@ -0,0 +1,49 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: hosts + become: yes + remote_user: root + + tasks: + - name: Get current timestamp + set_fact: + timestamp: "{{ lookup('pipe', 'date +%Y-%m-%d-%H-%M') }}" + + - name: Make some directories needed + file: + path: '{{ ansible_env.HOME }}/qtip_result/{{ timestamp }}/{{ ansible_hostname }}' + state: directory + + - include: ../inxi.yaml + vars: + dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ timestamp }}/{{ ansible_hostname }}/' + + - include: ../top.yaml + vars: + dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ timestamp }}/{{ ansible_hostname }}/' + + - name: Benchmarking RSA signatures and AES-128-cbc cipher encryption throughput + shell: '{{ item }}' + with_items: + - ./openssl speed rsa >> RSA_dump + - ./openssl speed -evp aes-128-cbc >> AES-128-CBC_dump + args: + chdir: '{{ ansible_env.HOME }}/Open_SSL/openssl-1.0.2f/apps' + + - name: Copying result to qtip result directory + shell: cp ~/Open_SSL/openssl-1.0.2f/apps/*_dump ./ + args: + chdir: '{{ ansible_env.HOME }}/qtip_result/{{ timestamp }}/{{ ansible_hostname }}/' + + - name: Fetch result files to local manchine + synchronize: + mode: pull + src: '{{ ansible_env.HOME }}/qtip_result/' + dest: '{{ result_dir }}/ssl/logs/' diff --git a/qtip/driver/playbook/openssl/setup.yaml b/qtip/driver/playbook/openssl/setup.yaml new file mode 100644 index 00000000..e93bfb8e --- /dev/null +++ b/qtip/driver/playbook/openssl/setup.yaml @@ -0,0 +1,87 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: localhost + connection: local + gather_facts: no + + tasks: + - name: Making ssl directory + file: + path: '{{ result_dir }}/ssl/logs/' + state: directory + +- hosts: hosts + become: yes + remote_user: root + + tasks: + - name: Cleaning Open_SSL directory + file: + path: '{{ ansible_env.HOME }}/Open_SSL' + state: absent + + - name: Cleaning qtip_result directory + file: + path: '{{ ansible_env.HOME }}/qtip_result' + state: absent + + - include: ../prepare_env.yaml + + - name: Installing UnixBench dependencies if CentOS + yum: + name: '{{ item }}' + state: present + when: ansible_os_family == "RedHat" + with_items: + - git + - gcc + - patch + - perl-Time-HiRes + - wget + - autofconf + - automake + - libpcap-devel + - libtool + + - name: Installing UnixBench dependencies if Ubuntu + apt: + name: '{{ item }}' + state: present + when: ansible_os_family == "Debian" + with_items: + - git + - gcc + - patch + - perl + - wget + - autoconf + - automake + - libpcap-dev + - libtool + + - name: Make Open_SSL directory + file: + path: '{{ ansible_env.HOME }}/Open_SSL' + state: directory + + - name: Untar OpenSSL + unarchive: + src: http://artifacts.opnfv.org/qtip/utilities/openssl-1.0.2f.tar.gz + dest: '{{ ansible_env.HOME }}/Open_SSL/' + remote_src: True + + - name: Configure && Make && Install OpenSSL + shell: "{{ item }}" + with_items: + - ./config + - make + - make install + args: + chdir: '{{ ansible_env.HOME }}/Open_SSL/openssl-1.0.2f'
\ No newline at end of file diff --git a/qtip/driver/playbook/ramspeed/clean.yaml b/qtip/driver/playbook/ramspeed/clean.yaml new file mode 100644 index 00000000..f0188159 --- /dev/null +++ b/qtip/driver/playbook/ramspeed/clean.yaml @@ -0,0 +1,23 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: hosts + become: yes + remote_user: root + + tasks: + - name: Cleaning ramspeed + file: + path: '{{ ansible_env.HOME }}/ramspeed' + state: absent + + - name: Cleaning qtip_result + file: + path: '{{ ansible_env.HOME }}/qtip_result' + state: absent diff --git a/qtip/driver/playbook/ramspeed/run.yaml b/qtip/driver/playbook/ramspeed/run.yaml new file mode 100644 index 00000000..33c9a6ef --- /dev/null +++ b/qtip/driver/playbook/ramspeed/run.yaml @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: hosts + become: yes + remote_user: root + + tasks: + - name: Get current timestamp + set_fact: + timestamp: "{{ lookup('pipe', 'date +%Y-%m-%d-%H-%M') }}" + + - name: Make some directories needed + file: + path: '{{ ansible_env.HOME }}/qtip_result/{{ timestamp }}/{{ ansible_hostname }}' + state: directory + + - include: ../inxi.yaml + vars: + dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ timestamp }}/{{ ansible_hostname }}/' + + - include: ../top.yaml + vars: + dest_path: '{{ ansible_env.HOME }}/qtip_result/{{ timestamp }}/{{ ansible_hostname }}/' + + - name: Benchmarking IntMem Bandwidth and FloatMem Bandwidth + shell: '{{ item }}' + with_items: + - ~/ramspeed/ramsmp-3.5.0/ramsmp -b 3 -l 5 -p 1 >> Intmem + - ~/ramspeed/ramsmp-3.5.0/ramsmp -b 6 -l 5 -p 1 >> Floatmem + args: + chdir: '{{ ansible_env.HOME }}/qtip_result/{{ timestamp }}/{{ ansible_hostname }}/' + + - name: Fetch result files to local manchine + synchronize: + mode: pull + src: '{{ ansible_env.HOME }}/qtip_result/' + dest: '{{ result_dir }}/ramspeed/logs/' diff --git a/qtip/driver/playbook/ramspeed/setup.yaml b/qtip/driver/playbook/ramspeed/setup.yaml new file mode 100644 index 00000000..60368605 --- /dev/null +++ b/qtip/driver/playbook/ramspeed/setup.yaml @@ -0,0 +1,70 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: localhost + connection: local + gather_facts: no + + tasks: + - name: Making ramspeed directory + file: + path: '{{ result_dir }}/ramspeed/logs/' + state: directory + +- hosts: hosts + become: yes + remote_user: root + + tasks: + - name: Cleaning ramspeed directory + file: + path: '{{ ansible_env.HOME }}/ramspeed' + state: absent + + - name: Cleaning qtip_result directory + file: + path: '{{ ansible_env.HOME }}/qtip_result' + state: absent + + - include: ../prepare_env.yaml + + - name: Installing RAM_Speed dependencies if CentOS + yum: + name: '{{ item }}' + state: present + when: ansible_os_family == "RedHat" + with_items: + - gcc + - wget + + - name: Installing RAM_Speed dependencies if Ubuntu + apt: + name: '{{ item }}' + state: present + when: ansible_os_family == "Debian" + with_items: + - gcc + - wget + + - name: Making ramspeed temporary directory + file: + path: '{{ ansible_env.HOME }}/ramspeed' + state: directory + + - name: Fetch and untar ramspeed.tar.gz + unarchive: + # TODO: Need to upload this file to http://artifacts.opnfv.org/qtip/utilities + src: https://docs.google.com/uc?id=0B92Bp5LZTM7gRFctalZLMktTNDQ + dest: '{{ ansible_env.HOME }}/ramspeed/' + remote_src: True + + - name: Build ramsmp + shell: ./build.sh + args: + chdir: '{{ ansible_env.HOME }}/ramspeed/ramsmp-3.5.0' diff --git a/qtip/driver/playbook/top.yaml b/qtip/driver/playbook/top.yaml index 8de7e3d6..64584338 100644 --- a/qtip/driver/playbook/top.yaml +++ b/qtip/driver/playbook/top.yaml @@ -7,4 +7,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## - name: Collect cpu usage - shell: top -bn1 > $HOME/qtip_result/top.log + shell: top -bn1 > top.log + args: + chdir: '{{ dest_path }}/' diff --git a/qtip/driver/playbook/dhrystone/clean.yaml b/qtip/driver/playbook/unixbench/clean.yaml index 72bfab7e..a7cb2540 100644 --- a/qtip/driver/playbook/dhrystone/clean.yaml +++ b/qtip/driver/playbook/unixbench/clean.yaml @@ -1,6 +1,6 @@ ############################################################################## # Copyright (c) 2017 ZTE Corporation and others. -# +# zhihui.wu1@zte.com.cn # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at @@ -12,16 +12,14 @@ remote_user: root tasks: - - name: Checking home directory - shell: echo $HOME - register: home_dir - - name: Cleaning tempT file: - path: '{{ home_dir.stdout }}/tempT' + path: '{{ ansible_env.HOME }}/tempT' state: absent - name: Cleaning qtip_result file: - path: '{{ home_dir.stdout }}/qtip_result' + path: '{{ ansible_env.HOME }}/qtip_result' state: absent + + diff --git a/qtip/driver/playbook/unixbench/dhrystone.yaml b/qtip/driver/playbook/unixbench/dhrystone.yaml new file mode 100644 index 00000000..431814dd --- /dev/null +++ b/qtip/driver/playbook/unixbench/dhrystone.yaml @@ -0,0 +1,27 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# zhihui.wu1@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: Run dhrystone + shell: ./Run -v dhrystone + args: + chdir: '{{ ansible_env.HOME }}/tempT/UnixBench/' + +- name: Copying result and system info to qtip result directory + shell: '{{ item }}' + with_items: + - mv ~/tempT/UnixBench/results/* ./ + - cp ~/qtip_result/inxi.log ~/qtip_result/top.log ./ + args: + chdir: '{{ ansible_env.HOME }}/qtip_result/{{ timestamp }}/{{ ansible_hostname }}' + +- name: Fetch dhrystone result files to local manchine + synchronize: + mode: pull + src: '{{ ansible_env.HOME }}/qtip_result/' + dest: '{{ result_dir }}/dhrystone/logs/' diff --git a/qtip/driver/playbook/unixbench/run.yaml b/qtip/driver/playbook/unixbench/run.yaml new file mode 100644 index 00000000..acef36a7 --- /dev/null +++ b/qtip/driver/playbook/unixbench/run.yaml @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# zhihui.wu1@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: hosts + become: yes + remote_user: root + + tasks: + - set_fact: + is_dhrystone: "{{ dhrystone | default(False) }}" + is_whetstone: "{{ whetstone | default(False) }}" + timestamp: "{{ lookup('pipe', 'date +%Y-%m-%d-%H-%M') }}" + + - name: Make some directories needed + file: + path: '{{ ansible_env.HOME }}/qtip_result/{{ timestamp }}/{{ ansible_hostname }}' + state: directory + + - include: ../inxi.yaml + vars: + dest_path: '{{ ansible_env.HOME }}/qtip_result/' + + - include: ../top.yaml + vars: + dest_path: '{{ ansible_env.HOME }}/qtip_result/' + + - include: ./dhrystone.yaml + when: "{{ is_dhrystone }}" + + - include: ./whetstone.yaml + when: "{{ is_whetstone }}"
\ No newline at end of file diff --git a/qtip/driver/playbook/dhrystone/setup.yaml b/qtip/driver/playbook/unixbench/setup.yaml index 430670c1..283884ac 100644 --- a/qtip/driver/playbook/dhrystone/setup.yaml +++ b/qtip/driver/playbook/unixbench/setup.yaml @@ -1,39 +1,46 @@ -############################################################################## +############################################################################# # Copyright (c) 2017 ZTE Corporation and others. -# +# zhihui.wu1@zte.com.cn # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## +############################################################################# - hosts: localhost connection: local gather_facts: no tasks: + - set_fact: + is_dhrystone: "{{ dhrystone | default(False) }}" + is_whetstone: "{{ whetstone | default(False) }}" + - name: Making dhrystone directory file: path: '{{ result_dir }}/dhrystone/logs/' state: directory + when: '{{ is_dhrystone }}' + + - name: Making whetstone directory + file: + path: '{{ result_dir }}/whetstone/logs/' + state: directory + when: '{{ is_whetstone }}' - hosts: hosts become: yes remote_user: root tasks: - - name: Checking home directory - shell: echo $HOME - register: home_dir - - name: Cleaning tempT directory file: - path: '{{ home_dir.stdout }}/tempT' + path: '{{ ansible_env.HOME }}/tempT' state: absent - name: Cleaning qtip_result directory file: - path: '{{ home_dir.stdout }}/qtip_result' + path: '{{ ansible_env.HOME }}/qtip_result' state: absent - include: ../prepare_env.yaml @@ -60,7 +67,10 @@ - patch - perl - - name: Clone unixbench + - name: Clone UnixBench git: repo: https://github.com/kdlucas/byte-unixbench.git - dest: '{{ home_dir.stdout }}/tempT' + dest: '{{ ansible_env.HOME }}/tempT/' + + - name: Make UnixBench + shell: make --directory $HOME/tempT/UnixBench/
\ No newline at end of file diff --git a/qtip/driver/playbook/unixbench/whetstone.yaml b/qtip/driver/playbook/unixbench/whetstone.yaml new file mode 100644 index 00000000..f09e4e53 --- /dev/null +++ b/qtip/driver/playbook/unixbench/whetstone.yaml @@ -0,0 +1,28 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# zhihui.wu1@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: Run whetstone + shell: ./Run -v whetstone + when: whetstone + args: + chdir: '{{ ansible_env.HOME }}/tempT/UnixBench/' + +- name: Copying result and system info to qtip result directory + shell: '{{ item }}' + with_items: + - mv ~/tempT/UnixBench/results/* ./ + - cp ~/qtip_result/inxi.log ~/qtip_result/top.log ./ + args: + chdir: '{{ ansible_env.HOME }}/qtip_result/{{ timestamp }}/{{ ansible_hostname }}' + +- name: Fetch whetstone result files to local manchine + synchronize: + mode: pull + src: '{{ ansible_env.HOME }}/qtip_result/' + dest: '{{ result_dir }}/whetstone/logs/' diff --git a/qtip/reporter/console.py b/qtip/reporter/console.py index 24c98e74..2b5130a6 100644 --- a/qtip/reporter/console.py +++ b/qtip/reporter/console.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2016 ZTE Corp and others. +# Copyright (c) 2017 taseer94@gmail.com and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -7,6 +7,9 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +from jinja2 import Environment +from jinja2 import FileSystemLoader +from os import path from qtip.base import BaseActor @@ -16,8 +19,12 @@ class ConsoleReporter(BaseActor): """ def __init__(self, config, parent=None): super(ConsoleReporter, self).__init__(config, parent=parent) - # TODO(yujunz) remove PoC code - self._fmt = "{title}: {description}" + + # TODO (taseer) load template from config + tpl_loader = FileSystemLoader(path.join(path.dirname(__file__), 'templates')) + env = Environment(loader=tpl_loader) + self._template = env.get_template('timeline.j2') def render(self, var_dict): - return self._fmt.format(**var_dict) + out = self._template.render(var_dict) + return out diff --git a/qtip/reporter/templates/report.j2 b/qtip/reporter/templates/report.j2 new file mode 100644 index 00000000..766e6dde --- /dev/null +++ b/qtip/reporter/templates/report.j2 @@ -0,0 +1,22 @@ +{{ title }} + +Plan: {{ plan.name }} + +{{ qpi.name }}: {{ qpi.score }} +Sections: +{% for section in sections %} + {{ section.name }}: {{ section.score }} + + Formula: {{ section.formula }} + Metrics: + {% for metric in section.metrics %} + {{ metric.name }}: {{ metric.score }} + Formula: {{ metric.formula }} + Workloads: + {% for workload in workloads %} + {{ workload.name }}: {{ workload.score }} + {% endfor %} + {% endfor %} +{% endfor %} + +{{ signature }} diff --git a/qtip/reporter/templates/timeline.j2 b/qtip/reporter/templates/timeline.j2 index 9c18a996..d4c95c46 100644 --- a/qtip/reporter/templates/timeline.j2 +++ b/qtip/reporter/templates/timeline.j2 @@ -1,4 +1,4 @@ -{% title %} +{{ title }} {% for phase in phases %} {{ phase.name|upper }}{{ "TIME" }} {% for cp in phase.checkpoints %} diff --git a/qtip/util/env.py b/qtip/util/env.py index 0585a4c1..24e08658 100644 --- a/qtip/util/env.py +++ b/qtip/util/env.py @@ -1,15 +1,18 @@ ############################################################################## -# Copyright (c) 2016 Dell Inc, ZTE and others. +# Copyright (c) 2017 ZTE and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +from collections import defaultdict import os +from os import path +import re import socket +import sys import time -from os import path import paramiko @@ -51,50 +54,134 @@ def clean_file(*files): return len(results) == len(files) and False not in results -def generate_host_file(hostfile=HOST_FILE): - installer_type = str(os.environ['INSTALLER_TYPE'].lower()) - installer_ip = str(os.environ['INSTALLER_IP']) - - if installer_type not in ["fuel"]: - raise ValueError("%s is not supported" % installer_type) - if not installer_ip: - raise ValueError("The value of environment variable INSTALLER_IP is empty") - - cmd = "bash %s/generate_host_file.sh -i %s -a %s -d %s" % \ - (SCRIPT_DIR, installer_type, installer_ip, hostfile) - os.system(cmd) - return all_files_exist(hostfile) - - -def generate_keypair(keyname='QtipKey'): - """Generating ssh keypair""" - cmd = "ssh-keygen -t rsa -N "" -f {0} -q -b 2048".format(keyname) - os.system(cmd) - return all_files_exist(PRIVATE_KEY, PUBLIC_KEY) - - -def pass_keypair(ip, private_key=PRIVATE_KEY): - os.system('ssh-keyscan %s >> /root/.ssh/known_hosts' % ip) - time.sleep(2) - - ssh_cmd = '%s/qtip_creds.sh %s %s' % (SCRIPT_DIR, ip, private_key) - os.system(ssh_cmd) +class AnsibleEnvSetup(object): + def __init__(self): + self.keypair = defaultdict(str) + self.hostfile = None + self.host_ip_list = [] + def setup(self, config={}): + try: + if 'hostfile' in config: + self.check_hostfile(config['hostfile']) + else: + self.generate_default_hostfile() + self.fetch_host_ip_from_hostfile() + if 'keypair' in config: + self.check_keypair(config['keypair']) + else: + self.generate_default_keypair() + self.pass_keypair_to_remote() + self.check_hosts_ssh_connectivity() + except Exception as error: + print(error) + sys.exit(1) + + def check_keypair(self, keypair): + self.keypair = defaultdict(str) + if all_files_exist(keypair, '{0}.pub'.format(keypair)): + self.keypair['private'] = keypair + self.keypair['public'] = '{0}.pub'.format(keypair) + else: + raise RuntimeError("The keypairs you in the configuration file" + " is invalid or not existed.") + + def generate_default_keypair(self): + if not all_files_exist(PRIVATE_KEY, PUBLIC_KEY): + print("Generate default keypair {0} under " + "{1}".format(KEYNAME, os.environ['HOME'])) + cmd = '''ssh-keygen -t rsa -N "" -f {0} -q -b 2048'''.format( + PRIVATE_KEY) + os.system(cmd) + self.keypair['private'] = PRIVATE_KEY + self.keypair['public'] = PUBLIC_KEY + + def pass_keypair_to_remote(self): + results = map(lambda ip: self._pass_keypair(ip, self.keypair['private']), + self.host_ip_list) + + if not (len(results) == len(self.host_ip_list) and False not in results): + raise RuntimeError("Failed on passing keypair to remote.") + + @staticmethod + def _pass_keypair(ip, private_key): + try: + os.system('ssh-keyscan %s >> /root/.ssh/known_hosts' % ip) + time.sleep(2) + ssh_cmd = '%s/qtip_creds.sh %s %s' % (SCRIPT_DIR, ip, private_key) + os.system(ssh_cmd) + print('Pass keypair to remote hosts {0} successfully'.format(ip)) + return True + except Exception as error: + print(error) + return False -def ssh_is_ok(ip, private_key=PRIVATE_KEY, attempts=100): - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(ip, key_filename=private_key) + def check_hostfile(self, hostfile): + if all_files_exist(hostfile): + self.hostfile = hostfile + else: + raise RuntimeError( + "The hostfile {0} is invalid or not existed.".format(hostfile)) - for attempt in range(attempts): + def generate_default_hostfile(self): try: - stdin, stdout, stderr = ssh.exec_command('uname') - if not stderr.readlines(): - print("{0}: SSH test successful".format(ip)) - return True - except socket.error: - if attempt == (attempts - 1): - return False - print("%s times ssh test......failed" % attempt) - time.sleep(2) - return False + # check whether the file is already existed + self.check_hostfile(HOST_FILE) + except Exception: + print("Generate default hostfile {0} under " + "{1}".format(HOST_FILE, os.environ['HOME'])) + self._generate_hostfile_via_installer() + + def _generate_hostfile_via_installer(self): + self.hostfile = None + + installer_type = str(os.environ['INSTALLER_TYPE'].lower()) + installer_ip = str(os.environ['INSTALLER_IP']) + + if installer_type not in ["fuel"]: + raise ValueError("{0} is not supported".format(installer_type)) + if not installer_ip: + raise ValueError( + "The value of environment variable INSTALLER_IP is empty.") + + cmd = "bash %s/generate_host_file.sh -t %s -i %s -d %s" % \ + (SCRIPT_DIR, installer_type, installer_ip, HOST_FILE) + os.system(cmd) + + self.hostfile = HOST_FILE + + def fetch_host_ip_from_hostfile(self): + self.host_ip_list = [] + print('Fetch host ips from hostfile...') + with open(self.hostfile, 'r') as f: + self.host_ip_list = re.findall('\d+.\d+.\d+.\d+', f.read()) + if self.host_ip_list: + print("The remote compute nodes: {0}".format(self.host_ip_list)) + else: + raise ValueError("The hostfile doesn't include host ip addresses.") + + def check_hosts_ssh_connectivity(self): + results = map(lambda ip: self._ssh_is_ok(ip, self.keypair['private']), + self.host_ip_list) + if not (len(results) == len(self.host_ip_list) and False not in results): + raise RuntimeError("Failed on checking hosts ssh connectivity.") + + @staticmethod + def _ssh_is_ok(ip, private_key, attempts=100): + print('Check hosts {0} ssh connectivity...'.format(ip)) + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, key_filename=private_key) + + for attempt in range(attempts): + try: + stdin, stdout, stderr = ssh.exec_command('uname') + if not stderr.readlines(): + print("{0}: SSH test successful.".format(ip)) + return True + except socket.error: + print("%s times ssh test......failed." % str(attempt + 1)) + if attempt == (attempts - 1): + return False + time.sleep(2) + return False diff --git a/test-requirements.txt b/test-requirements.txt index e434748e..cfbbdcdd 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,6 +5,7 @@ tox pytest pytest-cov +pytest-mock coverage pykwalify mock diff --git a/tests/data/benchmarks/plan/compute.yaml b/tests/data/benchmarks/plan/compute.yaml index 8529d8dc..f4a7a2dc 100644 --- a/tests/data/benchmarks/plan/compute.yaml +++ b/tests/data/benchmarks/plan/compute.yaml @@ -16,6 +16,80 @@ config: collectors: - type: logfile paths: + - '../../external/dpi/' + logs: + - filename: dpi_dump.txt + parsers: + - type: grep + regex: |- + ^\s+nDPI throughput:.+?(?P<pps>\d+.\d+)\sM\spps.+ + ?(?P<bps>\d+.\d+)\sGb\/sec + - type: logfile + paths: + - '../../external/ramspeed/' + logs: + - filename: Intmem + parsers: + - type: grep + regex: '^INTEGER\s+BatchRun\s+Copy:\s+?(?P<integer_copy>\d+\.\d+)\sMB/s$' + - type: grep + regex: '^INTEGER\s+BatchRun\s+Scale:\s+?(?P<integer_scale>\d+\.\d+)\sMB/s$' + - type: grep + regex: '^INTEGER\s+BatchRun\s+Add:\s+?(?P<integer_add>\d+\.\d+)\sMB/s$' + - type: grep + regex: '^INTEGER\s+BatchRun\s+Triad:\s+?(?P<integer_triad>\d+\.\d+)\sMB/s$' + - type: grep + regex: '^INTEGER\s+BatchRun\s+AVERAGE:\s+?(?P<integer_average>\d+\.\d+)\sMB/s$' + - filename: Floatmem + parsers: + - type: grep + regex: '^FL-POINT\s+BatchRun\s+Copy:\s+?(?P<float_copy>\d+\.\d+)\sMB/s$' + - type: grep + regex: '^FL-POINT\s+BatchRun\s+Scale:\s+?(?P<float_scale>\d+\.\d+)\sMB/s$' + - type: grep + regex: '^FL-POINT\s+BatchRun\s+Add:\s+?(?P<float_add>\d+\.\d+)\sMB/s$' + - type: grep + regex: '^FL-POINT\s+BatchRun\s+Triad:\s+?(?P<float_triad>\d+\.\d+)\sMB/s$' + - type: grep + regex: '^FL-POINT\s+BatchRun\s+AVERAGE:\s+?(?P<float_average>\d+\.\d+)\sMB/s$' + - type: logfile + paths: + - '../../external/ssl/' + logs: + - filename: RSA_dump + parsers: + - type: grep + regex: |- + ^rsa\s+512\sbits\s.+ + ?(?P<rsa_sign_512>\d+\.\d)\s+ + ?(?P<rsa_verify_512>\d+\.\d)$ + - type: grep + regex: |- + ^rsa\s+1024\sbits\s.+ + ?(?P<rsa_sign_1024>\d+\.\d)\s+ + ?(?P<rsa_verify_1024>\d+\.\d)$ + - type: grep + regex: |- + ^rsa\s+2048\sbits\s.+ + ?(?P<rsa_sign_2048>\d+\.\d)\s+ + ?(?P<rsa_verify_2048>\d+\.\d)$ + - type: grep + regex: |- + ^rsa\s+4096\sbits\s.+ + ?(?P<rsa_sign_4096>\d+\.\d)\s+ + ?(?P<rsa_verify_4096>\d+\.\d)$ + - filename: AES-128-CBC_dump + parsers: + - type: grep + regex: |- + ^aes-128-cbc\s+ + ?(?P<aes_128_cbc_16_bytes>\d+\.\w+)\s+ + ?(?P<aes_128_cbc_64_bytes>\d+\.\w+)\s+ + ?(?P<aes_128_cbc_256_bytes>\d+\.\w+)\s+ + ?(?P<aes_128_cbc_1024_bytes>\d+\.\w+)\s+ + ?(?P<aes_128_cbc_8192_bytes>\d+\.\w+)$ + - type: logfile + paths: - '../../external/sysinfo' logs: - filename: top.log diff --git a/tests/data/external/dpi/dpi_dump.txt b/tests/data/external/dpi/dpi_dump.txt new file mode 100644 index 00000000..1f4e2839 --- /dev/null +++ b/tests/data/external/dpi/dpi_dump.txt @@ -0,0 +1,809 @@ +----------------------------------------------------------- +* NOTE: This is demo app to show *some* nDPI features. +* In this demo we have implemented only some basic features +* just to show you what you can do with the library. Feel +* free to extend it and send us the patches for inclusion +------------------------------------------------------------ + +Using nDPI (1.8.0-dev-707-6a27b62) [1 thread(s)] +Reading packets from pcap file test.pcap... +Running thread 0... + +nDPI Memory statistics: + nDPI Memory (once): 110.55 KB + Flow Memory (per flow): 1.95 KB + Actual Memory: 3.46 MB + Peak Memory: 3.46 MB + +Traffic statistics: + Ethernet bytes: 75948198 (includes ethernet CRC/IFC/trailer) + Discarded bytes: 55256 + IP packets: 60265 of 61155 packets total + IP bytes: 74501838 (avg pkt size 1218 bytes) + Unique flows: 545 + TCP Packets: 21188 + UDP Packets: 39042 + VLAN Packets: 0 + MPLS Packets: 0 + PPPoE Packets: 0 + Fragmented Packets: 4 + Max Packet size: 64260 + Packet Len < 64: 30624 + Packet Len 64-128: 1158 + Packet Len 128-256: 674 + Packet Len 256-1024: 817 + Packet Len 1024-1500: 21924 + Packet Len > 1500: 5068 + nDPI throughput: 1.56 M pps / 14.62 Gb/sec + Traffic throughput: 228.03 pps / 2.19 Mb/sec + Traffic duration: 264.290 sec + Guessed flow protos: 9 + + +Detected protocols: + Unknown packets: 6126 bytes: 10913415 flows: 46 + FTP_CONTROL packets: 42 bytes: 4384 flows: 1 + DNS packets: 66 bytes: 6520 flows: 23 + IPP packets: 46 bytes: 12059 flows: 1 + HTTP packets: 377 bytes: 113616 flows: 24 + MDNS packets: 43 bytes: 11742 flows: 9 + NetBIOS packets: 48 bytes: 4754 flows: 12 + SSDP packets: 101 bytes: 32533 flows: 21 + DHCP packets: 21 bytes: 7346 flows: 2 + BitTorrent packets: 37639 bytes: 29656716 flows: 65 + ICMP packets: 15 bytes: 4956 flows: 5 + IGMP packets: 10 bytes: 600 flows: 9 + SSL packets: 1199 bytes: 622508 flows: 24 + ICMPV6 packets: 2 bytes: 172 flows: 2 + DHCPV6 packets: 1 bytes: 144 flows: 1 + Facebook packets: 349 bytes: 206444 flows: 21 + Twitter packets: 137 bytes: 45679 flows: 8 + Dropbox packets: 155 bytes: 45358 flows: 12 + GMail packets: 35 bytes: 10470 flows: 3 + YouTube packets: 11721 bytes: 31050728 flows: 33 + Skype packets: 237 bytes: 35082 flows: 30 + Google packets: 558 bytes: 164639 flows: 52 + WhatsApp packets: 969 bytes: 1477350 flows: 17 + Viber packets: 60 bytes: 27602 flows: 1 + LLMNR packets: 71 bytes: 5297 flows: 36 + Amazon packets: 89 bytes: 23572 flows: 9 + QUIC packets: 41 bytes: 3474 flows: 1 + BJNP packets: 72 bytes: 4320 flows: 72 + Microsoft packets: 35 bytes: 10358 flows: 5 + + +Protocol statistics: + Safe 632978 bytes + Acceptable 31693889 bytes + Fun 31257172 bytes + Unsafe 4384 bytes + Unrated 10913415 bytes + +----------------------------------------------------------- +* NOTE: This is demo app to show *some* nDPI features. +* In this demo we have implemented only some basic features +* just to show you what you can do with the library. Feel +* free to extend it and send us the patches for inclusion +------------------------------------------------------------ + +Using nDPI (1.8.0-dev-707-6a27b62) [1 thread(s)] +Reading packets from pcap file test.pcap... +Running thread 0... + +nDPI Memory statistics: + nDPI Memory (once): 110.55 KB + Flow Memory (per flow): 1.95 KB + Actual Memory: 3.46 MB + Peak Memory: 3.46 MB + +Traffic statistics: + Ethernet bytes: 75948198 (includes ethernet CRC/IFC/trailer) + Discarded bytes: 55256 + IP packets: 60265 of 61155 packets total + IP bytes: 74501838 (avg pkt size 1218 bytes) + Unique flows: 545 + TCP Packets: 21188 + UDP Packets: 39042 + VLAN Packets: 0 + MPLS Packets: 0 + PPPoE Packets: 0 + Fragmented Packets: 4 + Max Packet size: 64260 + Packet Len < 64: 30624 + Packet Len 64-128: 1158 + Packet Len 128-256: 674 + Packet Len 256-1024: 817 + Packet Len 1024-1500: 21924 + Packet Len > 1500: 5068 + nDPI throughput: 1.31 M pps / 12.31 Gb/sec + Traffic throughput: 228.03 pps / 2.19 Mb/sec + Traffic duration: 264.290 sec + Guessed flow protos: 9 + + +Detected protocols: + Unknown packets: 6126 bytes: 10913415 flows: 46 + FTP_CONTROL packets: 42 bytes: 4384 flows: 1 + DNS packets: 66 bytes: 6520 flows: 23 + IPP packets: 46 bytes: 12059 flows: 1 + HTTP packets: 377 bytes: 113616 flows: 24 + MDNS packets: 43 bytes: 11742 flows: 9 + NetBIOS packets: 48 bytes: 4754 flows: 12 + SSDP packets: 101 bytes: 32533 flows: 21 + DHCP packets: 21 bytes: 7346 flows: 2 + BitTorrent packets: 37639 bytes: 29656716 flows: 65 + ICMP packets: 15 bytes: 4956 flows: 5 + IGMP packets: 10 bytes: 600 flows: 9 + SSL packets: 1199 bytes: 622508 flows: 24 + ICMPV6 packets: 2 bytes: 172 flows: 2 + DHCPV6 packets: 1 bytes: 144 flows: 1 + Facebook packets: 349 bytes: 206444 flows: 21 + Twitter packets: 137 bytes: 45679 flows: 8 + Dropbox packets: 155 bytes: 45358 flows: 12 + GMail packets: 35 bytes: 10470 flows: 3 + YouTube packets: 11721 bytes: 31050728 flows: 33 + Skype packets: 237 bytes: 35082 flows: 30 + Google packets: 558 bytes: 164639 flows: 52 + WhatsApp packets: 969 bytes: 1477350 flows: 17 + Viber packets: 60 bytes: 27602 flows: 1 + LLMNR packets: 71 bytes: 5297 flows: 36 + Amazon packets: 89 bytes: 23572 flows: 9 + QUIC packets: 41 bytes: 3474 flows: 1 + BJNP packets: 72 bytes: 4320 flows: 72 + Microsoft packets: 35 bytes: 10358 flows: 5 + + +Protocol statistics: + Safe 632978 bytes + Acceptable 31693889 bytes + Fun 31257172 bytes + Unsafe 4384 bytes + Unrated 10913415 bytes + +----------------------------------------------------------- +* NOTE: This is demo app to show *some* nDPI features. +* In this demo we have implemented only some basic features +* just to show you what you can do with the library. Feel +* free to extend it and send us the patches for inclusion +------------------------------------------------------------ + +Using nDPI (1.8.0-dev-707-6a27b62) [1 thread(s)] +Reading packets from pcap file test.pcap... +Running thread 0... + +nDPI Memory statistics: + nDPI Memory (once): 110.55 KB + Flow Memory (per flow): 1.95 KB + Actual Memory: 3.46 MB + Peak Memory: 3.46 MB + +Traffic statistics: + Ethernet bytes: 75948198 (includes ethernet CRC/IFC/trailer) + Discarded bytes: 55256 + IP packets: 60265 of 61155 packets total + IP bytes: 74501838 (avg pkt size 1218 bytes) + Unique flows: 545 + TCP Packets: 21188 + UDP Packets: 39042 + VLAN Packets: 0 + MPLS Packets: 0 + PPPoE Packets: 0 + Fragmented Packets: 4 + Max Packet size: 64260 + Packet Len < 64: 30624 + Packet Len 64-128: 1158 + Packet Len 128-256: 674 + Packet Len 256-1024: 817 + Packet Len 1024-1500: 21924 + Packet Len > 1500: 5068 + nDPI throughput: 1.54 M pps / 14.45 Gb/sec + Traffic throughput: 228.03 pps / 2.19 Mb/sec + Traffic duration: 264.290 sec + Guessed flow protos: 9 + + +Detected protocols: + Unknown packets: 6126 bytes: 10913415 flows: 46 + FTP_CONTROL packets: 42 bytes: 4384 flows: 1 + DNS packets: 66 bytes: 6520 flows: 23 + IPP packets: 46 bytes: 12059 flows: 1 + HTTP packets: 377 bytes: 113616 flows: 24 + MDNS packets: 43 bytes: 11742 flows: 9 + NetBIOS packets: 48 bytes: 4754 flows: 12 + SSDP packets: 101 bytes: 32533 flows: 21 + DHCP packets: 21 bytes: 7346 flows: 2 + BitTorrent packets: 37639 bytes: 29656716 flows: 65 + ICMP packets: 15 bytes: 4956 flows: 5 + IGMP packets: 10 bytes: 600 flows: 9 + SSL packets: 1199 bytes: 622508 flows: 24 + ICMPV6 packets: 2 bytes: 172 flows: 2 + DHCPV6 packets: 1 bytes: 144 flows: 1 + Facebook packets: 349 bytes: 206444 flows: 21 + Twitter packets: 137 bytes: 45679 flows: 8 + Dropbox packets: 155 bytes: 45358 flows: 12 + GMail packets: 35 bytes: 10470 flows: 3 + YouTube packets: 11721 bytes: 31050728 flows: 33 + Skype packets: 237 bytes: 35082 flows: 30 + Google packets: 558 bytes: 164639 flows: 52 + WhatsApp packets: 969 bytes: 1477350 flows: 17 + Viber packets: 60 bytes: 27602 flows: 1 + LLMNR packets: 71 bytes: 5297 flows: 36 + Amazon packets: 89 bytes: 23572 flows: 9 + QUIC packets: 41 bytes: 3474 flows: 1 + BJNP packets: 72 bytes: 4320 flows: 72 + Microsoft packets: 35 bytes: 10358 flows: 5 + + +Protocol statistics: + Safe 632978 bytes + Acceptable 31693889 bytes + Fun 31257172 bytes + Unsafe 4384 bytes + Unrated 10913415 bytes + +----------------------------------------------------------- +* NOTE: This is demo app to show *some* nDPI features. +* In this demo we have implemented only some basic features +* just to show you what you can do with the library. Feel +* free to extend it and send us the patches for inclusion +------------------------------------------------------------ + +Using nDPI (1.8.0-dev-707-6a27b62) [1 thread(s)] +Reading packets from pcap file test.pcap... +Running thread 0... + +nDPI Memory statistics: + nDPI Memory (once): 110.55 KB + Flow Memory (per flow): 1.95 KB + Actual Memory: 3.46 MB + Peak Memory: 3.46 MB + +Traffic statistics: + Ethernet bytes: 75948198 (includes ethernet CRC/IFC/trailer) + Discarded bytes: 55256 + IP packets: 60265 of 61155 packets total + IP bytes: 74501838 (avg pkt size 1218 bytes) + Unique flows: 545 + TCP Packets: 21188 + UDP Packets: 39042 + VLAN Packets: 0 + MPLS Packets: 0 + PPPoE Packets: 0 + Fragmented Packets: 4 + Max Packet size: 64260 + Packet Len < 64: 30624 + Packet Len 64-128: 1158 + Packet Len 128-256: 674 + Packet Len 256-1024: 817 + Packet Len 1024-1500: 21924 + Packet Len > 1500: 5068 + nDPI throughput: 1.35 M pps / 12.71 Gb/sec + Traffic throughput: 228.03 pps / 2.19 Mb/sec + Traffic duration: 264.290 sec + Guessed flow protos: 9 + + +Detected protocols: + Unknown packets: 6126 bytes: 10913415 flows: 46 + FTP_CONTROL packets: 42 bytes: 4384 flows: 1 + DNS packets: 66 bytes: 6520 flows: 23 + IPP packets: 46 bytes: 12059 flows: 1 + HTTP packets: 377 bytes: 113616 flows: 24 + MDNS packets: 43 bytes: 11742 flows: 9 + NetBIOS packets: 48 bytes: 4754 flows: 12 + SSDP packets: 101 bytes: 32533 flows: 21 + DHCP packets: 21 bytes: 7346 flows: 2 + BitTorrent packets: 37639 bytes: 29656716 flows: 65 + ICMP packets: 15 bytes: 4956 flows: 5 + IGMP packets: 10 bytes: 600 flows: 9 + SSL packets: 1199 bytes: 622508 flows: 24 + ICMPV6 packets: 2 bytes: 172 flows: 2 + DHCPV6 packets: 1 bytes: 144 flows: 1 + Facebook packets: 349 bytes: 206444 flows: 21 + Twitter packets: 137 bytes: 45679 flows: 8 + Dropbox packets: 155 bytes: 45358 flows: 12 + GMail packets: 35 bytes: 10470 flows: 3 + YouTube packets: 11721 bytes: 31050728 flows: 33 + Skype packets: 237 bytes: 35082 flows: 30 + Google packets: 558 bytes: 164639 flows: 52 + WhatsApp packets: 969 bytes: 1477350 flows: 17 + Viber packets: 60 bytes: 27602 flows: 1 + LLMNR packets: 71 bytes: 5297 flows: 36 + Amazon packets: 89 bytes: 23572 flows: 9 + QUIC packets: 41 bytes: 3474 flows: 1 + BJNP packets: 72 bytes: 4320 flows: 72 + Microsoft packets: 35 bytes: 10358 flows: 5 + + +Protocol statistics: + Safe 632978 bytes + Acceptable 31693889 bytes + Fun 31257172 bytes + Unsafe 4384 bytes + Unrated 10913415 bytes + +----------------------------------------------------------- +* NOTE: This is demo app to show *some* nDPI features. +* In this demo we have implemented only some basic features +* just to show you what you can do with the library. Feel +* free to extend it and send us the patches for inclusion +------------------------------------------------------------ + +Using nDPI (1.8.0-dev-707-6a27b62) [1 thread(s)] +Reading packets from pcap file test.pcap... +Running thread 0... + +nDPI Memory statistics: + nDPI Memory (once): 110.55 KB + Flow Memory (per flow): 1.95 KB + Actual Memory: 3.46 MB + Peak Memory: 3.46 MB + +Traffic statistics: + Ethernet bytes: 75948198 (includes ethernet CRC/IFC/trailer) + Discarded bytes: 55256 + IP packets: 60265 of 61155 packets total + IP bytes: 74501838 (avg pkt size 1218 bytes) + Unique flows: 545 + TCP Packets: 21188 + UDP Packets: 39042 + VLAN Packets: 0 + MPLS Packets: 0 + PPPoE Packets: 0 + Fragmented Packets: 4 + Max Packet size: 64260 + Packet Len < 64: 30624 + Packet Len 64-128: 1158 + Packet Len 128-256: 674 + Packet Len 256-1024: 817 + Packet Len 1024-1500: 21924 + Packet Len > 1500: 5068 + nDPI throughput: 1.55 M pps / 14.53 Gb/sec + Traffic throughput: 228.03 pps / 2.19 Mb/sec + Traffic duration: 264.290 sec + Guessed flow protos: 9 + + +Detected protocols: + Unknown packets: 6126 bytes: 10913415 flows: 46 + FTP_CONTROL packets: 42 bytes: 4384 flows: 1 + DNS packets: 66 bytes: 6520 flows: 23 + IPP packets: 46 bytes: 12059 flows: 1 + HTTP packets: 377 bytes: 113616 flows: 24 + MDNS packets: 43 bytes: 11742 flows: 9 + NetBIOS packets: 48 bytes: 4754 flows: 12 + SSDP packets: 101 bytes: 32533 flows: 21 + DHCP packets: 21 bytes: 7346 flows: 2 + BitTorrent packets: 37639 bytes: 29656716 flows: 65 + ICMP packets: 15 bytes: 4956 flows: 5 + IGMP packets: 10 bytes: 600 flows: 9 + SSL packets: 1199 bytes: 622508 flows: 24 + ICMPV6 packets: 2 bytes: 172 flows: 2 + DHCPV6 packets: 1 bytes: 144 flows: 1 + Facebook packets: 349 bytes: 206444 flows: 21 + Twitter packets: 137 bytes: 45679 flows: 8 + Dropbox packets: 155 bytes: 45358 flows: 12 + GMail packets: 35 bytes: 10470 flows: 3 + YouTube packets: 11721 bytes: 31050728 flows: 33 + Skype packets: 237 bytes: 35082 flows: 30 + Google packets: 558 bytes: 164639 flows: 52 + WhatsApp packets: 969 bytes: 1477350 flows: 17 + Viber packets: 60 bytes: 27602 flows: 1 + LLMNR packets: 71 bytes: 5297 flows: 36 + Amazon packets: 89 bytes: 23572 flows: 9 + QUIC packets: 41 bytes: 3474 flows: 1 + BJNP packets: 72 bytes: 4320 flows: 72 + Microsoft packets: 35 bytes: 10358 flows: 5 + + +Protocol statistics: + Safe 632978 bytes + Acceptable 31693889 bytes + Fun 31257172 bytes + Unsafe 4384 bytes + Unrated 10913415 bytes + +----------------------------------------------------------- +* NOTE: This is demo app to show *some* nDPI features. +* In this demo we have implemented only some basic features +* just to show you what you can do with the library. Feel +* free to extend it and send us the patches for inclusion +------------------------------------------------------------ + +Using nDPI (1.8.0-dev-707-6a27b62) [1 thread(s)] +Reading packets from pcap file test.pcap... +Running thread 0... + +nDPI Memory statistics: + nDPI Memory (once): 110.55 KB + Flow Memory (per flow): 1.95 KB + Actual Memory: 3.46 MB + Peak Memory: 3.46 MB + +Traffic statistics: + Ethernet bytes: 75948198 (includes ethernet CRC/IFC/trailer) + Discarded bytes: 55256 + IP packets: 60265 of 61155 packets total + IP bytes: 74501838 (avg pkt size 1218 bytes) + Unique flows: 545 + TCP Packets: 21188 + UDP Packets: 39042 + VLAN Packets: 0 + MPLS Packets: 0 + PPPoE Packets: 0 + Fragmented Packets: 4 + Max Packet size: 64260 + Packet Len < 64: 30624 + Packet Len 64-128: 1158 + Packet Len 128-256: 674 + Packet Len 256-1024: 817 + Packet Len 1024-1500: 21924 + Packet Len > 1500: 5068 + nDPI throughput: 1.32 M pps / 12.42 Gb/sec + Traffic throughput: 228.03 pps / 2.19 Mb/sec + Traffic duration: 264.290 sec + Guessed flow protos: 9 + + +Detected protocols: + Unknown packets: 6126 bytes: 10913415 flows: 46 + FTP_CONTROL packets: 42 bytes: 4384 flows: 1 + DNS packets: 66 bytes: 6520 flows: 23 + IPP packets: 46 bytes: 12059 flows: 1 + HTTP packets: 377 bytes: 113616 flows: 24 + MDNS packets: 43 bytes: 11742 flows: 9 + NetBIOS packets: 48 bytes: 4754 flows: 12 + SSDP packets: 101 bytes: 32533 flows: 21 + DHCP packets: 21 bytes: 7346 flows: 2 + BitTorrent packets: 37639 bytes: 29656716 flows: 65 + ICMP packets: 15 bytes: 4956 flows: 5 + IGMP packets: 10 bytes: 600 flows: 9 + SSL packets: 1199 bytes: 622508 flows: 24 + ICMPV6 packets: 2 bytes: 172 flows: 2 + DHCPV6 packets: 1 bytes: 144 flows: 1 + Facebook packets: 349 bytes: 206444 flows: 21 + Twitter packets: 137 bytes: 45679 flows: 8 + Dropbox packets: 155 bytes: 45358 flows: 12 + GMail packets: 35 bytes: 10470 flows: 3 + YouTube packets: 11721 bytes: 31050728 flows: 33 + Skype packets: 237 bytes: 35082 flows: 30 + Google packets: 558 bytes: 164639 flows: 52 + WhatsApp packets: 969 bytes: 1477350 flows: 17 + Viber packets: 60 bytes: 27602 flows: 1 + LLMNR packets: 71 bytes: 5297 flows: 36 + Amazon packets: 89 bytes: 23572 flows: 9 + QUIC packets: 41 bytes: 3474 flows: 1 + BJNP packets: 72 bytes: 4320 flows: 72 + Microsoft packets: 35 bytes: 10358 flows: 5 + + +Protocol statistics: + Safe 632978 bytes + Acceptable 31693889 bytes + Fun 31257172 bytes + Unsafe 4384 bytes + Unrated 10913415 bytes + +----------------------------------------------------------- +* NOTE: This is demo app to show *some* nDPI features. +* In this demo we have implemented only some basic features +* just to show you what you can do with the library. Feel +* free to extend it and send us the patches for inclusion +------------------------------------------------------------ + +Using nDPI (1.8.0-dev-707-6a27b62) [1 thread(s)] +Reading packets from pcap file test.pcap... +Running thread 0... + +nDPI Memory statistics: + nDPI Memory (once): 110.55 KB + Flow Memory (per flow): 1.95 KB + Actual Memory: 3.46 MB + Peak Memory: 3.46 MB + +Traffic statistics: + Ethernet bytes: 75948198 (includes ethernet CRC/IFC/trailer) + Discarded bytes: 55256 + IP packets: 60265 of 61155 packets total + IP bytes: 74501838 (avg pkt size 1218 bytes) + Unique flows: 545 + TCP Packets: 21188 + UDP Packets: 39042 + VLAN Packets: 0 + MPLS Packets: 0 + PPPoE Packets: 0 + Fragmented Packets: 4 + Max Packet size: 64260 + Packet Len < 64: 30624 + Packet Len 64-128: 1158 + Packet Len 128-256: 674 + Packet Len 256-1024: 817 + Packet Len 1024-1500: 21924 + Packet Len > 1500: 5068 + nDPI throughput: 1.54 M pps / 14.46 Gb/sec + Traffic throughput: 228.03 pps / 2.19 Mb/sec + Traffic duration: 264.290 sec + Guessed flow protos: 9 + + +Detected protocols: + Unknown packets: 6126 bytes: 10913415 flows: 46 + FTP_CONTROL packets: 42 bytes: 4384 flows: 1 + DNS packets: 66 bytes: 6520 flows: 23 + IPP packets: 46 bytes: 12059 flows: 1 + HTTP packets: 377 bytes: 113616 flows: 24 + MDNS packets: 43 bytes: 11742 flows: 9 + NetBIOS packets: 48 bytes: 4754 flows: 12 + SSDP packets: 101 bytes: 32533 flows: 21 + DHCP packets: 21 bytes: 7346 flows: 2 + BitTorrent packets: 37639 bytes: 29656716 flows: 65 + ICMP packets: 15 bytes: 4956 flows: 5 + IGMP packets: 10 bytes: 600 flows: 9 + SSL packets: 1199 bytes: 622508 flows: 24 + ICMPV6 packets: 2 bytes: 172 flows: 2 + DHCPV6 packets: 1 bytes: 144 flows: 1 + Facebook packets: 349 bytes: 206444 flows: 21 + Twitter packets: 137 bytes: 45679 flows: 8 + Dropbox packets: 155 bytes: 45358 flows: 12 + GMail packets: 35 bytes: 10470 flows: 3 + YouTube packets: 11721 bytes: 31050728 flows: 33 + Skype packets: 237 bytes: 35082 flows: 30 + Google packets: 558 bytes: 164639 flows: 52 + WhatsApp packets: 969 bytes: 1477350 flows: 17 + Viber packets: 60 bytes: 27602 flows: 1 + LLMNR packets: 71 bytes: 5297 flows: 36 + Amazon packets: 89 bytes: 23572 flows: 9 + QUIC packets: 41 bytes: 3474 flows: 1 + BJNP packets: 72 bytes: 4320 flows: 72 + Microsoft packets: 35 bytes: 10358 flows: 5 + + +Protocol statistics: + Safe 632978 bytes + Acceptable 31693889 bytes + Fun 31257172 bytes + Unsafe 4384 bytes + Unrated 10913415 bytes + +----------------------------------------------------------- +* NOTE: This is demo app to show *some* nDPI features. +* In this demo we have implemented only some basic features +* just to show you what you can do with the library. Feel +* free to extend it and send us the patches for inclusion +------------------------------------------------------------ + +Using nDPI (1.8.0-dev-707-6a27b62) [1 thread(s)] +Reading packets from pcap file test.pcap... +Running thread 0... + +nDPI Memory statistics: + nDPI Memory (once): 110.55 KB + Flow Memory (per flow): 1.95 KB + Actual Memory: 3.46 MB + Peak Memory: 3.46 MB + +Traffic statistics: + Ethernet bytes: 75948198 (includes ethernet CRC/IFC/trailer) + Discarded bytes: 55256 + IP packets: 60265 of 61155 packets total + IP bytes: 74501838 (avg pkt size 1218 bytes) + Unique flows: 545 + TCP Packets: 21188 + UDP Packets: 39042 + VLAN Packets: 0 + MPLS Packets: 0 + PPPoE Packets: 0 + Fragmented Packets: 4 + Max Packet size: 64260 + Packet Len < 64: 30624 + Packet Len 64-128: 1158 + Packet Len 128-256: 674 + Packet Len 256-1024: 817 + Packet Len 1024-1500: 21924 + Packet Len > 1500: 5068 + nDPI throughput: 1.54 M pps / 14.49 Gb/sec + Traffic throughput: 228.03 pps / 2.19 Mb/sec + Traffic duration: 264.290 sec + Guessed flow protos: 9 + + +Detected protocols: + Unknown packets: 6126 bytes: 10913415 flows: 46 + FTP_CONTROL packets: 42 bytes: 4384 flows: 1 + DNS packets: 66 bytes: 6520 flows: 23 + IPP packets: 46 bytes: 12059 flows: 1 + HTTP packets: 377 bytes: 113616 flows: 24 + MDNS packets: 43 bytes: 11742 flows: 9 + NetBIOS packets: 48 bytes: 4754 flows: 12 + SSDP packets: 101 bytes: 32533 flows: 21 + DHCP packets: 21 bytes: 7346 flows: 2 + BitTorrent packets: 37639 bytes: 29656716 flows: 65 + ICMP packets: 15 bytes: 4956 flows: 5 + IGMP packets: 10 bytes: 600 flows: 9 + SSL packets: 1199 bytes: 622508 flows: 24 + ICMPV6 packets: 2 bytes: 172 flows: 2 + DHCPV6 packets: 1 bytes: 144 flows: 1 + Facebook packets: 349 bytes: 206444 flows: 21 + Twitter packets: 137 bytes: 45679 flows: 8 + Dropbox packets: 155 bytes: 45358 flows: 12 + GMail packets: 35 bytes: 10470 flows: 3 + YouTube packets: 11721 bytes: 31050728 flows: 33 + Skype packets: 237 bytes: 35082 flows: 30 + Google packets: 558 bytes: 164639 flows: 52 + WhatsApp packets: 969 bytes: 1477350 flows: 17 + Viber packets: 60 bytes: 27602 flows: 1 + LLMNR packets: 71 bytes: 5297 flows: 36 + Amazon packets: 89 bytes: 23572 flows: 9 + QUIC packets: 41 bytes: 3474 flows: 1 + BJNP packets: 72 bytes: 4320 flows: 72 + Microsoft packets: 35 bytes: 10358 flows: 5 + + +Protocol statistics: + Safe 632978 bytes + Acceptable 31693889 bytes + Fun 31257172 bytes + Unsafe 4384 bytes + Unrated 10913415 bytes + +----------------------------------------------------------- +* NOTE: This is demo app to show *some* nDPI features. +* In this demo we have implemented only some basic features +* just to show you what you can do with the library. Feel +* free to extend it and send us the patches for inclusion +------------------------------------------------------------ + +Using nDPI (1.8.0-dev-707-6a27b62) [1 thread(s)] +Reading packets from pcap file test.pcap... +Running thread 0... + +nDPI Memory statistics: + nDPI Memory (once): 110.55 KB + Flow Memory (per flow): 1.95 KB + Actual Memory: 3.46 MB + Peak Memory: 3.46 MB + +Traffic statistics: + Ethernet bytes: 75948198 (includes ethernet CRC/IFC/trailer) + Discarded bytes: 55256 + IP packets: 60265 of 61155 packets total + IP bytes: 74501838 (avg pkt size 1218 bytes) + Unique flows: 545 + TCP Packets: 21188 + UDP Packets: 39042 + VLAN Packets: 0 + MPLS Packets: 0 + PPPoE Packets: 0 + Fragmented Packets: 4 + Max Packet size: 64260 + Packet Len < 64: 30624 + Packet Len 64-128: 1158 + Packet Len 128-256: 674 + Packet Len 256-1024: 817 + Packet Len 1024-1500: 21924 + Packet Len > 1500: 5068 + nDPI throughput: 1.37 M pps / 12.89 Gb/sec + Traffic throughput: 228.03 pps / 2.19 Mb/sec + Traffic duration: 264.290 sec + Guessed flow protos: 9 + + +Detected protocols: + Unknown packets: 6126 bytes: 10913415 flows: 46 + FTP_CONTROL packets: 42 bytes: 4384 flows: 1 + DNS packets: 66 bytes: 6520 flows: 23 + IPP packets: 46 bytes: 12059 flows: 1 + HTTP packets: 377 bytes: 113616 flows: 24 + MDNS packets: 43 bytes: 11742 flows: 9 + NetBIOS packets: 48 bytes: 4754 flows: 12 + SSDP packets: 101 bytes: 32533 flows: 21 + DHCP packets: 21 bytes: 7346 flows: 2 + BitTorrent packets: 37639 bytes: 29656716 flows: 65 + ICMP packets: 15 bytes: 4956 flows: 5 + IGMP packets: 10 bytes: 600 flows: 9 + SSL packets: 1199 bytes: 622508 flows: 24 + ICMPV6 packets: 2 bytes: 172 flows: 2 + DHCPV6 packets: 1 bytes: 144 flows: 1 + Facebook packets: 349 bytes: 206444 flows: 21 + Twitter packets: 137 bytes: 45679 flows: 8 + Dropbox packets: 155 bytes: 45358 flows: 12 + GMail packets: 35 bytes: 10470 flows: 3 + YouTube packets: 11721 bytes: 31050728 flows: 33 + Skype packets: 237 bytes: 35082 flows: 30 + Google packets: 558 bytes: 164639 flows: 52 + WhatsApp packets: 969 bytes: 1477350 flows: 17 + Viber packets: 60 bytes: 27602 flows: 1 + LLMNR packets: 71 bytes: 5297 flows: 36 + Amazon packets: 89 bytes: 23572 flows: 9 + QUIC packets: 41 bytes: 3474 flows: 1 + BJNP packets: 72 bytes: 4320 flows: 72 + Microsoft packets: 35 bytes: 10358 flows: 5 + + +Protocol statistics: + Safe 632978 bytes + Acceptable 31693889 bytes + Fun 31257172 bytes + Unsafe 4384 bytes + Unrated 10913415 bytes + +----------------------------------------------------------- +* NOTE: This is demo app to show *some* nDPI features. +* In this demo we have implemented only some basic features +* just to show you what you can do with the library. Feel +* free to extend it and send us the patches for inclusion +------------------------------------------------------------ + +Using nDPI (1.8.0-dev-707-6a27b62) [1 thread(s)] +Reading packets from pcap file test.pcap... +Running thread 0... + +nDPI Memory statistics: + nDPI Memory (once): 110.55 KB + Flow Memory (per flow): 1.95 KB + Actual Memory: 3.46 MB + Peak Memory: 3.46 MB + +Traffic statistics: + Ethernet bytes: 75948198 (includes ethernet CRC/IFC/trailer) + Discarded bytes: 55256 + IP packets: 60265 of 61155 packets total + IP bytes: 74501838 (avg pkt size 1218 bytes) + Unique flows: 545 + TCP Packets: 21188 + UDP Packets: 39042 + VLAN Packets: 0 + MPLS Packets: 0 + PPPoE Packets: 0 + Fragmented Packets: 4 + Max Packet size: 64260 + Packet Len < 64: 30624 + Packet Len 64-128: 1158 + Packet Len 128-256: 674 + Packet Len 256-1024: 817 + Packet Len 1024-1500: 21924 + Packet Len > 1500: 5068 + nDPI throughput: 1.55 M pps / 14.57 Gb/sec + Traffic throughput: 228.03 pps / 2.19 Mb/sec + Traffic duration: 264.290 sec + Guessed flow protos: 9 + + +Detected protocols: + Unknown packets: 6126 bytes: 10913415 flows: 46 + FTP_CONTROL packets: 42 bytes: 4384 flows: 1 + DNS packets: 66 bytes: 6520 flows: 23 + IPP packets: 46 bytes: 12059 flows: 1 + HTTP packets: 377 bytes: 113616 flows: 24 + MDNS packets: 43 bytes: 11742 flows: 9 + NetBIOS packets: 48 bytes: 4754 flows: 12 + SSDP packets: 101 bytes: 32533 flows: 21 + DHCP packets: 21 bytes: 7346 flows: 2 + BitTorrent packets: 37639 bytes: 29656716 flows: 65 + ICMP packets: 15 bytes: 4956 flows: 5 + IGMP packets: 10 bytes: 600 flows: 9 + SSL packets: 1199 bytes: 622508 flows: 24 + ICMPV6 packets: 2 bytes: 172 flows: 2 + DHCPV6 packets: 1 bytes: 144 flows: 1 + Facebook packets: 349 bytes: 206444 flows: 21 + Twitter packets: 137 bytes: 45679 flows: 8 + Dropbox packets: 155 bytes: 45358 flows: 12 + GMail packets: 35 bytes: 10470 flows: 3 + YouTube packets: 11721 bytes: 31050728 flows: 33 + Skype packets: 237 bytes: 35082 flows: 30 + Google packets: 558 bytes: 164639 flows: 52 + WhatsApp packets: 969 bytes: 1477350 flows: 17 + Viber packets: 60 bytes: 27602 flows: 1 + LLMNR packets: 71 bytes: 5297 flows: 36 + Amazon packets: 89 bytes: 23572 flows: 9 + QUIC packets: 41 bytes: 3474 flows: 1 + BJNP packets: 72 bytes: 4320 flows: 72 + Microsoft packets: 35 bytes: 10358 flows: 5 + + +Protocol statistics: + Safe 632978 bytes + Acceptable 31693889 bytes + Fun 31257172 bytes + Unsafe 4384 bytes + Unrated 10913415 bytes
\ No newline at end of file diff --git a/tests/data/external/ramspeed/Floatmem b/tests/data/external/ramspeed/Floatmem new file mode 100644 index 00000000..f3cc1e6d --- /dev/null +++ b/tests/data/external/ramspeed/Floatmem @@ -0,0 +1,52 @@ +RAMspeed/SMP (Linux) v3.5.0 by Rhett M. Hollander and Paul V. Bolotoff, 2002-09 + +8Gb per pass mode, 1 processes + +5-benchmark FLOATmem BatchRun mode + +Benchmark #1: +FL-POINT Copy: 8220.47 MB/s +FL-POINT Scale: 8195.78 MB/s +FL-POINT Add: 10319.18 MB/s +FL-POINT Triad: 10403.87 MB/s +--- +FL-POINT AVERAGE: 9284.83 MB/s + +Benchmark #2: +FL-POINT Copy: 8086.66 MB/s +FL-POINT Scale: 7842.82 MB/s +FL-POINT Add: 9900.13 MB/s +FL-POINT Triad: 10029.43 MB/s +--- +FL-POINT AVERAGE: 8964.76 MB/s + +Benchmark #3: +FL-POINT Copy: 7766.98 MB/s +FL-POINT Scale: 7837.62 MB/s +FL-POINT Add: 9827.57 MB/s +FL-POINT Triad: 9918.89 MB/s +--- +FL-POINT AVERAGE: 8837.77 MB/s + +Benchmark #4: +FL-POINT Copy: 7647.89 MB/s +FL-POINT Scale: 7594.00 MB/s +FL-POINT Add: 9873.93 MB/s +FL-POINT Triad: 9970.89 MB/s +--- +FL-POINT AVERAGE: 8771.68 MB/s + +Benchmark #5: +FL-POINT Copy: 7821.21 MB/s +FL-POINT Scale: 7880.56 MB/s +FL-POINT Add: 9914.84 MB/s +FL-POINT Triad: 10044.05 MB/s +--- +FL-POINT AVERAGE: 8915.17 MB/s + +FL-POINT BatchRun Copy: 7908.64 MB/s +FL-POINT BatchRun Scale: 7870.16 MB/s +FL-POINT BatchRun Add: 9967.13 MB/s +FL-POINT BatchRun Triad: 10073.43 MB/s +--- +FL-POINT BatchRun AVERAGE: 8954.84 MB/s
\ No newline at end of file diff --git a/tests/data/external/ramspeed/Intmem b/tests/data/external/ramspeed/Intmem new file mode 100644 index 00000000..08fffe47 --- /dev/null +++ b/tests/data/external/ramspeed/Intmem @@ -0,0 +1,52 @@ +RAMspeed/SMP (Linux) v3.5.0 by Rhett M. Hollander and Paul V. Bolotoff, 2002-09 + +8Gb per pass mode, 1 processes + +5-benchmark INTmem BatchRun mode + +Benchmark #1: +INTEGER Copy: 11493.80 MB/s +INTEGER Scale: 11527.61 MB/s +INTEGER Add: 11567.60 MB/s +INTEGER Triad: 11546.06 MB/s +--- +INTEGER AVERAGE: 11533.77 MB/s + +Benchmark #2: +INTEGER Copy: 11580.06 MB/s +INTEGER Scale: 11617.25 MB/s +INTEGER Add: 11748.95 MB/s +INTEGER Triad: 11837.43 MB/s +--- +INTEGER AVERAGE: 11695.92 MB/s + +Benchmark #3: +INTEGER Copy: 11674.45 MB/s +INTEGER Scale: 11566.59 MB/s +INTEGER Add: 11643.59 MB/s +INTEGER Triad: 11633.42 MB/s +--- +INTEGER AVERAGE: 11629.51 MB/s + +Benchmark #4: +INTEGER Copy: 11496.38 MB/s +INTEGER Scale: 11528.06 MB/s +INTEGER Add: 11641.10 MB/s +INTEGER Triad: 11742.61 MB/s +--- +INTEGER AVERAGE: 11602.04 MB/s + +Benchmark #5: +INTEGER Copy: 11568.45 MB/s +INTEGER Scale: 11579.32 MB/s +INTEGER Add: 11646.55 MB/s +INTEGER Triad: 11596.57 MB/s +--- +INTEGER AVERAGE: 11597.72 MB/s + +INTEGER BatchRun Copy: 11562.63 MB/s +INTEGER BatchRun Scale: 11563.77 MB/s +INTEGER BatchRun Add: 11649.55 MB/s +INTEGER BatchRun Triad: 11671.22 MB/s +--- +INTEGER BatchRun AVERAGE: 11611.79 MB/s
\ No newline at end of file diff --git a/tests/data/external/ssl/AES-128-CBC_dump b/tests/data/external/ssl/AES-128-CBC_dump new file mode 100644 index 00000000..32568e14 --- /dev/null +++ b/tests/data/external/ssl/AES-128-CBC_dump @@ -0,0 +1,7 @@ +OpenSSL 1.0.2f 28 Jan 2016 +built on: reproducible build, date unspecified +options:bn(64,64) rc4(16x,int) des(idx,cisc,16,int) aes(partial) idea(int) blowfish(idx) +compiler: gcc -I. -I.. -I../include -DOPENSSL_THREADS -D_REENTRANT -DDSO_DLFCN -DHAVE_DLFCN_H -Wa,--noexecstack -m64 -DL_ENDIAN -O3 -Wall -DOPENSSL_IA32_SSE2 -DOPENSSL_BN_ASM_MONT -DOPENSSL_BN_ASM_MONT5 -DOPENSSL_BN_ASM_GF2m -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DMD5_ASM -DAES_ASM -DVPAES_ASM -DBSAES_ASM -DWHIRLPOOL_ASM -DGHASH_ASM -DECP_NISTZ256_ASM +The 'numbers' are in 1000s of bytes per second processed. +type 16 bytes 64 bytes 256 bytes 1024 bytes 8192 bytes +aes-128-cbc 533103.05k 570042.22k 580021.25k 584568.83k 599470.83k
\ No newline at end of file diff --git a/tests/data/external/ssl/RSA_dump b/tests/data/external/ssl/RSA_dump new file mode 100644 index 00000000..55e40792 --- /dev/null +++ b/tests/data/external/ssl/RSA_dump @@ -0,0 +1,9 @@ +OpenSSL 1.0.2f 28 Jan 2016 +built on: reproducible build, date unspecified +options:bn(64,64) rc4(16x,int) des(idx,cisc,16,int) aes(partial) idea(int) blowfish(idx) +compiler: gcc -I. -I.. -I../include -DOPENSSL_THREADS -D_REENTRANT -DDSO_DLFCN -DHAVE_DLFCN_H -Wa,--noexecstack -m64 -DL_ENDIAN -O3 -Wall -DOPENSSL_IA32_SSE2 -DOPENSSL_BN_ASM_MONT -DOPENSSL_BN_ASM_MONT5 -DOPENSSL_BN_ASM_GF2m -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DMD5_ASM -DAES_ASM -DVPAES_ASM -DBSAES_ASM -DWHIRLPOOL_ASM -DGHASH_ASM -DECP_NISTZ256_ASM + sign verify sign/s verify/s +rsa 512 bits 0.000056s 0.000004s 17828.4 222903.5 +rsa 1024 bits 0.000169s 0.000011s 5923.9 88397.9 +rsa 2048 bits 0.000793s 0.000037s 1261.4 26951.3 +rsa 4096 bits 0.008280s 0.000131s 120.8 7633.7 diff --git a/tests/data/reporter/timeline.pickle b/tests/data/reporter/timeline.pickle new file mode 100644 index 00000000..5c870d93 --- /dev/null +++ b/tests/data/reporter/timeline.pickle @@ -0,0 +1,3 @@ +VTimeline\u000a\u000aMONITOR TIME\u000a\u000aT00 1\u000a\u000a\u000aINSPECTOR TIME\u000a\u000aT01 2\u000a\u000aT02 5\u000a\u000aT03 8\u000a\u000a\u000aCONTROLLER TIME\u000a\u000aT04 11\u000a\u000a\u000aNOTIFIER TIME\u000a\u000aT05 16\u000a\u000a\u000aEVALUATOR TIME\u000a\u000aT06 40\u000a\u000a\u000aTotal: 312ms +p0 +. diff --git a/tests/unit/cli/cmd_report.py b/tests/unit/cli/cmd_report.py new file mode 100644 index 00000000..e010b960 --- /dev/null +++ b/tests/unit/cli/cmd_report.py @@ -0,0 +1,23 @@ +############################################################### +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import pytest +from click.testing import CliRunner + +from qtip.cli.entry import cli + + +@pytest.fixture(scope="module") +def runner(): + return CliRunner() + + +def test_show(runner): + result = runner.invoke(cli, ['report', 'show']) + assert result.output == '' diff --git a/tests/unit/cli/options_test.py b/tests/unit/cli/options_test.py index f9472814..9dbbe6f3 100644 --- a/tests/unit/cli/options_test.py +++ b/tests/unit/cli/options_test.py @@ -1,5 +1,5 @@ ############################################################### -# Copyright (c) 2016 ZTE Corp and others. +# Copyright (c) 2017 taseer94@gmail.com and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -8,8 +8,9 @@ ############################################################################## import pytest -from click.testing import CliRunner +import sys +from click.testing import CliRunner from qtip.cli.entry import cli @@ -28,5 +29,5 @@ class TestClass(object): assert 'dev' in result.output def test_debug(self, runner): - result = runner.invoke(cli, ['-d']) - assert '' in result.output + runner.invoke(cli, ['-d']) + assert sys.tracebacklimit == 8 diff --git a/tests/unit/reporter/console_test.py b/tests/unit/reporter/console_test.py index 8150239e..d2816690 100644 --- a/tests/unit/reporter/console_test.py +++ b/tests/unit/reporter/console_test.py @@ -7,7 +7,10 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +import pickle import pytest +import os + from qtip.reporter.console import ConsoleReporter @@ -21,9 +24,16 @@ def test_constructor(console_reporter): def test_render(console_reporter): - var_dict = { - 'title': 'fake title', - 'description': 'fake description' - } - output = console_reporter.render(var_dict=var_dict) - assert output == 'fake title: fake description' + var_dict = {'title': 'Timeline', 'total': '312ms', 'phases': [{'name': 'Monitor ', + 'checkpoints': [{'name': 'T00 ', 'timestamp': '1'}]}, + {'name': 'Inspector ', 'checkpoints': [{'name': 'T01 ', 'timestamp': '2'}, + {'name': 'T02 ', 'timestamp': '5'}, {'name': 'T03 ', 'timestamp': '8'}]}, + {'name': 'Controller ', 'checkpoints': [{'name': 'T04 ', 'timestamp': '11'}]}, + {'name': 'Notifier ', 'checkpoints': [{'name': 'T05 ', 'timestamp': '16'}]}, + {'name': 'Evaluator ', 'checkpoints': [{'name': 'T06 ', 'timestamp': '40'}]}]} + + result = console_reporter.render(var_dict=var_dict) + path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, + os.pardir, 'tests/data/reporter/') + timeline = pickle.load(open(path + 'timeline.pickle', 'rb')) + assert result == timeline diff --git a/tests/unit/util/env_test.py b/tests/unit/util/env_test.py index 38ac988b..62d12a13 100644 --- a/tests/unit/util/env_test.py +++ b/tests/unit/util/env_test.py @@ -9,10 +9,40 @@ import time -import mock import pytest +import mock +from collections import defaultdict +import socket from qtip.util import env +from qtip.util.env import AnsibleEnvSetup + + +@pytest.fixture(scope='session') +def ansible_envsetup(): + return AnsibleEnvSetup() + + +@pytest.fixture() +def hostfile(tmpdir): + fake_hostfile = tmpdir.join('hosts') + fake_hostfile.write("[hosts]\n") + fake_hostfile.write("10.20.0.3") + return fake_hostfile + + +@pytest.fixture() +def private_key(tmpdir): + fake_private_key = tmpdir.join('QtipKey') + fake_private_key.write("fake keypair") + return fake_private_key + + +@pytest.fixture() +def public_key(tmpdir): + fake_public_key = tmpdir.join('QtipKey.pub') + fake_public_key.write("fake public key") + return fake_public_key def test_all_files_exist(tmpdir): @@ -35,60 +65,259 @@ def test_clean_file(tmpdir): assert env.clean_file(non_exist_file) -def test_generate_host_file_without_setenv(monkeypatch): - def setenv(*args): - monkeypatch.setenv('INSTALLER_TYPE', args[0]) - monkeypatch.setenv('INSTALLER_IP', args[1]) +def test_init(ansible_envsetup): + assert 'AnsibleEnvSetup' in str(type(ansible_envsetup)) + assert ansible_envsetup.keypair == defaultdict(str) + assert ansible_envsetup.hostfile is None + assert ansible_envsetup.host_ip_list == [] - with pytest.raises(KeyError) as excinfo: - env.generate_host_file() - assert 'INSTALLER_TYPE' in str(excinfo.value) - with pytest.raises(ValueError) as excinfo: - setenv('fuel_1', '10.20.0.2') - env.generate_host_file() - assert 'fuel_1 is not supported' in str(excinfo.value) +def test_setup_exception(capsys, mocker, ansible_envsetup, hostfile): + with mock.patch.object(AnsibleEnvSetup, 'check_hostfile', side_effect=RuntimeError()): + mock_os = mocker.patch('sys.exit') + ansible_envsetup.setup({'hostfile': str(hostfile)}) + out, error = capsys.readouterr() + assert out == '\n' + assert mock_os.call_count == 1 - with pytest.raises(ValueError) as excinfo: - setenv('fuel', '') - env.generate_host_file() - assert 'The value of environment variable INSTALLER_IP is empty' \ - in str(excinfo.value) + +# TODO(zhihui_wu) Need find a smart way to write this pytest +def test_setup(mocker, ansible_envsetup): + mock_check_hostfile = \ + mocker.patch.object(AnsibleEnvSetup, 'check_hostfile') + mock_generate_default_hostfile = \ + mocker.patch.object(AnsibleEnvSetup, 'generate_default_hostfile') + mock_fetch_ip = \ + mocker.patch.object(AnsibleEnvSetup, 'fetch_host_ip_from_hostfile') + mock_check_keypair = \ + mocker.patch.object(AnsibleEnvSetup, 'check_keypair') + mock_generate_default_keypair = \ + mocker.patch.object(AnsibleEnvSetup, 'generate_default_keypair') + mock_pass_keypair = \ + mocker.patch.object(AnsibleEnvSetup, 'pass_keypair_to_remote') + mock_check_ssh = \ + mocker.patch.object(AnsibleEnvSetup, 'check_hosts_ssh_connectivity') + + ansible_envsetup.setup({'keypair': str(private_key), + 'hostfile': str(hostfile)}) + mock_check_hostfile.assert_called_with(str(hostfile)) + mock_fetch_ip.assert_called_with() + mock_check_keypair.assert_called_with(str(private_key)) + mock_pass_keypair.assert_called_with() + mock_check_ssh.assert_called_with() + + ansible_envsetup.setup({'keypair': str(private_key)}) + mock_generate_default_hostfile.assert_called_with() + mock_fetch_ip.assert_called_with() + mock_check_keypair.assert_called_with(str(private_key)) + mock_pass_keypair.assert_called_with() + mock_check_ssh.assert_called_with() + + ansible_envsetup.setup({'hostfile': str(hostfile)}) + mock_check_hostfile.assert_called_with(str(hostfile)) + mock_fetch_ip.assert_called_with() + mock_generate_default_keypair.assert_called_with() + mock_pass_keypair.assert_called_with() + mock_check_ssh.assert_called_with() + + ansible_envsetup.setup() + mock_generate_default_hostfile.assert_called_with() + mock_fetch_ip.assert_called_with() + mock_generate_default_keypair.assert_called_with() + mock_pass_keypair.assert_called_with() + mock_check_ssh.assert_called_with() + + +def test_check_keypair(mocker, ansible_envsetup, private_key, public_key): + with mocker.patch.object(env, 'all_files_exist', return_value=True): + ansible_envsetup.check_keypair(str(private_key)) + assert ansible_envsetup.keypair['private'] == str(private_key) + assert ansible_envsetup.keypair['public'] == str(public_key) + + +def test_check_keypair_failed(mocker, ansible_envsetup): + mocker.patch.object(env, 'all_files_exist', return_value=False) + with pytest.raises(RuntimeError) as excinfo: + ansible_envsetup.check_keypair(str(private_key)) + assert 'The keypairs you in the configuration file ' \ + 'is invalid or not existed.' == str(excinfo.value) + assert ansible_envsetup.keypair['private'] == '' + assert ansible_envsetup.keypair['public'] == '' + + +@pytest.mark.parametrize("file_existence, expected", [ + (True, 0), + (False, 1) +]) +def test_generate_default_keypair(mocker, ansible_envsetup, file_existence, expected): + mock_os = mocker.patch('os.system') + mocker.patch.object(env, 'all_files_exist', return_value=file_existence) + ansible_envsetup.generate_default_keypair() + assert mock_os.call_count == expected + assert ansible_envsetup.keypair['private'] == env.PRIVATE_KEY + assert ansible_envsetup.keypair['public'] == env.PUBLIC_KEY + + +@pytest.mark.parametrize("ips, expected", [ + (['10.20.0.3'], 1), + (['10.20.0.3', '10.20.0.4'], 2) +]) +def test_pass_keypair_to_remote_successful(mocker, ansible_envsetup, ips, expected): + ansible_envsetup.host_ip_list = ips + mock_pass_keypair = \ + mocker.patch.object(AnsibleEnvSetup, '_pass_keypair', return_value=True) + ansible_envsetup.pass_keypair_to_remote() + assert mock_pass_keypair.call_count == expected + + +def test_pass_keypair_to_remote_failed(mocker, ansible_envsetup): + ansible_envsetup.host_ip_list = ['10.20.0.3'] + mocker.patch.object(AnsibleEnvSetup, '_pass_keypair', return_value=False) + with pytest.raises(RuntimeError) as excinfo: + ansible_envsetup.pass_keypair_to_remote() + assert "Failed on passing keypair to remote." in str(excinfo.value) -def test_generate_host_file(monkeypatch, tmpdir): +def test_pass_keypair(monkeypatch, capsys, mocker, ansible_envsetup): + monkeypatch.setattr(time, 'sleep', lambda s: None) + mock_os = mocker.patch('os.system') + ansible_envsetup._pass_keypair('10.20.0.3', str(private_key)) + assert mock_os.call_count == 2 + out, error = capsys.readouterr() + assert "Pass keypair to remote hosts 10.20.0.3 successfully" in out + + +def test_pass_keypair_exception(capsys, ansible_envsetup): + with mock.patch('os.system', side_effect=Exception()) as mock_os: + result = ansible_envsetup._pass_keypair('10.20.0.3', str(private_key)) + assert result is False + out, error = capsys.readouterr() + assert out == '\n' + assert mock_os.call_count == 1 + + +def test_check_hostfile(mocker, ansible_envsetup, hostfile): + ansible_envsetup.check_hostfile(str(hostfile)) + assert ansible_envsetup.hostfile == str(hostfile) + + with pytest.raises(RuntimeError) as excinfo: + mocker.patch.object(env, 'all_files_exist', return_value=False) + ansible_envsetup.check_hostfile(str(hostfile)) + assert str(excinfo.value) == 'The hostfile {0} is invalid or not ' \ + 'existed.'.format(str(hostfile)) + + +def test_default_hostfile_non_existed(mocker, ansible_envsetup): + with mocker.patch.object(env, 'all_files_exist', return_value=False): + mock_generate_hostfile_via_installer = \ + mocker.patch.object(AnsibleEnvSetup, + '_generate_hostfile_via_installer') + ansible_envsetup.generate_default_hostfile() + mock_generate_hostfile_via_installer.assert_called_once_with() + + +def test_default_hostfile_existed(mocker, ansible_envsetup): + with mocker.patch.object(env, 'all_files_exist', return_value=True): + mock_generate_hostfile_via_installer = \ + mocker.patch.object(AnsibleEnvSetup, + '_generate_hostfile_via_installer') + ansible_envsetup.generate_default_hostfile() + mock_generate_hostfile_via_installer.assert_not_called() + + +@pytest.mark.parametrize("test_input, expected", [ + (({}, KeyError), 'INSTALLER_TYPE'), + (({'INSTALLER_TYPE': 'fuel'}, KeyError), 'INSTALLER_IP'), + (({'INSTALLER_TYPE': 'fuel_1', 'INSTALLER_IP': '10.20.0.2'}, ValueError), + 'fuel_1 is not supported'), + (({'INSTALLER_TYPE': 'fuel', 'INSTALLER_IP': ''}, ValueError), + 'The value of environment variable INSTALLER_IP is empty') +]) +def test_generate_hostfile_via_installer_exception(monkeypatch, ansible_envsetup, test_input, expected): + if test_input[0]: + for key in test_input[0]: + monkeypatch.setenv(key, test_input[0][key]) + + with pytest.raises(test_input[1]) as excinfo: + ansible_envsetup._generate_hostfile_via_installer() + assert expected in str(excinfo.value) + + +def test_generate_hostfile_via_installer(monkeypatch, mocker, ansible_envsetup): monkeypatch.setenv('INSTALLER_TYPE', 'fuel') monkeypatch.setenv('INSTALLER_IP', '10.20.0.2') - hostfile = tmpdir.mkdir('qtip').join('hosts') - hostfile.write('') - assert env.generate_host_file(str(hostfile)) + mock_os = mocker.patch('os.system') + ansible_envsetup._generate_hostfile_via_installer() + assert mock_os.call_count == 1 + assert ansible_envsetup.hostfile == env.HOST_FILE -def test_generate_keypair(): - with mock.patch('os.system') as mock_os: - env.generate_keypair() - assert mock_os.call_count == 1 +def test_fetch_host_ip_from_hostfile(ansible_envsetup, hostfile): + ansible_envsetup.hostfile = str(hostfile) + ansible_envsetup.fetch_host_ip_from_hostfile() + assert ansible_envsetup.host_ip_list == ['10.20.0.3'] -def test_pass_keypair(monkeypatch): - monkeypatch.setattr(time, 'sleep', lambda s: None) - with mock.patch('os.system') as mock_os: - env.pass_keypair('10.20.0.10') - assert mock_os.call_count == 2 +def test_fetch_host_ip_from_empty_hostfile(ansible_envsetup, tmpdir): + empty_hostfile = tmpdir.join('empty_hostfile') + empty_hostfile.write("") + ansible_envsetup.hostfile = str(empty_hostfile) + with pytest.raises(ValueError) as excinfo: + ansible_envsetup.fetch_host_ip_from_hostfile() + assert str(excinfo.value) == "The hostfile doesn't include host ip addresses." + + +@pytest.mark.parametrize("ips, expected", [ + (['10.20.0.3'], 1), + (['10.20.0.3', '10.20.0.4'], 2) +]) +def test_check_hosts_ssh_connectivity(mocker, ansible_envsetup, ips, expected): + ansible_envsetup.host_ip_list = ips + mock_ssh_is_ok = \ + mocker.patch.object(AnsibleEnvSetup, '_ssh_is_ok', return_value=True) + ansible_envsetup.check_hosts_ssh_connectivity() + assert mock_ssh_is_ok.call_count == expected + + +def test_check_hosts_ssh_connectivity_failed(mocker, ansible_envsetup): + ansible_envsetup.host_ip_list = ['10.20.0.3'] + mocker.patch.object(AnsibleEnvSetup, '_ssh_is_ok', return_value=False) + with pytest.raises(RuntimeError) as excinfo: + ansible_envsetup.check_hosts_ssh_connectivity() + assert "Failed on checking hosts ssh connectivity." == str(excinfo.value) @pytest.mark.parametrize("stderrinfo, expected", [ ('', True), ('sorry', False) ]) -@mock.patch('paramiko.SSHClient') -def test_ssh_is_ok(mock_sshclient, stderrinfo, expected): +def test_ssh_is_ok(mocker, ansible_envsetup, private_key, stderrinfo, expected): stderr = mock.MagicMock() stderr.readlines.return_value = stderrinfo + mock_sshclient = mocker.patch('paramiko.SSHClient') test_ssh_client = mock_sshclient.return_value test_ssh_client.exec_command.return_value = ('', '', stderr) - result = env.ssh_is_ok('10.20.0.3') + result = ansible_envsetup._ssh_is_ok('10.20.0.3', str(private_key)) assert result == expected test_ssh_client.connect.assert_called_once_with( - '10.20.0.3', key_filename=env.PRIVATE_KEY) + '10.20.0.3', key_filename=str(private_key)) test_ssh_client.exec_command.assert_called_with('uname') + + +@pytest.mark.parametrize("attempts, expected", [ + (1, + 'Check hosts 10.20.0.3 ssh connectivity...\n1 times ssh test......failed.\n'), + (2, + 'Check hosts 10.20.0.3 ssh connectivity...\n' + '1 times ssh test......failed.\n' + '2 times ssh test......failed.\n') +]) +def test_ssh_exception(capsys, monkeypatch, mocker, ansible_envsetup, attempts, expected): + monkeypatch.setattr(time, 'sleep', lambda s: None) + mock_sshclient = mocker.patch('paramiko.SSHClient') + test_ssh_client = mock_sshclient.return_value + test_ssh_client.exec_command.side_effect = socket.error() + result = ansible_envsetup._ssh_is_ok('10.20.0.3', str(private_key), attempts=attempts) + out, error = capsys.readouterr() + assert expected == out + assert result is False diff --git a/third-party/License/.gitrepo b/third-party/License/.gitrepo index 29cd5a75..03fab6f7 100644 --- a/third-party/License/.gitrepo +++ b/third-party/License/.gitrepo @@ -5,7 +5,7 @@ ; [subrepo] remote = git@github.com:openzero-zte/License.git - branch = master - commit = 61489dae4453b66887d0d90a2244610a30f7e53c - parent = 3e443dff14a2be02b914e66f27b549d0ed4cc600 + branch = develop + commit = 88b1440008b6acac8ad65afc93606cddef63cea9 + parent = 473deae9a10162f000c49ca49b4e31b28c4bf0d8 cmdver = 0.3.0 diff --git a/third-party/License/README.md b/third-party/License/README.md index 0232de7e..57444c1f 100644 --- a/third-party/License/README.md +++ b/third-party/License/README.md @@ -5,6 +5,7 @@ A script for checking and adding license header according to [OPNFV contribution ## Quick Start ``` -$ cd <project-folder> -$ curl https://raw.githubusercontent.com/Justin-chi/License/master/add_license.sh |bash +$ cd <project-root> +$ curl https://raw.githubusercontent.com/openzero-team/License/develop/add_license.sh |bash ``` + |