summaryrefslogtreecommitdiffstats
path: root/utils/test
diff options
context:
space:
mode:
authorSerena Feng <feng.xiaowei@zte.com.cn>2016-10-12 00:49:41 +0000
committerGerrit Code Review <gerrit@opnfv.org>2016-10-12 00:49:41 +0000
commitd9ab655653e2d343a8eac7560c50c6bed632a09f (patch)
tree5abb194dad3bacfb3d5ee3fe8c6f79735b654305 /utils/test
parent9955bf7bbdd2f9a51f710d1a129daf99e6b10cda (diff)
parent7b460351f5eed62696c174191e6a1c3d095fe574 (diff)
Merge "visualize qpi using kibana_dashboard"
Diffstat (limited to 'utils/test')
-rw-r--r--utils/test/dashboard/dashboard/elastic2kibana/templates/qpi.json45
-rw-r--r--utils/test/dashboard/dashboard/mongo2elastic/format.py18
-rw-r--r--utils/test/dashboard/dashboard/mongo2elastic/main.py10
-rw-r--r--utils/test/dashboard/dashboard/qtip/__init__.py0
-rw-r--r--utils/test/dashboard/dashboard/qtip/testcases.yaml28
5 files changed, 99 insertions, 2 deletions
diff --git a/utils/test/dashboard/dashboard/elastic2kibana/templates/qpi.json b/utils/test/dashboard/dashboard/elastic2kibana/templates/qpi.json
new file mode 100644
index 000000000..a46f315ef
--- /dev/null
+++ b/utils/test/dashboard/dashboard/elastic2kibana/templates/qpi.json
@@ -0,0 +1,45 @@
+{% set aggs = aggs|default([]) -%}
+
+{
+ "title": "qpi",
+ "type": "line",
+ "listeners": {},
+ "params": {
+ "addLegend": true,
+ "shareYAxis": true,
+ "addTooltip": true,
+ "smoothLines": false,
+ "scale": "linear",
+ "interpolate": "linear",
+ "times": [],
+ "addTimeMarker": false,
+ "defaultYExtents": false,
+ "setYExtents": false,
+ "yAxis": {},
+ "mode": "stacked"
+ },
+ "aggs": [
+ {% for agg in aggs %}
+ {
+ "id": {{agg.id }},
+ "type": "avg",
+ "schema": "metric",
+ "params": {
+ "field": "{{agg.field}}"
+ }
+ },
+ {% endfor %}
+ {
+ "id": {{ aggs|length + 1 }},
+ "type": "date_histogram",
+ "schema": "segment",
+ "params": {
+ "field": "start_date",
+ "interval": "auto",
+ "customInterval": "2h",
+ "min_doc_count": 1,
+ "extended_bounds": {}
+ }
+ }
+ ]
+}
diff --git a/utils/test/dashboard/dashboard/mongo2elastic/format.py b/utils/test/dashboard/dashboard/mongo2elastic/format.py
index ef485bae0..0bbde1746 100644
--- a/utils/test/dashboard/dashboard/mongo2elastic/format.py
+++ b/utils/test/dashboard/dashboard/mongo2elastic/format.py
@@ -184,3 +184,21 @@ def format_vims(testcase):
}
}
return True
+
+
+def format_qpi(testcase):
+ """
+ Look for these and leave any of those:
+ details.index
+
+ If none are present, then return False
+ """
+ details = testcase['details']
+ if 'index' not in details:
+ return False
+
+ for key, value in details.items():
+ if key != 'index':
+ del details[key]
+
+ return True
diff --git a/utils/test/dashboard/dashboard/mongo2elastic/main.py b/utils/test/dashboard/dashboard/mongo2elastic/main.py
index a526d5319..303d82c6f 100644
--- a/utils/test/dashboard/dashboard/mongo2elastic/main.py
+++ b/utils/test/dashboard/dashboard/mongo2elastic/main.py
@@ -91,10 +91,16 @@ class DocumentVerification(object):
return self.skip
def _fix_date(self, date_string):
+ if date_string == 'None':
+ return None
if isinstance(date_string, dict):
return date_string['$date']
- else:
- return date_string[:-3].replace(' ', 'T') + 'Z'
+ if 'T' not in date_string:
+ date_string = date_string[:-3].replace(' ', 'T')
+ if not date_string.endswith('Z'):
+ date_string += 'Z'
+
+ return date_string
class DocumentPublisher(object):
diff --git a/utils/test/dashboard/dashboard/qtip/__init__.py b/utils/test/dashboard/dashboard/qtip/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/utils/test/dashboard/dashboard/qtip/__init__.py
diff --git a/utils/test/dashboard/dashboard/qtip/testcases.yaml b/utils/test/dashboard/dashboard/qtip/testcases.yaml
new file mode 100644
index 000000000..cd337cd73
--- /dev/null
+++ b/utils/test/dashboard/dashboard/qtip/testcases.yaml
@@ -0,0 +1,28 @@
+qtip:
+ -
+ name: compute_test_suite
+ format: qpi
+ test_family: compute
+ visualizations:
+ -
+ name: qpi
+ fields:
+ - field: details.index
+ -
+ name: network_test_suite
+ test_family: network
+ format: qpi
+ visualizations:
+ -
+ name: qpi
+ fields:
+ - field: details.index
+ -
+ name:storage_test_suite
+ format: qpi
+ test_family: storage
+ visualizations:
+ -
+ name: qpi
+ fields:
+ - field: details.index
.Double */ .highlight .se { color: #ae81ff } /* Literal.String.Escape */ .highlight .sh { color: #e6db74 } /* Literal.String.Heredoc */ .highlight .si { color: #e6db74 } /* Literal.String.Interpol */ .highlight .sx { color: #e6db74 } /* Literal.String.Other */ .highlight .sr { color: #e6db74 } /* Literal.String.Regex */ .highlight .s1 { color: #e6db74 } /* Literal.String.Single */ .highlight .ss { color: #e6db74 } /* Literal.String.Symbol */ .highlight .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #a6e22e } /* Name.Function.Magic */ .highlight .vc { color: #f8f8f2 } /* Name.Variable.Class */ .highlight .vg { color: #f8f8f2 } /* Name.Variable.Global */ .highlight .vi { color: #f8f8f2 } /* Name.Variable.Instance */ .highlight .vm { color: #f8f8f2 } /* Name.Variable.Magic */ .highlight .il { color: #ae81ff } /* Literal.Number.Integer.Long */ } @media (prefers-color-scheme: light) { .highlight .hll { background-color: #ffffcc } .highlight .c { color: #888888 } /* Comment */ .highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */ .highlight .k { color: #008800; font-weight: bold } /* Keyword */ .highlight .ch { color: #888888 } /* Comment.Hashbang */ .highlight .cm { color: #888888 } /* Comment.Multiline */ .highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */ .highlight .cpf { color: #888888 } /* Comment.PreprocFile */ .highlight .c1 { color: #888888 } /* Comment.Single */ .highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */ .highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */ .highlight .ge { font-style: italic } /* Generic.Emph */ .highlight .gr { color: #aa0000 } /* Generic.Error */ .highlight .gh { color: #333333 } /* Generic.Heading */ .highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */ .highlight .go { color: #888888 } /* Generic.Output */ .highlight .gp { color: #555555 } /* Generic.Prompt */ .highlight .gs { font-weight: bold } /* Generic.Strong */ .highlight .gu { color: #666666 } /* Generic.Subheading */ .highlight .gt { color: #aa0000 } /* Generic.Traceback */ .highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */ .highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */ .highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */ .highlight .kp { color: #008800 } /* Keyword.Pseudo */ .highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */ .highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */ .highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */ .highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */ .highlight .na { color: #336699 } /* Name.Attribute */ .highlight .nb { color: #003388 } /* Name.Builtin */ .highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */ .highlight .no { color: #003366; font-weight: bold } /* Name.Constant */ .highlight .nd { color: #555555 } /* Name.Decorator */ .highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */ .highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */ .highlight .nl { color: #336699; font-style: italic } /* Name.Label */ .highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */ .highlight .py { color: #336699; font-weight: bold } /* Name.Property */ .highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */ .highlight .nv { color: #336699 } /* Name.Variable */ .highlight .ow { color: #008800 } /* Operator.Word */ .highlight .w { color: #bbbbbb } /* Text.Whitespace */ .highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */ .highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */ .highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */ .highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ .highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */ .highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */ .highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */ .highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */ .highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */ .highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */ .highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */ .highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */ .highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */ .highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */ .highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */ .highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */ .highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */ .highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */ .highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */ .highlight .vc { color: #336699 } /* Name.Variable.Class */ .highlight .vg { color: #dd7700 } /* Name.Variable.Global */ .highlight .vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */ }
#!/usr/bin/env python
"""Bootstrap setuptools installation

To use setuptools in your package's setup.py, include this
file in the same directory and add this to the top of your setup.py::

    from ez_setup import use_setuptools
    use_setuptools()

To require a specific version of setuptools, set a download
mirror, or use an alternate download directory, simply supply
the appropriate options to ``use_setuptools()``.

This file can also be run as a script to install or upgrade setuptools.
"""
from __future__ import absolute_import
import os
import shutil
import sys
import tempfile
import zipfile
import optparse
import subprocess
import platform
import contextlib

from distutils import log

try:
    from urllib.request import urlopen
except ImportError:
    from six.moves.urllib import urlopen

try:
    from site import USER_SITE
except ImportError:
    USER_SITE = None

DEFAULT_VERSION = "6.1"
DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/"


def _python_cmd(*args):
    """
    Return True if the command succeeded.
    """
    args = (sys.executable,) + args
    return subprocess.call(args) == 0


def _install(archive_filename, install_args=()):
    with archive_context(archive_filename):
        # installing
        log.warn('Installing Setuptools')
        if not _python_cmd('setup.py', 'install', *install_args):
            log.warn('Something went wrong during the installation.')
            log.warn('See the error message above.')
            # exitcode will be 2
            return 2


def _build_egg(egg, archive_filename, to_dir):
    with archive_context(archive_filename):
        # building an egg
        log.warn('Building a Setuptools egg in %s', to_dir)
        _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir)
    # returning the result
    log.warn(egg)
    if not os.path.exists(egg):
        raise IOError('Could not build the egg.')


class ContextualZipFile(zipfile.ZipFile):
    """
    Supplement ZipFile class to support context manager for Python 2.6
    """

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.close()

    def __new__(cls, *args, **kwargs):
        """
        Construct a ZipFile or ContextualZipFile as appropriate
        """
        if hasattr(zipfile.ZipFile, '__exit__'):
            return zipfile.ZipFile(*args, **kwargs)
        return super(ContextualZipFile, cls).__new__(cls)


@contextlib.contextmanager
def archive_context(filename):
    # extracting the archive
    tmpdir = tempfile.mkdtemp()
    log.warn('Extracting in %s', tmpdir)
    old_wd = os.getcwd()
    try:
        os.chdir(tmpdir)
        with ContextualZipFile(filename) as archive:
            archive.extractall()

        # going in the directory
        subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
        os.chdir(subdir)
        log.warn('Now working in %s', subdir)
        yield

    finally:
        os.chdir(old_wd)
        shutil.rmtree(tmpdir)


def _do_download(version, download_base, to_dir, download_delay):
    egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg'
                       % (version, sys.version_info[0], sys.version_info[1]))
    if not os.path.exists(egg):
        archive = download_setuptools(version, download_base,
                                      to_dir, download_delay)
        _build_egg(egg, archive, to_dir)
    sys.path.insert(0, egg)

    # Remove previously-imported pkg_resources if present (see
    # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details).
    if 'pkg_resources' in sys.modules:
        del sys.modules['pkg_resources']

    import setuptools
    setuptools.bootstrap_install_from = egg


def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
                   to_dir=os.curdir, download_delay=15):
    to_dir = os.path.abspath(to_dir)
    rep_modules = 'pkg_resources', 'setuptools'
    imported = set(sys.modules).intersection(rep_modules)
    try:
        import pkg_resources
    except ImportError:
        return _do_download(version, download_base, to_dir, download_delay)
    try:
        pkg_resources.require("setuptools>=" + version)
        return
    except pkg_resources.DistributionNotFound:
        return _do_download(version, download_base, to_dir, download_delay)
    except pkg_resources.VersionConflict as VC_err:
        if imported:
            msg = """\
The required version of setuptools (>={version}) is not available,
and can't be installed while this script is running. Please
install a more recent version first, using
'easy_install -U setuptools'.

(Currently using {VC_err.args[0]!r})
""".format(VC_err=VC_err, version=version)
            sys.stderr.write(msg)
            sys.exit(2)

        # otherwise, reload ok
        del pkg_resources, sys.modules['pkg_resources']
        return _do_download(version, download_base, to_dir, download_delay)


def _clean_check(cmd, target):
    """
    Run the command to download target. If the command fails, clean up before
    re-raising the error.
    """
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError:
        if os.access(target, os.F_OK):
            os.unlink(target)
        raise


def download_file_powershell(url, target):
    """
    Download the file at url to target using Powershell (which will validate
    trust). Raise an exception if the command cannot complete.
    """
    target = os.path.abspath(target)
    ps_cmd = (
        "[System.Net.WebRequest]::DefaultWebProxy.Credentials = "
        "[System.Net.CredentialCache]::DefaultCredentials; "
        "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)"
        % vars()
    )
    cmd = [
        'powershell',
        '-Command',
        ps_cmd,
    ]
    _clean_check(cmd, target)


def has_powershell():
    if platform.system() != 'Windows':
        return False
    cmd = ['powershell', '-Command', 'echo test']
    with open(os.path.devnull, 'wb') as devnull:
        try:
            subprocess.check_call(cmd, stdout=devnull, stderr=devnull)
        except Exception:
            return False
    return True


download_file_powershell.viable = has_powershell


def download_file_curl(url, target):
    cmd = ['curl', url, '--silent', '--output', target]
    _clean_check(cmd, target)


def has_curl():
    cmd = ['curl', '--version']
    with open(os.path.devnull, 'wb') as devnull:
        try:
            subprocess.check_call(cmd, stdout=devnull, stderr=devnull)
        except Exception:
            return False
    return True


download_file_curl.viable = has_curl


def download_file_wget(url, target):
    cmd = ['wget', url, '--quiet', '--output-document', target]
    _clean_check(cmd, target)


def has_wget():
    cmd = ['wget', '--version']
    with open(os.path.devnull, 'wb') as devnull:
        try:
            subprocess.check_call(cmd, stdout=devnull, stderr=devnull)
        except Exception:
            return False
    return True


download_file_wget.viable = has_wget


def download_file_insecure(url, target):
    """
    Use Python to download the file, even though it cannot authenticate the
    connection.
    """
    src = urlopen(url)
    try:
        # Read all the data in one block.
        data = src.read()
    finally:
        src.close()

    # Write all the data in one block to avoid creating a partial file.
    with open(target, "wb") as dst:
        dst.write(data)


download_file_insecure.viable = lambda: True


def get_best_downloader():
    downloaders = (
        download_file_powershell,
        download_file_curl,
        download_file_wget,
        download_file_insecure,
    )
    viable_downloaders = (dl for dl in downloaders if dl.viable())
    return next(viable_downloaders, None)


def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
                        to_dir=os.curdir, delay=15,
                        downloader_factory=get_best_downloader):
    """
    Download setuptools from a specified location and return its filename

    `version` should be a valid setuptools version number that is available
    as an sdist for download under the `download_base` URL (which should end
    with a '/'). `to_dir` is the directory where the egg will be downloaded.
    `delay` is the number of seconds to pause before an actual download
    attempt.

    ``downloader_factory`` should be a function taking no arguments and
    returning a function for downloading a URL to a target.
    """
    # making sure we use the absolute path
    to_dir = os.path.abspath(to_dir)
    zip_name = "setuptools-%s.zip" % version
    url = download_base + zip_name
    saveto = os.path.join(to_dir, zip_name)
    if not os.path.exists(saveto):  # Avoid repeated downloads
        log.warn("Downloading %s", url)
        downloader = downloader_factory()
        downloader(url, saveto)
    return os.path.realpath(saveto)


def _build_install_args(options):
    """
    Build the arguments to 'python setup.py install' on the setuptools package
    """
    return ['--user'] if options.user_install else []


def _parse_args():
    """
    Parse the command line for options
    """
    parser = optparse.OptionParser()
    parser.add_option(
        '--user', dest='user_install', action='store_true', default=False,
        help='install in user site package (requires Python 2.6 or later)')
    parser.add_option(
        '--download-base', dest='download_base', metavar="URL",
        default=DEFAULT_URL,
        help='alternative URL from where to download the setuptools package')
    parser.add_option(
        '--insecure', dest='downloader_factory', action='store_const',
        const=lambda: download_file_insecure, default=get_best_downloader,
        help='Use internal, non-validating downloader'
    )
    parser.add_option(
        '--version', help="Specify which version to download",
        default=DEFAULT_VERSION,
    )
    options, args = parser.parse_args()
    # positional arguments are ignored
    return options


def main():
    """Install or upgrade setuptools and EasyInstall"""
    options = _parse_args()
    archive = download_setuptools(
        version=options.version,
        download_base=options.download_base,
        downloader_factory=options.downloader_factory,
    )
    return _install(archive, _build_install_args(options))


if __name__ == '__main__':
    sys.exit(main())