Module to manage pip packages

This new module provides methods to manage Python PIP packages from
a URL, from a local directory or from a build PIP package.

The implemented commands are:
- Install package.
- Remove package.
- List all installed packages in the system.

JIRA: YARDSTICK-910

Change-Id: I8f7d1b77c0c384b801cc6f5e67d8b45ce7c6bfdf
Signed-off-by: Rodolfo Alonso Hernandez <rodolfo.alonso.hernandez@intel.com>

10 files changed, 211 insertions, 0 deletions

diff --git a/yardstick/tests/functional/base.py b/yardstick/tests/functional/base.py
new file mode 100644
index 000000000..51be013a1
--- /dev/null
+++ b/yardstick/tests/functional/base.py
@@ -0,0 +1,46 @@
+# Copyright (c) 2018 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+import six
+
+from oslo_config import cfg
+from oslotest import base
+
+
+CONF = cfg.CONF
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseFunctionalTestCase(base.BaseTestCase):
+    """Base class for functional tests."""
+
+    def setUp(self):
+        super(BaseFunctionalTestCase, self).setUp()
+
+    def config(self, **kw):
+        """Override some configuration values.
+
+        The keyword arguments are the names of configuration options to
+        override and their values.
+
+        If a group argument is supplied, the overrides are applied to
+        the specified configuration option group.
+
+        All overrides are automatically cleared at the end of the current
+        test by the fixtures cleanup process.
+        """
+        group = kw.pop('group', None)
+        for k, v in kw.items():
+            CONF.set_override(k, v, group)
diff --git a/yardstick/tests/functional/common/fake_directory_package/README.md b/yardstick/tests/functional/common/fake_directory_package/README.md
new file mode 100644
index 000000000..689e47039
--- /dev/null
+++ b/yardstick/tests/functional/common/fake_directory_package/README.md
@@ -0,0 +1,2 @@
+# yardstick_new_plugin
+Yardstick plugin
diff --git a/yardstick/tests/functional/common/fake_directory_package/setup.py b/yardstick/tests/functional/common/fake_directory_package/setup.py
new file mode 100644
index 000000000..cf938ef4f
--- /dev/null
+++ b/yardstick/tests/functional/common/fake_directory_package/setup.py
@@ -0,0 +1,29 @@
+# Copyright (c) 2018 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from setuptools import setup, find_packages
+
+setup(
+    name='yardstick_new_plugin_2',
+    version='1.0.0',
+    packages=find_packages(),
+    include_package_data=True,
+    url='https://www.opnfv.org',
+    entry_points={
+        'yardstick.scenarios': [
+            'Dummy2 = yardstick_new_plugin.benchmark.scenarios.dummy2.dummy2:'
+            'Dummy2',
+        ]
+    },
+)
diff --git a/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/__init__.py b/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/__init__.py
diff --git a/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/__init__.py b/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/__init__.py
diff --git a/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/__init__.py b/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/__init__.py
diff --git a/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/dummy2/__init__.py b/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/dummy2/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/dummy2/__init__.py
diff --git a/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/dummy2/dummy2.py b/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/dummy2/dummy2.py
new file mode 100644
index 000000000..a2211ec51
--- /dev/null
+++ b/yardstick/tests/functional/common/fake_directory_package/yardstick_new_plugin_2/benchmark/scenarios/dummy2/dummy2.py
@@ -0,0 +1,40 @@
+# Copyright (c) 2018 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from yardstick.benchmark.scenarios import base
+
+
+LOG = logging.getLogger(__name__)
+
+
+class Dummy2(base.Scenario):
+    """Execute Dummy (v2!) echo"""
+    __scenario_type__ = "Dummy2"
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+        self.setup_done = False
+
+    def setup(self):
+        self.setup_done = True
+
+    def run(self, result):
+        if not self.setup_done:
+            self.setup()
+
+        result["hello"] = "yardstick"
+        LOG.info("Dummy (v2!) echo hello yardstick!")
diff --git a/yardstick/tests/functional/common/fake_pip_package/yardstick_new_plugin-1.0.0.tar.gz b/yardstick/tests/functional/common/fake_pip_package/yardstick_new_plugin-1.0.0.tar.gz
new file mode 100644
index 000000000..e5379a78a
--- /dev/null
+++ b/yardstick/tests/functional/common/fake_pip_package/yardstick_new_plugin-1.0.0.tar.gz
diff --git a/yardstick/tests/functional/common/test_packages.py b/yardstick/tests/functional/common/test_packages.py
new file mode 100644
index 000000000..5dead4e55
--- /dev/null
+++ b/yardstick/tests/functional/common/test_packages.py
@@ -0,0 +1,94 @@
+# Copyright (c) 2018 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from os import path
+import re
+
+from yardstick.common import packages
+from yardstick.common import utils
+from yardstick.tests.functional import base
+
+
+class PipPackagesTestCase(base.BaseFunctionalTestCase):
+
+    TMP_FOLDER = '/tmp/pip_packages/'
+    PYTHONPATH = 'PYTHONPATH=%s' % TMP_FOLDER
+
+    def setUp(self):
+        super(PipPackagesTestCase, self).setUp()
+        privsep_helper = os.path.join(
+            os.getenv('VIRTUAL_ENV'), 'bin', 'privsep-helper')
+        self.config(
+            helper_command=' '.join(['sudo', '-EH', privsep_helper]),
+            group='yardstick_privileged')
+        self.addCleanup(self._cleanup)
+
+    def _cleanup(self):
+        utils.execute_command('sudo rm -rf %s' % self.TMP_FOLDER)
+
+    def _remove_package(self, package):
+        os.system('%s pip uninstall %s -y' % (self.PYTHONPATH, package))
+
+    def _list_packages(self):
+        pip_list_regex = re.compile(
+            r"(?P<name>[\w\.-]+) \((?P<version>[\w\d_\.\-]+),*.*\)")
+        pkg_dict = {}
+        pkgs = utils.execute_command('pip list',
+                                     env={'PYTHONPATH': self.TMP_FOLDER})
+        for line in pkgs:
+            match = pip_list_regex.match(line)
+            if match and match.group('name'):
+                pkg_dict[match.group('name')] = match.group('version')
+        return pkg_dict
+
+    def test_install_from_folder(self):
+        dirname = path.dirname(__file__)
+        package_dir = dirname + '/fake_directory_package'
+        package_name = 'yardstick-new-plugin-2'
+        self.addCleanup(self._remove_package, package_name)
+        self._remove_package(package_name)
+        self.assertFalse(package_name in self._list_packages())
+
+        self.assertEqual(0, packages.pip_install(package_dir, self.TMP_FOLDER))
+        self.assertTrue(package_name in self._list_packages())
+
+    def test_install_from_pip_package(self):
+        dirname = path.dirname(__file__)
+        package_path = (dirname +
+                        '/fake_pip_package/yardstick_new_plugin-1.0.0.tar.gz')
+        package_name = 'yardstick-new-plugin'
+        self.addCleanup(self._remove_package, package_name)
+        self._remove_package(package_name)
+        self.assertFalse(package_name in self._list_packages())
+
+        self.assertEqual(0, packages.pip_install(package_path, self.TMP_FOLDER))
+        self.assertTrue(package_name in self._list_packages())
+
+    # NOTE(ralonsoh): an stable test plugin project is needed in OPNFV git
+    # server to execute this test.
+    # def test_install_from_url(self):
+
+    def test_pip_freeze(self):
+        # NOTE (ralonsoh): from requirements.txt file. The best way to test
+        # this function is to parse requirements.txt and test-requirements.txt
+        # and check all packages.
+        pkgs_ref = {'Babel': '2.3.4',
+                    'SQLAlchemy': '1.1.12',
+                    'influxdb': '4.1.1',
+                    'netifaces': '0.10.6',
+                    'unicodecsv': '0.14.1'}
+        pkgs = packages.pip_list()
+        for name, version in (pkgs_ref.items()):
+            self.assertEqual(version, pkgs[name])