Repo structure modification

- create functest subdirectory
- rename unit tests
- adapt path in exec and config files

JIRA: FUNCTEST-525

Change-Id: Ifd5c6edfb5bda1b09f82848e2269ad5fbeb84d0a
Signed-off-by: Morgan Richomme <morgan.richomme@orange.com>

7 files changed, 597 insertions, 0 deletions

diff --git a/functest/opnfv_tests/security_scan/config.ini b/functest/opnfv_tests/security_scan/config.ini
new file mode 100644
index 00000000..b97de80f
--- /dev/null
+++ b/functest/opnfv_tests/security_scan/config.ini
@@ -0,0 +1,29 @@
+[undercloud]
+port = 22
+user = stack
+remotekey = /home/stack/.ssh/id_rsa
+localkey = /root/.ssh/overCloudKey
+
+[controller]
+port = 22
+user = heat-admin
+scantype = xccdf
+secpolicy = /usr/share/xml/scap/ssg/content/ssg-centos7-xccdf.xml
+cpe = /usr/share/xml/scap/ssg/content/ssg-rhel7-cpe-dictionary.xml
+profile = stig-rhel7-server-upstream
+report = report.html
+results = results.xml
+reports_dir=/home/opnfv/functest/results/security_scan/
+clean = True
+
+[compute]
+port = 22
+user = heat-admin
+scantype = xccdf
+secpolicy = /usr/share/xml/scap/ssg/content/ssg-centos7-xccdf.xml
+cpe = /usr/share/xml/scap/ssg/content/ssg-rhel7-cpe-dictionary.xml
+profile = sstig-rhel7-server-upstream
+report = report.html
+results = results.xml
+reports_dir=/home/opnfv/functest/results/security_scan/
+clean = True
diff --git a/functest/opnfv_tests/security_scan/connect.py b/functest/opnfv_tests/security_scan/connect.py
new file mode 100644
index 00000000..18ca96d8
--- /dev/null
+++ b/functest/opnfv_tests/security_scan/connect.py
@@ -0,0 +1,244 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2016 Red Hat
+# Luke Hinds (lhinds@redhat.com)
+# This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# 0.1: OpenSCAP paramiko connection functions
+
+import os
+import socket
+import paramiko
+
+import functest.utils.functest_logger as ft_logger
+
+# add installer IP from env
+INSTALLER_IP = os.getenv('INSTALLER_IP')
+
+# Set up loggers
+logger = ft_logger.Logger("security_scan").getLogger()
+paramiko.util.log_to_file("/var/log/paramiko.log")
+
+
+class SetUp:
+    def __init__(self, *args):
+        self.args = args
+
+    def keystonepass(self):
+        com = self.args[0]
+        client = paramiko.SSHClient()
+        privatekeyfile = os.path.expanduser('/root/.ssh/id_rsa')
+        selectedkey = paramiko.RSAKey.from_private_key_file(privatekeyfile)
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        try:
+            client.connect(INSTALLER_IP, port=22, username='stack',
+                           pkey=selectedkey)
+        except paramiko.SSHException:
+            logger.error("Password is invalid for "
+                         "undercloud host: {0}".format(INSTALLER_IP))
+        except paramiko.AuthenticationException:
+            logger.error("Authentication failed for "
+                         "undercloud host: {0}".format(INSTALLER_IP))
+        except socket.error:
+            logger.error("Socker Connection failed for "
+                         "undercloud host: {0}".format(INSTALLER_IP))
+        stdin, stdout, stderr = client.exec_command(com)
+        return stdout.read()
+        client.close()
+
+    def getockey(self):
+        remotekey = self.args[0]
+        localkey = self.args[1]
+        privatekeyfile = os.path.expanduser('/root/.ssh/id_rsa')
+        selectedkey = paramiko.RSAKey.from_private_key_file(privatekeyfile)
+        transport = paramiko.Transport((INSTALLER_IP, 22))
+        transport.connect(username='stack', pkey=selectedkey)
+        try:
+            sftp = paramiko.SFTPClient.from_transport(transport)
+        except paramiko.SSHException:
+            logger.error("Authentication failed for "
+                         "host: {0}".format(INSTALLER_IP))
+        except paramiko.AuthenticationException:
+            logger.error("Authentication failed for "
+                         "host: {0}".format(INSTALLER_IP))
+        except socket.error:
+            logger.error("Socker Connection failed for "
+                         "undercloud host: {0}".format(INSTALLER_IP))
+        sftp.get(remotekey, localkey)
+        sftp.close()
+        transport.close()
+
+
+class ConnectionManager:
+    def __init__(self, host, port, user, localkey, *args):
+        self.host = host
+        self.port = port
+        self.user = user
+        self.localkey = localkey
+        self.args = args
+
+    def remotescript(self):
+        localpath = self.args[0]
+        remotepath = self.args[1]
+        com = self.args[2]
+
+        client = paramiko.SSHClient()
+        privatekeyfile = os.path.expanduser('/root/.ssh/id_rsa')
+        selectedkey = paramiko.RSAKey.from_private_key_file(privatekeyfile)
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        # Connection to undercloud
+        try:
+            client.connect(INSTALLER_IP, port=22, username='stack',
+                           pkey=selectedkey)
+        except paramiko.SSHException:
+            logger.error("Authentication failed for "
+                         "host: {0}".format(self.host))
+        except paramiko.AuthenticationException:
+            logger.error("Authentication failed for "
+                         "host: {0}".format(self.host))
+        except socket.error:
+            logger.error("Socker Connection failed for "
+                         "undercloud host: {0}".format(self.host))
+
+        transport = client.get_transport()
+        local_addr = ('127.0.0.1', 0)
+        channel = transport.open_channel("direct-tcpip",
+                                         (self.host, int(self.port)),
+                                         (local_addr))
+        remote_client = paramiko.SSHClient()
+        remote_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        # Tunnel to overcloud
+        try:
+            remote_client.connect('127.0.0.1', port=22, username=self.user,
+                                  key_filename=self.localkey, sock=channel)
+            sftp = remote_client.open_sftp()
+            sftp.put(localpath, remotepath)
+        except paramiko.SSHException:
+            logger.error("Authentication failed for "
+                         "host: {0}".format(self.host))
+        except paramiko.AuthenticationException:
+            logger.error("Authentication failed for "
+                         "host: {0}".format(self.host))
+        except socket.error:
+            logger.error("Socker Connection failed for "
+                         "undercloud host: {0}".format(self.host))
+
+        output = ""
+        stdin, stdout, stderr = remote_client.exec_command(com)
+        stdout = stdout.readlines()
+        # remove script
+        sftp.remove(remotepath)
+        remote_client.close()
+        client.close()
+        # Pipe back stout
+        for line in stdout:
+            output = output + line
+        if output != "":
+            return output
+
+    def remotecmd(self):
+        com = self.args[0]
+
+        client = paramiko.SSHClient()
+        privatekeyfile = os.path.expanduser('/root/.ssh/id_rsa')
+        selectedkey = paramiko.RSAKey.from_private_key_file(privatekeyfile)
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        # Connection to undercloud
+        try:
+            client.connect(INSTALLER_IP, port=22, username='stack',
+                           pkey=selectedkey)
+        except paramiko.SSHException:
+            logger.error("Authentication failed for "
+                         "host: {0}".format(self.host))
+        except paramiko.AuthenticationException:
+            logger.error("Authentication failed for "
+                         "host: {0}".format(self.host))
+        except socket.error:
+            logger.error("Socker Connection failed for "
+                         "undercloud host: {0}".format(self.host))
+
+        transport = client.get_transport()
+        local_addr = ('127.0.0.1', 0)  # 0 denotes choose random port
+        channel = transport.open_channel("direct-tcpip",
+                                         (self.host, int(self.port)),
+                                         (local_addr))
+        remote_client = paramiko.SSHClient()
+        remote_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        # Tunnel to overcloud
+        try:
+            remote_client.connect('127.0.0.1', port=22, username=self.user,
+                                  key_filename=self.localkey, sock=channel)
+        except paramiko.SSHException:
+            logger.error("Authentication failed for "
+                         "host: {0}".format(self.host))
+        except paramiko.AuthenticationException:
+            logger.error("Authentication failed for "
+                         "host: {0}".format(self.host))
+        except socket.error:
+            logger.error("Socker Connection failed for "
+                         "undercloud host: {0}".format(self.host))
+
+        chan = remote_client.get_transport().open_session()
+        chan.get_pty()
+        feed = chan.makefile()
+        chan.exec_command(com)
+        print feed.read()
+
+        remote_client.close()
+        client.close()
+
+    def download_reports(self):
+        dl_folder = self.args[0]
+        reportfile = self.args[1]
+        reportname = self.args[2]
+        resultsname = self.args[3]
+        client = paramiko.SSHClient()
+        privatekeyfile = os.path.expanduser('/root/.ssh/id_rsa')
+        selectedkey = paramiko.RSAKey.from_private_key_file(privatekeyfile)
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        # Connection to overcloud
+        try:
+            client.connect(INSTALLER_IP, port=22, username='stack',
+                           pkey=selectedkey)
+        except paramiko.SSHException:
+            logger.error("Authentication failed for "
+                         "host: {0}".format(self.host))
+        except paramiko.AuthenticationException:
+            logger.error("Authentication failed for "
+                         "host: {0}".format(self.host))
+        except socket.error:
+            logger.error("Socker Connection failed for "
+                         "undercloud host: {0}".format(self.host))
+
+        transport = client.get_transport()
+        local_addr = ('127.0.0.1', 0)  # 0 denotes choose random port
+        channel = transport.open_channel("direct-tcpip",
+                                         (self.host, int(self.port)),
+                                         (local_addr))
+        remote_client = paramiko.SSHClient()
+        remote_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        # Tunnel to overcloud
+        try:
+            remote_client.connect('127.0.0.1', port=22, username=self.user,
+                                  key_filename=self.localkey, sock=channel)
+        except paramiko.SSHException:
+            logger.error("Authentication failed for "
+                         "host: {0}".format(self.host))
+        except paramiko.AuthenticationException:
+            logger.error("Authentication failed for "
+                         "host: {0}".format(self.host))
+        except socket.error:
+            logger.error("Socker Connection failed for "
+                         "undercloud host: {0}".format(self.host))
+        # Download the reports
+        sftp = remote_client.open_sftp()
+        logger.info("Downloading \"{0}\"...".format(reportname))
+        sftp.get(reportfile, ('{0}/{1}'.format(dl_folder, reportname)))
+        logger.info("Downloading \"{0}\"...".format(resultsname))
+        sftp.get(reportfile, ('{0}/{1}'.format(dl_folder, resultsname)))
+        sftp.close()
+        transport.close()
diff --git a/functest/opnfv_tests/security_scan/examples/xccdf-rhel7-server-upstream.ini b/functest/opnfv_tests/security_scan/examples/xccdf-rhel7-server-upstream.ini
new file mode 100644
index 00000000..43b2e82d
--- /dev/null
+++ b/functest/opnfv_tests/security_scan/examples/xccdf-rhel7-server-upstream.ini
@@ -0,0 +1,29 @@
+[undercloud]
+port = 22
+user = stack
+remotekey = /home/stack/.ssh/id_rsa
+localkey = /root/.ssh/overCloudKey
+
+[controller]
+port = 22
+user = heat-admin
+scantype = xccdf
+secpolicy = /usr/share/xml/scap/ssg/content/ssg-centos7-xccdf.xml
+cpe = /usr/share/xml/scap/ssg/content/ssg-rhel7-cpe-dictionary.xml
+profile = stig-rhel7-server-upstream
+report = report.hmtl
+results = results.xml
+reports_dir=/home/opnfv/functest/results/security_scan/
+clean = True
+
+[compute]
+port = 22
+user = heat-admin
+scantype = xccdf
+secpolicy = /usr/share/xml/scap/ssg/content/ssg-centos7-xccdf.xml
+cpe = /usr/share/xml/scap/ssg/content/ssg-rhel7-cpe-dictionary.xml
+profile = stig-rhel7-server-upstream
+report = report.hmtl
+results = results.xml
+reports_dir=/home/opnfv/functest/results/security_scan/
+clean = True
diff --git a/functest/opnfv_tests/security_scan/examples/xccdf-standard.ini b/functest/opnfv_tests/security_scan/examples/xccdf-standard.ini
new file mode 100644
index 00000000..bfbcf82d
--- /dev/null
+++ b/functest/opnfv_tests/security_scan/examples/xccdf-standard.ini
@@ -0,0 +1,29 @@
+[undercloud]
+port = 22
+user = stack
+remotekey = /home/stack/.ssh/id_rsa
+localkey = /root/.ssh/overCloudKey
+
+[controller]
+port = 22
+user = heat-admin
+scantype = xccdf
+secpolicy = /usr/share/xml/scap/ssg/content/ssg-centos7-xccdf.xml
+cpe = /usr/share/xml/scap/ssg/content/ssg-rhel7-cpe-dictionary.xml
+profile = standard
+report = report.hmtl
+results = results.xml
+reports_dir=/home/opnfv/functest/results/security_scan/
+clean = True
+
+[compute]
+port = 22
+user = heat-admin
+scantype = xccdf
+secpolicy = /usr/share/xml/scap/ssg/content/ssg-centos7-xccdf.xml
+cpe = /usr/share/xml/scap/ssg/content/ssg-rhel7-cpe-dictionary.xml
+profile = standard
+report = report.hmtl
+results = results.xml
+reports_dir=/home/opnfv/functest/results/security_scan/
+clean = True
diff --git a/functest/opnfv_tests/security_scan/scripts/createfiles.py b/functest/opnfv_tests/security_scan/scripts/createfiles.py
new file mode 100644
index 00000000..b828901a
--- /dev/null
+++ b/functest/opnfv_tests/security_scan/scripts/createfiles.py
@@ -0,0 +1,26 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2016 Red Hat
+# Luke Hinds (lhinds@redhat.com)
+# This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# 0.1: This script creates the needed local files into a tmp directory. Should
+# '--clean' be passed, all files will be removed, post scan.
+
+
+import os
+import tempfile
+
+files = ['results.xml', 'report.html', 'syschar.xml']
+
+
+directory_name = tempfile.mkdtemp()
+
+for i in files:
+    os.system("touch %s/%s" % (directory_name, i))
+
+print directory_name
diff --git a/functest/opnfv_tests/security_scan/scripts/internet_check.py b/functest/opnfv_tests/security_scan/scripts/internet_check.py
new file mode 100644
index 00000000..1bed50a7
--- /dev/null
+++ b/functest/opnfv_tests/security_scan/scripts/internet_check.py
@@ -0,0 +1,25 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2016 Red Hat
+# Luke Hinds (lhinds@redhat.com)
+# This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Performs simple connection check, falls to default timeout of 10 seconds
+
+import socket
+
+TEST_HOST = "google.com"
+
+
+def is_connected():
+    try:
+        host = socket.gethostbyname(TEST_HOST)
+        socket.create_connection((host, 80), 2)
+        return True
+    except:
+        return False
+print is_connected()
diff --git a/functest/opnfv_tests/security_scan/security_scan.py b/functest/opnfv_tests/security_scan/security_scan.py
new file mode 100755
index 00000000..4e0407fb
--- /dev/null
+++ b/functest/opnfv_tests/security_scan/security_scan.py
@@ -0,0 +1,215 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2016 Red Hat
+# Luke Hinds (lhinds@redhat.com)
+# This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# 0.1: This script installs OpenSCAP on the remote host, and scans the
+# nominated node. Post scan a report is downloaded and if '--clean' is passed
+# all trace of the scan is removed from the remote system.
+
+
+import datetime
+import os
+import sys
+from ConfigParser import SafeConfigParser
+
+import argparse
+from keystoneclient import session
+from keystoneclient.auth.identity import v2
+from novaclient import client
+
+import connect
+import functest.utils.functest_utils as ft_utils
+
+__version__ = 0.1
+__author__ = 'Luke Hinds (lhinds@redhat.com)'
+__url__ = 'https://wiki.opnfv.org/display/functest/Functest+Security'
+
+# Global vars
+INSTALLER_IP = os.getenv('INSTALLER_IP')
+oscapbin = 'sudo /bin/oscap'
+functest_dir = '%s/opnfv_tests/security_scan/' % ft_utils.FUNCTEST_REPO
+
+# Apex Spefic var needed to query Undercloud
+if os.getenv('OS_AUTH_URL') is None:
+    connect.logger.error(" Enviroment variable OS_AUTH_URL is not set")
+    sys.exit(0)
+else:
+    OS_AUTH_URL = os.getenv('OS_AUTH_URL')
+
+# args
+parser = argparse.ArgumentParser(description='OPNFV OpenSCAP Scanner')
+parser.add_argument('--config', action='store', dest='cfgfile',
+                    help='Config file', required=True)
+args = parser.parse_args()
+
+# Config Parser
+cfgparse = SafeConfigParser()
+cfgparse.read(args.cfgfile)
+
+#  Grab Undercloud key
+remotekey = cfgparse.get('undercloud', 'remotekey')
+localkey = cfgparse.get('undercloud', 'localkey')
+setup = connect.SetUp(remotekey, localkey)
+setup.getockey()
+
+
+# Configure Nova Credentials
+com = 'sudo /usr/bin/hiera admin_password'
+setup = connect.SetUp(com)
+keypass = setup.keystonepass()
+auth = v2.Password(auth_url=OS_AUTH_URL,
+                   username='admin',
+                   password=str(keypass).rstrip(),
+                   tenant_name='admin')
+sess = session.Session(auth=auth)
+nova = client.Client(2, session=sess)
+
+
+def run_tests(host, nodetype):
+    user = cfgparse.get(nodetype, 'user')
+    port = cfgparse.get(nodetype, 'port')
+    connect.logger.info("Host: {0} Selected Profile: {1}".format(host,
+                                                                 nodetype))
+    connect.logger.info("Checking internet for package installation...")
+    if internet_check(host, nodetype):
+        connect.logger.info("Internet Connection OK.")
+        connect.logger.info("Creating temp file structure..")
+        createfiles(host, port, user, localkey)
+        connect.logger.debug("Installing OpenSCAP...")
+        install_pkg(host, port, user, localkey)
+        connect.logger.debug("Running scan...")
+        run_scanner(host, port, user, localkey, nodetype)
+        clean = cfgparse.get(nodetype, 'clean')
+        connect.logger.info("Post installation tasks....")
+        post_tasks(host, port, user, localkey, nodetype)
+        if clean:
+            connect.logger.info("Cleaning down environment....")
+            connect.logger.debug("Removing OpenSCAP....")
+            removepkg(host, port, user, localkey, nodetype)
+            connect.logger.info("Deleting tmp file and reports (remote)...")
+            cleandir(host, port, user, localkey, nodetype)
+    else:
+        connect.logger.error("Internet timeout. Moving on to next node..")
+        pass
+
+
+def nova_iterate():
+    # Find compute nodes, active with network on ctlplane
+    for server in nova.servers.list():
+        if server.status == 'ACTIVE' and 'compute' in server.name:
+            networks = server.networks
+            nodetype = 'compute'
+            for host in networks['ctlplane']:
+                run_tests(host, nodetype)
+        # Find controller nodes, active with network on ctlplane
+        elif server.status == 'ACTIVE' and 'controller' in server.name:
+            networks = server.networks
+            nodetype = 'controller'
+            for host in networks['ctlplane']:
+                run_tests(host, nodetype)
+
+
+def internet_check(host, nodetype):
+    import connect
+    user = cfgparse.get(nodetype, 'user')
+    port = cfgparse.get(nodetype, 'port')
+    localpath = functest_dir + 'scripts/internet_check.py'
+    remotepath = '/tmp/internet_check.py'
+    com = 'python /tmp/internet_check.py'
+    testconnect = connect.ConnectionManager(host, port, user, localkey,
+                                            localpath, remotepath, com)
+    connectionresult = testconnect.remotescript()
+    if connectionresult.rstrip() == 'True':
+        return True
+    else:
+        return False
+
+
+def createfiles(host, port, user, localkey):
+    import connect
+    global tmpdir
+    localpath = functest_dir + 'scripts/createfiles.py'
+    remotepath = '/tmp/createfiles.py'
+    com = 'python /tmp/createfiles.py'
+    connect = connect.ConnectionManager(host, port, user, localkey,
+                                        localpath, remotepath, com)
+    tmpdir = connect.remotescript()
+
+
+def install_pkg(host, port, user, localkey):
+    import connect
+    com = 'sudo yum -y install openscap-scanner scap-security-guide'
+    connect = connect.ConnectionManager(host, port, user, localkey, com)
+    connect.remotecmd()
+
+
+def run_scanner(host, port, user, localkey, nodetype):
+    import connect
+    scantype = cfgparse.get(nodetype, 'scantype')
+    profile = cfgparse.get(nodetype, 'profile')
+    results = cfgparse.get(nodetype, 'results')
+    report = cfgparse.get(nodetype, 'report')
+    secpolicy = cfgparse.get(nodetype, 'secpolicy')
+    # Here is where we contruct the actual scan command
+    if scantype == 'xccdf':
+        cpe = cfgparse.get(nodetype, 'cpe')
+        com = '{0} xccdf eval --profile {1} --results {2}/{3}' \
+              ' --report {2}/{4} --cpe {5} {6}'.format(oscapbin,
+                                                       profile,
+                                                       tmpdir.rstrip(),
+                                                       results,
+                                                       report,
+                                                       cpe,
+                                                       secpolicy)
+        connect = connect.ConnectionManager(host, port, user, localkey, com)
+        connect.remotecmd()
+    elif scantype == 'oval':
+        com = '{0} oval eval --results {1}/{2} '
+        '--report {1}/{3} {4}'.format(oscapbin, tmpdir.rstrip(),
+                                      results, report, secpolicy)
+        connect = connect.ConnectionManager(host, port, user, localkey, com)
+        connect.remotecmd()
+    else:
+        com = '{0} oval-collect '.format(oscapbin)
+        connect = connect.ConnectionManager(host, port, user, localkey, com)
+        connect.remotecmd()
+
+
+def post_tasks(host, port, user, localkey, nodetype):
+    import connect
+    # Create the download folder for functest dashboard and download reports
+    reports_dir = cfgparse.get(nodetype, 'reports_dir')
+    dl_folder = os.path.join(reports_dir, host + "_" +
+                             datetime.datetime.
+                             now().strftime('%Y-%m-%d_%H-%M-%S'))
+    os.makedirs(dl_folder, 0755)
+    report = cfgparse.get(nodetype, 'report')
+    results = cfgparse.get(nodetype, 'results')
+    reportfile = '{0}/{1}'.format(tmpdir.rstrip(), report)
+    connect = connect.ConnectionManager(host, port, user, localkey, dl_folder,
+                                        reportfile, report, results)
+    connect.download_reports()
+
+
+def removepkg(host, port, user, localkey, nodetype):
+    import connect
+    com = 'sudo yum -y remove openscap-scanner scap-security-guide'
+    connect = connect.ConnectionManager(host, port, user, localkey, com)
+    connect.remotecmd()
+
+
+def cleandir(host, port, user, localkey, nodetype):
+    import connect
+    com = 'sudo rm -r {0}'.format(tmpdir.rstrip())
+    connect = connect.ConnectionManager(host, port, user, localkey, com)
+    connect.remotecmd()
+
+
+if __name__ == '__main__':
+    nova_iterate()