Add Apex based security scan code

JIRA: FUNCTEST-169

Performs a SCAP based security scan of all overcloud
nodes, followed by report download to functest dashboard

Change-Id: I9058227e19c0877e5f8913931febdd667a908cdd
Signed-off-by: lukehinds <lukehinds@gmail.com>

6 files changed, 490 insertions, 0 deletions

diff --git a/testcases/security_scan/config.ini b/testcases/security_scan/config.ini
new file mode 100644
index 00000000..440b23cb
--- /dev/null
+++ b/testcases/security_scan/config.ini
@@ -0,0 +1,25 @@
+[controller]
+port = 22
+user = heat-admin
+user_key = /root/.ssh/stackkey
+scantype = xccdf
+secpolicy = /usr/share/xml/scap/ssg/content/ssg-centos7-xccdf.xml
+cpe = /usr/share/xml/scap/ssg/content/ssg-rhel7-cpe-dictionary.xml
+profile = standard
+report = report.hmtl
+results = results.xml
+reports_dir=/home/opnfv/functest/results/security_scan/
+clean = True
+
+[compute]
+port = 22
+user = heat-admin
+user_key = /root/.ssh/stackkey
+scantype = xccdf
+secpolicy = /usr/share/xml/scap/ssg/content/ssg-centos7-xccdf.xml
+cpe = /usr/share/xml/scap/ssg/content/ssg-rhel7-cpe-dictionary.xml
+profile = standard
+report = report.hmtl
+results = results.xml
+reports_dir=/home/opnfv/functest/results/security_scan/
+clean = True
diff --git a/testcases/security_scan/connect.py b/testcases/security_scan/connect.py
new file mode 100644
index 00000000..9430e9a7
--- /dev/null
+++ b/testcases/security_scan/connect.py
@@ -0,0 +1,211 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2016 Red Hat
+# Luke Hinds (lhinds@redhat.com)
+# This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# 0.1: OpenSCAP paramiko connection functions
+
+import os
+import socket
+import paramiko
+
+import functest.utils.functest_logger as ft_logger
+
+# Enable below for connection debugging
+# paramiko.util.log_to_file('ssh.log')
+
+# add installer IP from env
+INSTALLER_IP = os.getenv('INSTALLER_IP')
+
+# Set up logger
+logger = ft_logger.Logger("security_scan").getLogger()
+
+
+class novaManager:
+    def __init__(self, *args):
+        self.args = args
+
+    def keystonepass(self):
+        com = self.args[0]
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        try:
+            client.connect(INSTALLER_IP, port=22, username='stack')
+        except paramiko.SSHException:
+            logger.error("Password is invalid for "
+                         "undercloud host: {0}").format(INSTALLER_IP)
+        except paramiko.AuthenticationException:
+            logger.error("Authentication failed for "
+                         "undercloud host: {0}").format(INSTALLER_IP)
+        except socket.error:
+            logger.error("Socker Connection failed for "
+                         "undercloud host: {0}").format(INSTALLER_IP)
+        stdin, stdout, stderr = client.exec_command(com)
+        return stdout.read()
+
+
+class connectionManager:
+    def __init__(self, host, port, user, user_key, *args):
+        self.host = host
+        self.port = port
+        self.user = user
+        self.user_key = user_key
+        self.args = args
+
+    def remotescript(self):
+        localpath = self.args[0]
+        remotepath = self.args[1]
+        com = self.args[2]
+
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        # Connection to undercloud
+        try:
+            client.connect(INSTALLER_IP, port=22, username='stack')
+        except paramiko.SSHException:
+            logger.error("Authentication failed for "
+                         "host: {0}").format(self.host)
+        except paramiko.AuthenticationException:
+            logger.error("Authentication failed for "
+                         "host: {0}").format(self.host)
+        except socket.error:
+            logger.error("Socker Connection failed for "
+                         "undercloud host: {0}").format(self.host)
+
+        transport = client.get_transport()
+        local_addr = ('127.0.0.1', 0)
+        channel = transport.open_channel("direct-tcpip",
+                                         (self.host, int(self.port)),
+                                         (local_addr))
+        remote_client = paramiko.SSHClient()
+        remote_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        # Tunnel to overcloud
+        try:
+            remote_client.connect('127.0.0.1', port=22, username=self.user,
+                                  key_filename=self.user_key, sock=channel)
+            sftp = remote_client.open_sftp()
+            sftp.put(localpath, remotepath)
+        except paramiko.SSHException:
+            logger.error("Authentication failed for "
+                         "host: {0}").format(self.host)
+        except paramiko.AuthenticationException:
+            logger.error("Authentication failed for "
+                         "host: {0}").format(self.host)
+        except socket.error:
+            logger.error("Socker Connection failed for "
+                         "undercloud host: {0}").format(self.host)
+
+        output = ""
+        stdin, stdout, stderr = remote_client.exec_command(com)
+        stdout = stdout.readlines()
+        # remove script
+        sftp.remove(remotepath)
+        remote_client.close()
+        client.close()
+        # Pipe back stout
+        for line in stdout:
+            output = output + line
+        if output != "":
+            return output
+
+    def remotecmd(self):
+        com = self.args[0]
+
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        # Connection to undercloud
+        try:
+            client.connect(INSTALLER_IP, port=22, username='stack')
+        except paramiko.SSHException:
+            logger.error("Authentication failed for "
+                         "host: {0}").format(self.host)
+        except paramiko.AuthenticationException:
+            logger.error("Authentication failed for "
+                         "host: {0}").format(self.host)
+        except socket.error:
+            logger.error("Socker Connection failed for "
+                         "undercloud host: {0}").format(self.host)
+
+        transport = client.get_transport()
+        local_addr = ('127.0.0.1', 0)  # 0 denotes choose random port
+        channel = transport.open_channel("direct-tcpip",
+                                         (self.host, int(self.port)),
+                                         (local_addr))
+        remote_client = paramiko.SSHClient()
+        remote_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        # Tunnel to overcloud
+        try:
+            remote_client.connect('127.0.0.1', port=22, username=self.user,
+                                  key_filename=self.user_key, sock=channel)
+        except paramiko.SSHException:
+            logger.error("Authentication failed for "
+                         "host: {0}").format(self.host)
+        except paramiko.AuthenticationException:
+            logger.error("Authentication failed for "
+                         "host: {0}").format(self.host)
+        except socket.error:
+            logger.error("Socker Connection failed for "
+                         "undercloud host: {0}").format(self.host)
+
+        chan = remote_client.get_transport().open_session()
+        chan.get_pty()
+        f = chan.makefile()
+        chan.exec_command(com)
+        print f.read()
+
+        remote_client.close()
+        client.close()
+
+    def download_reports(self):
+        dl_folder = self.args[0]
+        reportfile = self.args[1]
+        reportname = self.args[2]
+        resultsname = self.args[3]
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        # Connection to overcloud
+        try:
+            client.connect(INSTALLER_IP, port=22, username='stack')
+        except paramiko.SSHException:
+            logger.error("Authentication failed for "
+                         "host: {0}").format(self.host)
+        except paramiko.AuthenticationException:
+            logger.error("Authentication failed for "
+                         "host: {0}").format(self.host)
+        except socket.error:
+            logger.error("Socker Connection failed for "
+                         "undercloud host: {0}").format(self.host)
+
+        transport = client.get_transport()
+        local_addr = ('127.0.0.1', 0)  # 0 denotes choose random port
+        channel = transport.open_channel("direct-tcpip",
+                                         (self.host, int(self.port)),
+                                         (local_addr))
+        remote_client = paramiko.SSHClient()
+        remote_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        # Tunnel to overcloud
+        try:
+            remote_client.connect('127.0.0.1', port=22, username=self.user,
+                                  key_filename=self.user_key, sock=channel)
+        except paramiko.SSHException:
+            logger.error("Authentication failed for "
+                         "host: {0}").format(self.host)
+        except paramiko.AuthenticationException:
+            logger.error("Authentication failed for "
+                         "host: {0}").format(self.host)
+        except socket.error:
+            logger.error("Socker Connection failed for "
+                         "undercloud host: {0}").format(self.host)
+        # Download the reports
+        sftp = remote_client.open_sftp()
+        logger.info('Downloading \"{0}\"...\n').format(reportname)
+        sftp.get(reportfile, ('{0}/{1}'.format(dl_folder, reportname)))
+        logger.info('Downloading \"{0}\"...\n').format(resultsname)
+        sftp.get(reportfile, ('{0}/{1}'.format(dl_folder, resultsname)))
+        sftp.close()
+        transport.close()
diff --git a/testcases/security_scan/examples/xccdf-rhel7-server-upstream.ini b/testcases/security_scan/examples/xccdf-rhel7-server-upstream.ini
new file mode 100644
index 00000000..cd5d4d27
--- /dev/null
+++ b/testcases/security_scan/examples/xccdf-rhel7-server-upstream.ini
@@ -0,0 +1,25 @@
+[controller]
+port = 22
+user = heat-admin
+user_key = /root/.ssh/stackkey
+scantype = xccdf
+secpolicy = /usr/share/xml/scap/ssg/content/ssg-centos7-xccdf.xml
+cpe = /usr/share/xml/scap/ssg/content/ssg-rhel7-cpe-dictionary.xml
+profile = stig-rhel7-server-upstream
+report = report.hmtl
+results = results.xml
+reports_dir=/home/opnfv/functest/results/security_scan/
+clean = True
+
+[compute]
+port = 22
+user = heat-admin
+user_key = /root/.ssh/stackkey
+scantype = xccdf
+secpolicy = /usr/share/xml/scap/ssg/content/ssg-centos7-xccdf.xml
+cpe = /usr/share/xml/scap/ssg/content/ssg-rhel7-cpe-dictionary.xml
+profile = stig-rhel7-server-upstream
+report = report.hmtl
+results = results.xml
+reports_dir=/home/opnfv/functest/results/security_scan/
+clean = True
diff --git a/testcases/security_scan/examples/xccdf-standard.ini b/testcases/security_scan/examples/xccdf-standard.ini
new file mode 100644
index 00000000..440b23cb
--- /dev/null
+++ b/testcases/security_scan/examples/xccdf-standard.ini
@@ -0,0 +1,25 @@
+[controller]
+port = 22
+user = heat-admin
+user_key = /root/.ssh/stackkey
+scantype = xccdf
+secpolicy = /usr/share/xml/scap/ssg/content/ssg-centos7-xccdf.xml
+cpe = /usr/share/xml/scap/ssg/content/ssg-rhel7-cpe-dictionary.xml
+profile = standard
+report = report.hmtl
+results = results.xml
+reports_dir=/home/opnfv/functest/results/security_scan/
+clean = True
+
+[compute]
+port = 22
+user = heat-admin
+user_key = /root/.ssh/stackkey
+scantype = xccdf
+secpolicy = /usr/share/xml/scap/ssg/content/ssg-centos7-xccdf.xml
+cpe = /usr/share/xml/scap/ssg/content/ssg-rhel7-cpe-dictionary.xml
+profile = standard
+report = report.hmtl
+results = results.xml
+reports_dir=/home/opnfv/functest/results/security_scan/
+clean = True
diff --git a/testcases/security_scan/scripts/createfiles.py b/testcases/security_scan/scripts/createfiles.py
new file mode 100644
index 00000000..b828901a
--- /dev/null
+++ b/testcases/security_scan/scripts/createfiles.py
@@ -0,0 +1,26 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2016 Red Hat
+# Luke Hinds (lhinds@redhat.com)
+# This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# 0.1: This script creates the needed local files into a tmp directory. Should
+# '--clean' be passed, all files will be removed, post scan.
+
+
+import os
+import tempfile
+
+files = ['results.xml', 'report.html', 'syschar.xml']
+
+
+directory_name = tempfile.mkdtemp()
+
+for i in files:
+    os.system("touch %s/%s" % (directory_name, i))
+
+print directory_name
diff --git a/testcases/security_scan/security_scan.py b/testcases/security_scan/security_scan.py
new file mode 100644
index 00000000..f7f02cd9
--- /dev/null
+++ b/testcases/security_scan/security_scan.py
@@ -0,0 +1,178 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2016 Red Hat
+# Luke Hinds (lhinds@redhat.com)
+# This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# 0.1: This script installs OpenSCAP on the remote host, and scans the
+# nominated node. Post scan a report is downloaded and if '--clean' is passed
+# all trace of the scan is removed from the remote system.
+
+import argparse
+import connect
+import datetime
+import os
+
+from ConfigParser import SafeConfigParser
+from keystoneclient.auth.identity import v2
+from keystoneclient import session
+from novaclient import client
+
+__version__ = 0.1
+__author__ = 'Luke Hinds (lhinds@redhat.com)'
+__url__ = 'https://wiki.opnfv.org/display/functest/Functest+Security'
+
+# Global vars
+INSTALLER_IP = os.getenv('INSTALLER_IP')
+oscapbin = 'sudo /bin/oscap'
+
+# Configure Nova Credentials
+com = 'sudo hiera admin_password'
+connect = connect.novaManager(com)
+keypass = connect.keystonepass()
+auth = v2.Password(auth_url='http://{0}:5000/v2.0'.format(INSTALLER_IP),
+                   username='admin',
+                   password=str(keypass).rstrip(),
+                   tenant_name='admin')
+sess = session.Session(auth=auth)
+nova = client.Client(2, session=sess)
+
+
+# args
+parser = argparse.ArgumentParser(description='OPNFV OpenSCAP Scanner')
+parser.add_argument('--config', action='store', dest='cfgfile',
+                    help='Config file', required=True)
+args = parser.parse_args()
+
+# functest logger
+logger = ft_logger.Logger("security_scan").getLogger()
+
+# Config Parser
+cfgparse = SafeConfigParser()
+cfgparse.read(args.cfgfile)
+
+
+def run_tests(host, nodetype):
+    port = cfgparse.get(nodetype, 'port')
+    user = cfgparse.get(nodetype, 'user')
+    user_key = cfgparse.get(nodetype, 'user_key')
+    logger.info("Host: {0} Selected Profile: {1}").format(host, nodetype)
+    logger.info("Creating temp file structure..")
+    createfiles(host, port, user, user_key)
+    logger.info("Installing OpenSCAP...")
+    install_pkg(host, port, user, user_key)
+    logger.info("Running scan...")
+    run_scanner(host, port, user, user_key, nodetype)
+    clean = cfgparse.get(nodetype, 'clean')
+    logger.info("Post installation tasks....")
+    post_tasks(host, port, user, user_key, nodetype)
+    if clean:
+        logger.info("Cleaning down environment....")
+        logger.info("Removing OpenSCAP....")
+        removepkg(host, port, user, user_key, nodetype)
+        logger.info("Deleting tmp file and reports (remote)...")
+        cleandir(host, port, user, user_key, nodetype)
+
+
+def nova_iterate():
+    # Find compute nodes, active with network on ctlplane
+    for server in nova.servers.list():
+        if server.status == 'ACTIVE' and 'compute' in server.name:
+            networks = server.networks
+            nodetype = 'compute'
+            for host in networks['ctlplane']:
+                run_tests(host, nodetype)
+        # Find controller nodes, active with network on ctlplane
+        elif server.status == 'ACTIVE' and 'controller' in server.name:
+            networks = server.networks
+            nodetype = 'controller'
+            for host in networks['ctlplane']:
+                run_tests(host, nodetype)
+
+
+def createfiles(host, port, user, user_key):
+    import connect
+    global tmpdir
+    localpath = os.getcwd() + '/scripts/createfiles.py'
+    remotepath = '/tmp/createfiles.py'
+    com = 'python /tmp/createfiles.py'
+    connect = connect.connectionManager(host, port, user, user_key,
+                                        localpath, remotepath, com)
+    tmpdir = connect.remotescript()
+
+
+def install_pkg(host, port, user, user_key):
+    import connect
+    com = 'sudo yum -y install openscap-scanner scap-security-guide'
+    connect = connect.connectionManager(host, port, user, user_key, com)
+    connect.remotecmd()
+
+
+def run_scanner(host, port, user, user_key, nodetype):
+    import connect
+    scantype = cfgparse.get(nodetype, 'scantype')
+    profile = cfgparse.get(nodetype, 'profile')
+    results = cfgparse.get(nodetype, 'results')
+    report = cfgparse.get(nodetype, 'report')
+    secpolicy = cfgparse.get(nodetype, 'secpolicy')
+    # Here is where we contruct the actual scan command
+    if scantype == 'xccdf':
+        cpe = cfgparse.get(nodetype, 'cpe')
+        com = '{0} xccdf eval --profile {1} --results {2}/{3}' \
+              ' --report {2}/{4} --cpe {5} {6}'.format(oscapbin,
+                                                       profile,
+                                                       tmpdir.rstrip(),
+                                                       results,
+                                                       report,
+                                                       cpe,
+                                                       secpolicy)
+        connect = connect.connectionManager(host, port, user, user_key, com)
+        connect.remotecmd()
+    elif scantype == 'oval':
+        com = '{0} oval eval --results {1}/{2} '
+        '--report {1}/{3} {4}'.format(oscapbin, tmpdir.rstrip(),
+                                      results, report, secpolicy)
+        connect = connect.connectionManager(host, port, user, user_key, com)
+        connect.remotecmd()
+    else:
+        com = '{0} oval-collect '.format(oscapbin)
+        connect = connect.connectionManager(host, port, user, user_key, com)
+        connect.remotecmd()
+
+
+def post_tasks(host, port, user, user_key, nodetype):
+    import connect
+    # Create the download folder for functest dashboard and download reports
+    reports_dir = cfgparse.get(nodetype, 'reports_dir')
+    dl_folder = os.path.join(reports_dir, host + "_" +
+                             datetime.datetime.
+                             now().strftime('%Y-%m-%d_%H-%M-%S'))
+    os.makesdir(dl_folder, 0755)
+    report = cfgparse.get(nodetype, 'report')
+    results = cfgparse.get(nodetype, 'results')
+    reportfile = '{0}/{1}'.format(tmpdir.rstrip(), report)
+    connect = connect.connectionManager(host, port, user, user_key, dl_folder,
+                                        reportfile, report, results)
+    connect.download_reports()
+
+
+def removepkg(host, port, user, user_key, nodetype):
+    import connect
+    com = 'sudo yum -y remove openscap-scanner scap-security-guide'
+    connect = connect.connectionManager(host, port, user, user_key, com)
+    connect.remotecmd()
+
+
+def cleandir(host, port, user, user_key, nodetype):
+    import connect
+    com = 'sudo rm -r {0}'.format(tmpdir.rstrip())
+    connect = connect.connectionManager(host, port, user, user_key, com)
+    connect.remotecmd()
+
+
+if __name__ == '__main__':
+    nova_iterate()