Add Agent Port

Allows the port number to be added to the IP addresses used
for initiating worklaods.  This is done specifically for
containerized workload where the exposed port might need to be
specified per workload agent to allow multiple agents to run
on the same host.

STORPERF-270

Change-Id: If011e686d89e5667509ca57e25f05fba1ed52ed1
Signed-off-by: Mark Beierl <mbeierl@vmware.com>

7 files changed, 155 insertions, 12 deletions

diff --git a/docker/storperf-master/rest_server.py b/docker/storperf-master/rest_server.py
index 98f1400..7606eca 100644
--- a/docker/storperf-master/rest_server.py
+++ b/docker/storperf-master/rest_server.py
@@ -596,7 +596,8 @@ class WarmUpModel:
         'mkfs': fields.String,
         'mount_point': fields.String,
         'file_size': fields.String,
-        'file_count': fields.String
+        'nrfiles': fields.String,
+        'numjobs': fields.String,
     }
 
 
diff --git a/docker/storperf-master/storperf/fio/fio_invoker.py b/docker/storperf-master/storperf/fio/fio_invoker.py
index 2437763..bb81eef 100644
--- a/docker/storperf-master/storperf/fio/fio_invoker.py
+++ b/docker/storperf-master/storperf/fio/fio_invoker.py
@@ -11,6 +11,7 @@ import json
 import logging
 from threading import Thread
 import paramiko
+from storperf.utilities import ip_helper
 
 
 class FIOInvoker(object):
@@ -158,18 +159,25 @@ class FIOInvoker(object):
     def _ssh_client(self):
         ssh = paramiko.SSHClient()
         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        address, port = ip_helper.parse_address_and_port(self.remote_host)
         if 'username' in self.metadata and 'password' in self.metadata:
-            ssh.connect(self.remote_host,
+            ssh.connect(address,
+                        port=port,
                         username=self.metadata['username'],
-                        password=self.metadata['password'])
+                        password=self.metadata['password'],
+                        timeout=5)
             return ssh
         elif 'username' in self.metadata and 'ssh_key' in self.metadata:
-            ssh.connect(self.remote_host,
+            ssh.connect(address,
+                        port=port,
                         username=self.metadata['username'],
-                        pkey=self.metadata['ssh_key'])
+                        pkey=self.metadata['ssh_key'],
+                        timeout=5)
             return ssh
         else:
-            ssh.connect(self.remote_host, username='storperf',
+            ssh.connect(address,
+                        port=port,
+                        username='storperf',
                         key_filename='storperf/resources/ssh/storperf_rsa',
-                        timeout=2)
+                        timeout=5)
             return ssh
diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py
index afcd018..73f8f0d 100644
--- a/docker/storperf-master/storperf/storperf_master.py
+++ b/docker/storperf-master/storperf/storperf_master.py
@@ -8,8 +8,8 @@
 ##############################################################################
 
 
-from _io import StringIO
 from datetime import datetime
+from io import StringIO
 from multiprocessing.pool import ThreadPool
 from scp import SCPClient
 from snaps.config.stack import StackConfig
@@ -19,6 +19,7 @@ from snaps.openstack.utils import heat_utils, cinder_utils, glance_utils
 from snaps.thread_utils import worker_pool
 from storperf.db.job_db import JobDB
 from storperf.test_executor import TestExecutor
+from storperf.utilities import ip_helper
 from time import sleep
 import json
 import logging
@@ -364,7 +365,7 @@ class StorPerfMaster(object):
     def ssh_key(self):
         if self._ssh_key is None:
             return None
-        key = StringIO.StringIO(self._ssh_key)
+        key = StringIO(self._ssh_key)
         pkey = paramiko.RSAKey.from_private_key(key)
         key.close()
         return pkey
@@ -578,7 +579,8 @@ class StorPerfMaster(object):
         timer = 10
         while not alive:
             s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            result = s.connect_ex((slave, 22))
+            host, port = ip_helper.parse_address_and_port(slave)
+            result = s.connect_ex((host, port))
             s.close()
 
             if result:
@@ -596,19 +598,22 @@ class StorPerfMaster(object):
         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
         if self.username and self.password:
             ssh.connect(
-                slave,
+                host,
+                port=port,
                 username=self.username,
                 password=self.password,
                 timeout=2)
         elif self.username and self.ssh_key:
             ssh.connect(
-                slave,
+                host,
+                port=port,
                 username=self.username,
                 pkey=self.ssh_key,
                 timeout=2)
         else:
             ssh.connect(
                 slave,
+                port=port,
                 username='storperf',
                 key_filename='storperf/resources/ssh/storperf_rsa',
                 timeout=2)
diff --git a/docker/storperf-master/storperf/utilities/ip_helper.py b/docker/storperf-master/storperf/utilities/ip_helper.py
new file mode 100644
index 0000000..06087b0
--- /dev/null
+++ b/docker/storperf-master/storperf/utilities/ip_helper.py
@@ -0,0 +1,27 @@
+##############################################################################
+# Copyright (c) 2019 VMware and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+def parse_address_and_port(address):
+    port = 22
+    if '.' in address:
+        # this is IPv4
+        if ':' in address:
+            host = address.split(':')[0]
+            port = int(address.split(':')[1])
+        else:
+            host = address
+    else:
+        if ']' in address:
+            # this is IPv6
+            host = address.split(']')[0].split('[')[1]
+            port = int(address.split(']')[1].split(':')[1])
+        else:
+            host = address
+    return (host, port)
diff --git a/docker/storperf-master/tests/utilities_tests/ip_helper_test.py b/docker/storperf-master/tests/utilities_tests/ip_helper_test.py
new file mode 100644
index 0000000..f2d662b
--- /dev/null
+++ b/docker/storperf-master/tests/utilities_tests/ip_helper_test.py
@@ -0,0 +1,39 @@
+##############################################################################
+# Copyright (c) 2017 Dell EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import unittest
+
+from storperf.utilities import ip_helper
+
+
+class Test(unittest.TestCase):
+
+    def testNoPortInIPv4(self):
+        host, port = ip_helper.parse_address_and_port("127.0.0.1")
+        self.assertEqual("127.0.0.1", host)
+        self.assertEqual(22, port)
+
+    def testPortInIPv4(self):
+        host, port = ip_helper.parse_address_and_port("127.0.0.1:2222")
+        self.assertEqual("127.0.0.1", host)
+        self.assertEqual(2222, port)
+
+    def testNoPortInIPv6(self):
+        host, port = ip_helper.parse_address_and_port(
+            "1fe80::58bb:c8b:f2f2:c888")
+        self.assertEqual("1fe80::58bb:c8b:f2f2:c888",
+                         host)
+        self.assertEqual(22, port)
+
+    def testPortInIPv6(self):
+        host, port = ip_helper.parse_address_and_port(
+            "[1fe80::58bb:c8b:f2f2:c888]:2222")
+        self.assertEqual("1fe80::58bb:c8b:f2f2:c888",
+                         host)
+        self.assertEqual(2222, port)
diff --git a/docker/storperf-workloadagent/Dockerfile b/docker/storperf-workloadagent/Dockerfile
new file mode 100644
index 0000000..e6662a9
--- /dev/null
+++ b/docker/storperf-workloadagent/Dockerfile
@@ -0,0 +1,37 @@
+##############################################################################
+# Copyright (c) 2019 VMware and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+#   Docker container for workload
+#
+# Purpose: docker image for Storperf to control as a synthetic workload
+#
+# Maintained by Mark Beierl
+# Build:
+#    $ docker build -t opnfv/storperf-workloadagent:tag .
+#
+
+ARG ARCH=x86_64
+ARG ALPINE_VERSION=v3.10
+FROM multiarch/alpine:$ARCH-$ALPINE_VERSION
+
+RUN apk add --no-cache --upgrade \
+	logrotate \
+	openssh-client \
+	openssh-server \
+	sudo
+
+RUN sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/g' /etc/ssh/sshd_config
+RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config
+
+RUN echo "root ALL=(ALL) ALL" >> /etc/sudoers
+RUN ssh-keygen -f /etc/ssh/ssh_host_rsa_key -N '' -t rsa
+RUN ssh-keygen -f /etc/ssh/ssh_host_dsa_key -N '' -t dsa
+
+RUN echo root:password | chpasswd
+
+CMD /usr/sbin/sshd -D -e
\ No newline at end of file
diff --git a/docs/testing/user/test-usage.rst b/docs/testing/user/test-usage.rst
index ab59cab..0fb3a6c 100644
--- a/docs/testing/user/test-usage.rst
+++ b/docs/testing/user/test-usage.rst
@@ -110,7 +110,33 @@ hypervisor, or even a bare metal server.  In the bare metal case, it even
 allows for performing RADOS or RDB performance tests using the appropriate
 FIO engine.
 
+If the slave SSH server is listening to a port other than 22, the port number
+can be specified as part of the address as follows:
 
+IPv4 example for port 2222:
+
+.. code-block::
+    192.168.1.10:2222
+
+IPv6 example for port 2222:
+
+.. code-block::
+    [1fe80::58bb:c8b:f2f2:c888]:2222
+
+Helper Container Image for Workloads
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A new docker container is provided with StorPerf that can be used to test
+under docker or Kubernetes environments.  It has hard coded credentials
+of root/password with an SSH server built it, so be cautious about security
+concerns when using this image.  It listens internally on port 22, so that
+port must be exposed to a free port on the host in order for StorPerf to
+reach the synthetic workload container.
+
+.. code-block:: bash
+
+    docker run --name=storperf-workloadagent -p 2222:22
+       opnfv/storperf-workloadagent:latest
 
 Initialize the Target Volumes
 =============================