Add Agent Port

Allows the port number to be added to the IP addresses used
for initiating worklaods.  This is done specifically for
containerized workload where the exposed port might need to be
specified per workload agent to allow multiple agents to run
on the same host.

STORPERF-270

Change-Id: If011e686d89e5667509ca57e25f05fba1ed52ed1
Signed-off-by: Mark Beierl <mbeierl@vmware.com>
(cherry picked from commit 24aeda6b73bfa9c31dee8454fdc49bbdb5e1c81b)

5 files changed, 92 insertions, 12 deletions

diff --git a/docker/storperf-master/rest_server.py b/docker/storperf-master/rest_server.py
index 547a0fd..e7f00bb 100644
--- a/docker/storperf-master/rest_server.py
+++ b/docker/storperf-master/rest_server.py
@@ -581,7 +581,8 @@ class WarmUpModel:
         'mkfs': fields.String,
         'mount_point': fields.String,
         'file_size': fields.String,
-        'file_count': fields.String
+        'nrfiles': fields.String,
+        'numjobs': fields.String,
     }
 
 
diff --git a/docker/storperf-master/storperf/fio/fio_invoker.py b/docker/storperf-master/storperf/fio/fio_invoker.py
index 2437763..bb81eef 100644
--- a/docker/storperf-master/storperf/fio/fio_invoker.py
+++ b/docker/storperf-master/storperf/fio/fio_invoker.py
@@ -11,6 +11,7 @@ import json
 import logging
 from threading import Thread
 import paramiko
+from storperf.utilities import ip_helper
 
 
 class FIOInvoker(object):
@@ -158,18 +159,25 @@ class FIOInvoker(object):
     def _ssh_client(self):
         ssh = paramiko.SSHClient()
         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        address, port = ip_helper.parse_address_and_port(self.remote_host)
         if 'username' in self.metadata and 'password' in self.metadata:
-            ssh.connect(self.remote_host,
+            ssh.connect(address,
+                        port=port,
                         username=self.metadata['username'],
-                        password=self.metadata['password'])
+                        password=self.metadata['password'],
+                        timeout=5)
             return ssh
         elif 'username' in self.metadata and 'ssh_key' in self.metadata:
-            ssh.connect(self.remote_host,
+            ssh.connect(address,
+                        port=port,
                         username=self.metadata['username'],
-                        pkey=self.metadata['ssh_key'])
+                        pkey=self.metadata['ssh_key'],
+                        timeout=5)
             return ssh
         else:
-            ssh.connect(self.remote_host, username='storperf',
+            ssh.connect(address,
+                        port=port,
+                        username='storperf',
                         key_filename='storperf/resources/ssh/storperf_rsa',
-                        timeout=2)
+                        timeout=5)
             return ssh
diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py
index afcd018..73f8f0d 100644
--- a/docker/storperf-master/storperf/storperf_master.py
+++ b/docker/storperf-master/storperf/storperf_master.py
@@ -8,8 +8,8 @@
 ##############################################################################
 
 
-from _io import StringIO
 from datetime import datetime
+from io import StringIO
 from multiprocessing.pool import ThreadPool
 from scp import SCPClient
 from snaps.config.stack import StackConfig
@@ -19,6 +19,7 @@ from snaps.openstack.utils import heat_utils, cinder_utils, glance_utils
 from snaps.thread_utils import worker_pool
 from storperf.db.job_db import JobDB
 from storperf.test_executor import TestExecutor
+from storperf.utilities import ip_helper
 from time import sleep
 import json
 import logging
@@ -364,7 +365,7 @@ class StorPerfMaster(object):
     def ssh_key(self):
         if self._ssh_key is None:
             return None
-        key = StringIO.StringIO(self._ssh_key)
+        key = StringIO(self._ssh_key)
         pkey = paramiko.RSAKey.from_private_key(key)
         key.close()
         return pkey
@@ -578,7 +579,8 @@ class StorPerfMaster(object):
         timer = 10
         while not alive:
             s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            result = s.connect_ex((slave, 22))
+            host, port = ip_helper.parse_address_and_port(slave)
+            result = s.connect_ex((host, port))
             s.close()
 
             if result:
@@ -596,19 +598,22 @@ class StorPerfMaster(object):
         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
         if self.username and self.password:
             ssh.connect(
-                slave,
+                host,
+                port=port,
                 username=self.username,
                 password=self.password,
                 timeout=2)
         elif self.username and self.ssh_key:
             ssh.connect(
-                slave,
+                host,
+                port=port,
                 username=self.username,
                 pkey=self.ssh_key,
                 timeout=2)
         else:
             ssh.connect(
                 slave,
+                port=port,
                 username='storperf',
                 key_filename='storperf/resources/ssh/storperf_rsa',
                 timeout=2)
diff --git a/docker/storperf-master/storperf/utilities/ip_helper.py b/docker/storperf-master/storperf/utilities/ip_helper.py
new file mode 100644
index 0000000..06087b0
--- /dev/null
+++ b/docker/storperf-master/storperf/utilities/ip_helper.py
@@ -0,0 +1,27 @@
+##############################################################################
+# Copyright (c) 2019 VMware and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+def parse_address_and_port(address):
+    port = 22
+    if '.' in address:
+        # this is IPv4
+        if ':' in address:
+            host = address.split(':')[0]
+            port = int(address.split(':')[1])
+        else:
+            host = address
+    else:
+        if ']' in address:
+            # this is IPv6
+            host = address.split(']')[0].split('[')[1]
+            port = int(address.split(']')[1].split(':')[1])
+        else:
+            host = address
+    return (host, port)
diff --git a/docker/storperf-master/tests/utilities_tests/ip_helper_test.py b/docker/storperf-master/tests/utilities_tests/ip_helper_test.py
new file mode 100644
index 0000000..f2d662b
--- /dev/null
+++ b/docker/storperf-master/tests/utilities_tests/ip_helper_test.py
@@ -0,0 +1,39 @@
+##############################################################################
+# Copyright (c) 2017 Dell EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import unittest
+
+from storperf.utilities import ip_helper
+
+
+class Test(unittest.TestCase):
+
+    def testNoPortInIPv4(self):
+        host, port = ip_helper.parse_address_and_port("127.0.0.1")
+        self.assertEqual("127.0.0.1", host)
+        self.assertEqual(22, port)
+
+    def testPortInIPv4(self):
+        host, port = ip_helper.parse_address_and_port("127.0.0.1:2222")
+        self.assertEqual("127.0.0.1", host)
+        self.assertEqual(2222, port)
+
+    def testNoPortInIPv6(self):
+        host, port = ip_helper.parse_address_and_port(
+            "1fe80::58bb:c8b:f2f2:c888")
+        self.assertEqual("1fe80::58bb:c8b:f2f2:c888",
+                         host)
+        self.assertEqual(22, port)
+
+    def testPortInIPv6(self):
+        host, port = ip_helper.parse_address_and_port(
+            "[1fe80::58bb:c8b:f2f2:c888]:2222")
+        self.assertEqual("1fe80::58bb:c8b:f2f2:c888",
+                         host)
+        self.assertEqual(2222, port)