Openstack: Using VSPERF to Test on Openstack.

This patch will support running VSPERF Tests with Openstack.
This patch adds the following:
1. Provide --openstack parameter.
2. New Configuration file for openstack
3. Deploy Trafficgenerator based on configuration provided
4. Run Tests after Trafficgenerator are deployed on openstack

Update-1: Minor bug-fixes and Documentation Added.
Update-2: Add user-config to heat.
Update-3: Update Python Requirements
Update-4: Add dogpile
Update-5: Update decription of the Hot files.

Signed-off-by: Sridhar K. N. Rao <sridhar.rao@spirent.com>
Change-Id: Iebec356eb893e0e6726cac6a10537b99e41f67f4

12 files changed, 662 insertions, 0 deletions

diff --git a/tools/os_deploy_tgen/templates/hotfiles.md b/tools/os_deploy_tgen/templates/hotfiles.md
new file mode 100644
index 00000000..6e21157e
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/hotfiles.md
@@ -0,0 +1,13 @@
+# How to use these HOT Files.
+
+These hot files are referenced in the yaml files.
+Please ensure you are using correct HOT file.
+
+## L2 - No Routers are setup - Same Subnet.
+
+l2fip.hot - Floating IP is configured. Use this if the Openstack environment supports floating IP.
+l2up - Use this if you want username and password configured for the TestVNFs.
+l2.hot - Use this if the 2 interfaces has fixed IPs from 2 different networks. This applies when TestVNF has connectivity to provider network.
+
+## L3 - Routers are setup - Different Subnets
+l3.hot - Setup TestVNFs on two different subnet and connect them with a router.
diff --git a/tools/os_deploy_tgen/templates/l2.hot b/tools/os_deploy_tgen/templates/l2.hot
new file mode 100644
index 00000000..226e8433
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l2.hot
@@ -0,0 +1,89 @@
+heat_template_version: 2013-05-23
+
+description:
+  This Heat template creates a new Neutron network, a router to the external
+  network and plugs instances into this new network. All instances are located
+  in the same L2 domain.
+
+parameters:
+  image:
+    type: string
+    description: Name of image to use for servers
+  flavor:
+    type: string
+    description: Flavor to use for servers
+  external_net:
+    type: string
+    description: ID or name of external network
+#  server_endpoint:
+#    type: string
+#    description: Server endpoint address
+  dns_nameservers:
+    type: comma_delimited_list
+    description: DNS nameservers for the subnet
+
+resources:
+  private_net:
+    type: OS::Neutron::Net
+    properties:
+      name: {{ unique }}_net
+
+  private_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      network_id: { get_resource: private_net }
+      cidr: 172.172.172.0/24
+      dns_nameservers: { get_param: dns_nameservers }
+
+  router:
+    type: OS::Neutron::Router
+    properties:
+      external_gateway_info:
+        network: { get_param: external_net }
+
+  router_interface:
+    type: OS::Neutron::RouterInterface
+    properties:
+      router_id: { get_resource: router }
+      subnet_id: { get_resource: private_subnet }
+
+{% for agent in agents.values() %}
+
+  {{ agent.id }}:
+    type: OS::Nova::Server
+    properties:
+      name: {{ agent.id }}
+      image: { get_param: image }
+      flavor: { get_param: flavor }
+      availability_zone: "{{ agent.availability_zone }}"
+      networks:
+        - port: { get_resource: {{ agent.id }}_port }
+        - port: { get_resource: {{ agent.id }}_mgmt_port }
+
+  {{ agent.id }}_port:
+    type: OS::Neutron::Port
+    properties:
+      network_id: { get_resource: private_net }
+      fixed_ips:
+        - subnet_id: { get_resource: private_subnet }
+
+  {{ agent.id }}_mgmt_port:
+    type: OS::Neutron::Port
+    properties:
+      network_id: { get_param: external_net }
+
+{% endfor %}
+
+outputs:
+{% for agent in agents.values() %}
+  {{ agent.id }}_instance_name:
+    value: { get_attr: [ {{ agent.id }}, instance_name ] }
+  {{ agent.id }}_ip:
+    value: { get_attr: [ {{ agent.id }}_port, fixed_ips, 0, ip_address ] }
+#    value: { get_attr: [ {{ agent.id }}, networks, { get_attr: [private_net, name] }, 0 ] }
+  {{ agent.id }}_pip:
+    value: { get_attr: [ {{ agent.id }}_mgmt_port, fixed_ips, 0, ip_address ] }
+  {{ agent.id }}_dmac:
+    value: { get_attr: [ {{ agent.id }}_port, mac_address ] }
+  
+{% endfor %}
diff --git a/tools/os_deploy_tgen/templates/l2_1c_1i.yaml b/tools/os_deploy_tgen/templates/l2_1c_1i.yaml
new file mode 100644
index 00000000..ec931107
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l2_1c_1i.yaml
@@ -0,0 +1,8 @@
+title: OpenStack L2 Performance
+
+description:
+  In this scenario tdep launches single instances on a tenant network.
+
+deployment:
+  template: l2.hot
+  accommodation: [single_room, compute_nodes: 1]
diff --git a/tools/os_deploy_tgen/templates/l2_1c_2i.yaml b/tools/os_deploy_tgen/templates/l2_1c_2i.yaml
new file mode 100644
index 00000000..4241a80c
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l2_1c_2i.yaml
@@ -0,0 +1,10 @@
+title: OpenStack L2 Performance
+
+description:
+  In this scenario tdep launches 1 pair of instances in the same tenant
+  network. Both the instances are hosted on same compute node.
+  The traffic goes within the tenant network (L2 domain).
+
+deployment:
+  template: l2up.hot
+  accommodation: [pair, single_room, best_effort, compute_nodes: 1]
diff --git a/tools/os_deploy_tgen/templates/l2_2c_2i.yaml b/tools/os_deploy_tgen/templates/l2_2c_2i.yaml
new file mode 100644
index 00000000..b1f54f0a
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l2_2c_2i.yaml
@@ -0,0 +1,10 @@
+title: OpenStack L2 Performance
+
+description:
+  In this scenario tdep launches 1 pair of instances in the same tenant
+  network. Each instance is hosted on a separate compute node. The traffic goes
+  within the tenant network (L2 domain).
+
+deployment:
+  template: l2fip.hot
+  accommodation: [pair, single_room, compute_nodes: 2]
diff --git a/tools/os_deploy_tgen/templates/l2_old.hot b/tools/os_deploy_tgen/templates/l2_old.hot
new file mode 100644
index 00000000..d2553d76
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l2_old.hot
@@ -0,0 +1,93 @@
+heat_template_version: 2013-05-23
+
+description:
+  This Heat template creates a new Neutron network, a router to the external
+  network and plugs instances into this new network. All instances are located
+  in the same L2 domain.
+
+parameters:
+  image:
+    type: string
+    description: Name of image to use for servers
+  flavor:
+    type: string
+    description: Flavor to use for servers
+  external_net:
+    type: string
+    description: ID or name of external network
+#  server_endpoint:
+#    type: string
+#    description: Server endpoint address
+  dns_nameservers:
+    type: comma_delimited_list
+    description: DNS nameservers for the subnet
+
+resources:
+  private_net:
+    type: OS::Neutron::Net
+    properties:
+      name: {{ unique }}_net
+
+  private_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      network_id: { get_resource: private_net }
+      cidr: 10.0.0.0/16
+      dns_nameservers: { get_param: dns_nameservers }
+
+  router:
+    type: OS::Neutron::Router
+    properties:
+      external_gateway_info:
+        network: { get_param: external_net }
+
+  router_interface:
+    type: OS::Neutron::RouterInterface
+    properties:
+      router_id: { get_resource: router }
+      subnet_id: { get_resource: private_subnet }
+
+  server_security_group:
+    type: OS::Neutron::SecurityGroup
+    properties:
+      rules: [
+        {remote_ip_prefix: 0.0.0.0/0,
+        protocol: tcp,
+        port_range_min: 1,
+        port_range_max: 65535},
+        {remote_ip_prefix: 0.0.0.0/0,
+        protocol: udp,
+        port_range_min: 1,
+        port_range_max: 65535},
+        {remote_ip_prefix: 0.0.0.0/0,
+        protocol: icmp}]
+
+{% for agent in agents.values() %}
+
+  {{ agent.id }}:
+    type: OS::Nova::Server
+    properties:
+      name: {{ agent.id }}
+      image: { get_param: image }
+      flavor: { get_param: flavor }
+      availability_zone: "{{ agent.availability_zone }}"
+      networks:
+        - port: { get_resource: {{ agent.id }}_port }
+
+  {{ agent.id }}_port:
+    type: OS::Neutron::Port
+    properties:
+      network_id: { get_resource: private_net }
+      fixed_ips:
+        - subnet_id: { get_resource: private_subnet }
+      security_groups: [{ get_resource: server_security_group }]
+
+{% endfor %}
+
+outputs:
+{% for agent in agents.values() %}
+  {{ agent.id }}_instance_name:
+    value: { get_attr: [ {{ agent.id }}, instance_name ] }
+  {{ agent.id }}_ip:
+    value: { get_attr: [ {{ agent.id }}, networks, { get_attr: [private_net, name] }, 0 ] }
+{% endfor %}
diff --git a/tools/os_deploy_tgen/templates/l2fip.hot b/tools/os_deploy_tgen/templates/l2fip.hot
new file mode 100644
index 00000000..4d4b52f7
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l2fip.hot
@@ -0,0 +1,122 @@
+heat_template_version: 2013-05-23
+
+description:
+  This Heat template creates a new Neutron network, a router to the external
+  network and plugs instances into this new network. All instances are located
+  in the same L2 domain.
+
+parameters:
+  image:
+    type: string
+    description: Name of image to use for servers
+  flavor:
+    type: string
+    description: Flavor to use for servers
+  external_net:
+    type: string
+    description: ID or name of external network
+#  server_endpoint:
+#    type: string
+#    description: Server endpoint address
+  dns_nameservers:
+    type: comma_delimited_list
+    description: DNS nameservers for the subnet
+
+resources:
+  user_config:
+    type: OS::Heat::CloudConfig
+    properties:
+      cloud_config:
+        spirent:
+          driver: "sockets"
+
+  private_net:
+    type: OS::Neutron::Net
+    properties:
+      name: {{ unique }}_net
+      port_security_enabled: false
+
+  private_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      network_id: { get_resource: private_net }
+      cidr: 172.172.172.0/24
+      dns_nameservers: { get_param: dns_nameservers }
+
+  private_datanet:
+    type: OS::Neutron::Net
+    properties:
+      name: {{ unique }}_datanet
+      port_security_enabled: false
+
+  private_datasubnet:
+    type: OS::Neutron::Subnet
+    properties:
+      network_id: { get_resource: private_datanet }
+      cidr: 172.172.168.0/24
+      dns_nameservers: { get_param: dns_nameservers }
+
+  router:
+    type: OS::Neutron::Router
+    properties:
+      external_gateway_info:
+        network: { get_param: external_net }
+
+  router_interface:
+    type: OS::Neutron::RouterInterface
+    properties:
+      router_id: { get_resource: router }
+      subnet_id: { get_resource: private_subnet }
+
+{% for agent in agents.values() %}
+
+  {{ agent.id }}:
+    type: OS::Nova::Server
+    properties:
+      name: {{ agent.id }}
+      image: { get_param: image }
+      flavor: { get_param: flavor }
+      availability_zone: "{{ agent.availability_zone }}"
+      networks:
+        - port: { get_resource: {{ agent.id }}_port }
+        - port: { get_resource: {{ agent.id }}_dataport }
+
+  {{ agent.id }}_port:
+    type: OS::Neutron::Port
+    properties:
+      network_id: { get_resource: private_net }
+      port_security_enabled: false
+      fixed_ips:
+        - subnet_id: { get_resource: private_subnet }
+
+  {{ agent.id }}_dataport:
+    type: OS::Neutron::Port
+    properties:
+      network_id: { get_resource: private_datanet }
+      port_security_enabled: false
+      fixed_ips:
+        - subnet_id: { get_resource: private_datasubnet }
+
+  {{ agent.id }}_fip_port:
+    type: OS::Neutron::FloatingIP
+    depends_on:
+      - router_interface
+    properties:
+      floating_network: { get_param: external_net }
+      port_id: { get_resource: {{ agent.id }}_port }
+      
+
+{% endfor %}
+
+outputs:
+{% for agent in agents.values() %}
+  {{ agent.id }}_instance_name:
+    value: { get_attr: [ {{ agent.id }}, instance_name ] }
+  {{ agent.id }}_ip:
+    value: { get_attr: [ {{ agent.id }}_dataport, fixed_ips, 0, ip_address ] }
+  {{ agent.id }}_pip:
+    value: { get_attr: [ {{ agent.id }}_fip_port, floating_ip_address ] }
+  {{ agent.id }}_dmac:
+    value: { get_attr: [ {{ agent.id }}_dataport, mac_address ] }
+  
+{% endfor %}
diff --git a/tools/os_deploy_tgen/templates/l2up.hot b/tools/os_deploy_tgen/templates/l2up.hot
new file mode 100644
index 00000000..58f25831
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l2up.hot
@@ -0,0 +1,126 @@
+heat_template_version: 2013-05-23
+
+description:
+  This Heat template creates a new Neutron network, a router to the external
+  network and plugs instances into this new network. All instances are located
+  in the same L2 domain.
+
+parameters:
+  image:
+    type: string
+    description: Name of image to use for servers
+  flavor:
+    type: string
+    description: Flavor to use for servers
+  external_net:
+    type: string
+    description: ID or name of external network
+#  server_endpoint:
+#    type: string
+#    description: Server endpoint address
+  dns_nameservers:
+    type: comma_delimited_list
+    description: DNS nameservers for the subnet
+
+resources:
+  private_net:
+    type: OS::Neutron::Net
+    properties:
+      name: {{ unique }}_net
+
+  private_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      network_id: { get_resource: private_net }
+      cidr: 172.172.172.0/24
+      dns_nameservers: { get_param: dns_nameservers }
+
+  router:
+    type: OS::Neutron::Router
+    properties:
+      external_gateway_info:
+        network: { get_param: external_net }
+
+  router_interface:
+    type: OS::Neutron::RouterInterface
+    properties:
+      router_id: { get_resource: router }
+      subnet_id: { get_resource: private_subnet }
+
+  user_config:
+    type: OS::Heat::CloudConfig
+    properties:
+      cloud_config:
+        users:
+        - default
+        - name: test
+          groups: "users,root"
+          lock-passwd: false
+          passwd: 'test'
+          shell: "/bin/bash"
+          sudo: "ALL=(ALL) NOPASSWD:ALL"
+        ssh_pwauth: true
+        chpasswd:
+          list:  |
+              test:test
+          expire: False
+
+  server_security_group:
+    type: OS::Neutron::SecurityGroup
+    properties:
+      rules: [
+        {remote_ip_prefix: 0.0.0.0/0,
+        protocol: tcp,
+        port_range_min: 1,
+        port_range_max: 65535},
+        {remote_ip_prefix: 0.0.0.0/0,
+        protocol: udp,
+        port_range_min: 1,
+        port_range_max: 65535},
+        {remote_ip_prefix: 0.0.0.0/0,
+        protocol: icmp}]
+
+{% for agent in agents.values() %}
+
+  {{ agent.id }}:
+    type: OS::Nova::Server
+    properties:
+      name: {{ agent.id }}
+      image: { get_param: image }
+      flavor: { get_param: flavor }
+      availability_zone: "{{ agent.availability_zone }}"
+      networks:
+        - port: { get_resource: {{ agent.id }}_port }
+        - port: { get_resource: {{ agent.id }}_mgmt_port }
+      user_data: {get_resource: user_config}
+      user_data_format: RAW
+
+  {{ agent.id }}_port:
+    type: OS::Neutron::Port
+    properties:
+      network_id: { get_resource: private_net }
+      fixed_ips:
+        - subnet_id: { get_resource: private_subnet }
+      security_groups: [{ get_resource: server_security_group }]
+
+  {{ agent.id }}_mgmt_port:
+    type: OS::Neutron::Port
+    properties:
+      network_id: { get_param: external_net }
+      security_groups: [{ get_resource: server_security_group }]
+
+{% endfor %}
+
+outputs:
+{% for agent in agents.values() %}
+  {{ agent.id }}_instance_name:
+    value: { get_attr: [ {{ agent.id }}, instance_name ] }
+  {{ agent.id }}_ip:
+    value: { get_attr: [ {{ agent.id }}_port, fixed_ips, 0, ip_address ] }
+#    value: { get_attr: [ {{ agent.id }}, networks, { get_attr: [private_net, name] }, 0 ] }
+  {{ agent.id }}_pip:
+    value: { get_attr: [ {{ agent.id }}_mgmt_port, fixed_ips, 0, ip_address ] }
+  {{ agent.id }}_dmac:
+    value: { get_attr: [ {{ agent.id }}_port, mac_address ] }
+  
+{% endfor %}
diff --git a/tools/os_deploy_tgen/templates/l3.hot b/tools/os_deploy_tgen/templates/l3.hot
new file mode 100644
index 00000000..4a5ea02c
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l3.hot
@@ -0,0 +1,125 @@
+heat_template_version: 2013-05-23
+
+description:
+  This Heat template creates a pair of networks plugged into the same router.
+  Master instances and slave instances are connected into different networks.
+
+parameters:
+  image:
+    type: string
+    description: Name of image to use for servers
+  flavor:
+    type: string
+    description: Flavor to use for servers
+  external_net:
+    type: string
+    description: ID or name of external network for which floating IP addresses will be allocated
+#  server_endpoint:
+#    type: string
+#    description: Server endpoint address
+  dns_nameservers:
+    type: comma_delimited_list
+    description: DNS nameservers for the subnets
+
+resources:
+  east_private_net:
+    type: OS::Neutron::Net
+    properties:
+      name: {{ unique }}_net_east
+
+  east_private_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      network_id: { get_resource: east_private_net }
+      cidr: 10.1.0.0/16
+      dns_nameservers: { get_param: dns_nameservers }
+
+  router:
+    type: OS::Neutron::Router
+    properties:
+      external_gateway_info:
+        network: { get_param: external_net }
+
+  router_interface:
+    type: OS::Neutron::RouterInterface
+    properties:
+      router_id: { get_resource: router }
+      subnet_id: { get_resource: east_private_subnet }
+
+  west_private_net:
+    type: OS::Neutron::Net
+    properties:
+      name: {{ unique }}_net_west
+
+  west_private_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      network_id: { get_resource: west_private_net }
+      cidr: 10.2.0.0/16
+      dns_nameservers: { get_param: dns_nameservers }
+
+  router_interface_2:
+    type: OS::Neutron::RouterInterface
+    properties:
+      router_id: { get_resource: router }
+      subnet_id: { get_resource: west_private_subnet }
+
+  server_security_group:
+    type: OS::Neutron::SecurityGroup
+    properties:
+      rules: [
+        {remote_ip_prefix: 0.0.0.0/0,
+        protocol: tcp,
+        port_range_min: 1,
+        port_range_max: 65535},
+        {remote_ip_prefix: 0.0.0.0/0,
+        protocol: udp,
+        port_range_min: 1,
+        port_range_max: 65535},
+        {remote_ip_prefix: 0.0.0.0/0,
+        protocol: icmp}]
+
+{% for agent in agents.values() %}
+
+  {{ agent.id }}:
+    type: OS::Nova::Server
+    properties:
+      name: {{ agent.id }}
+      image: { get_param: image }
+      flavor: { get_param: flavor }
+      availability_zone: "{{ agent.availability_zone }}"
+      networks:
+        - port: { get_resource: {{ agent.id }}_port }
+
+{% if agent.mode == 'master' %}
+  {{ agent.id }}_port:
+    type: OS::Neutron::Port
+    properties:
+      network_id: { get_resource: east_private_net }
+      fixed_ips:
+        - subnet_id: { get_resource: east_private_subnet }
+      security_groups: [{ get_resource: server_security_group }]
+{% else %}
+  {{ agent.id }}_port:
+    type: OS::Neutron::Port
+    properties:
+      network_id: { get_resource: west_private_net }
+      fixed_ips:
+        - subnet_id: { get_resource: west_private_subnet }
+      security_groups: [{ get_resource: server_security_group }]
+{% endif %}
+
+{% endfor %}
+
+outputs:
+{% for agent in agents.values() %}
+  {{ agent.id }}_instance_name:
+    value: { get_attr: [ {{ agent.id }}, instance_name ] }
+{% if agent.mode == 'master' %}
+  {{ agent.id }}_ip:
+    value: { get_attr: [ {{ agent.id }}, networks, { get_attr: [east_private_net, name] }, 0 ] }
+{% else %}
+  {{ agent.id }}_ip:
+    value: { get_attr: [ {{ agent.id }}, networks, { get_attr: [west_private_net, name] }, 0 ] }
+{% endif %}
+{% endfor %}
diff --git a/tools/os_deploy_tgen/templates/l3_1c_2i.yaml b/tools/os_deploy_tgen/templates/l3_1c_2i.yaml
new file mode 100644
index 00000000..0908843c
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l3_1c_2i.yaml
@@ -0,0 +1,11 @@
+title: OpenStack L3 East-West Performance
+
+description:
+  In this scenario tdep launches 1 pair of instances, both instances on same
+  compute node. Instances are connected to one of 2 tenant networks, which
+  plugged into single router. The traffic goes from one network to the other
+  (L3 east-west).
+
+deployment:
+  template: l3.hot
+  accommodation: [pair, single_room, best_effort, compute_nodes: 2]
diff --git a/tools/os_deploy_tgen/templates/l3_2c_2i.yaml b/tools/os_deploy_tgen/templates/l3_2c_2i.yaml
new file mode 100644
index 00000000..67aee170
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l3_2c_2i.yaml
@@ -0,0 +1,11 @@
+title: OpenStack L3 East-West Performance
+
+description:
+  In this scenario tdep launches 1 pair of instances, each instance on its own
+  compute node. Instances are connected to one of 2 tenant networks, which
+  plugged into single router. The traffic goes from one network to the other
+  (L3 east-west).
+
+deployment:
+  template: l3.hot
+  accommodation: [pair, single_room, compute_nodes: 2]
diff --git a/tools/os_deploy_tgen/templates/scenario.yaml b/tools/os_deploy_tgen/templates/scenario.yaml
new file mode 100644
index 00000000..c66ec734
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/scenario.yaml
@@ -0,0 +1,44 @@
+name: tdep scenario schema
+type: map
+allowempty: True
+mapping:
+  title:
+    type: str
+  description:
+    type: str
+  deployment:
+    type: map
+    mapping:
+      support_templates:
+        type: seq
+        sequence:
+          - type: map
+            mapping:
+              name:
+                type: str
+              template:
+                type: str
+              env_file:
+                type: str
+      template:
+        type: str
+      env_file:
+        type: str
+      agents:
+        type: any
+      accommodation:
+        type: seq
+        matching: any
+        sequence:
+        - type: str
+          enum: [pair, alone, double_room, single_room, mixed_room, cross_az, best_effort]
+        - type: map
+          mapping:
+            density:
+              type: number
+            compute_nodes:
+              type: number
+            zones:
+              type: seq
+              sequence:
+              - type: str