14 files changed, 386 insertions, 5 deletions
diff --git a/ci/environments/multinode_major_upgrade.yaml b/ci/environments/multinode_major_upgrade.yaml
index e5c1b411..2251cc0c 100644
--- a/ci/environments/multinode_major_upgrade.yaml
+++ b/ci/environments/multinode_major_upgrade.yaml
@@ -54,6 +54,7 @@ parameter_defaults:
     - OS::TripleO::Services::NovaCompute
     - OS::TripleO::Services::NovaLibvirt
     - OS::TripleO::Services::Pacemaker
+    - OS::TripleO::Services::Horizon
   ControllerExtraConfig:
     nova::compute::libvirt::services::libvirt_virt_type: qemu
     nova::compute::libvirt::libvirt_virt_type: qemu
diff --git a/docker/docker-toool b/docker/docker-toool
new file mode 100755
index 00000000..36aba4a7
--- /dev/null
+++ b/docker/docker-toool
@@ -0,0 +1,189 @@
+#!/usr/bin/env python
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import argparse
+import os
+import shutil
+import sys
+import json
+
+docker_cmd = '/bin/docker'
+
+# Tool to start docker containers as configured via
+# tripleo-heat-templates.
+#
+# This tool reads data from a json file generated from heat when the
+# TripleO stack is run.  All the configuration data used to start the
+# containerized services is in this file.
+#
+# By default this tool lists all the containers that are started and
+# their start order.
+#
+# If you wish to see the command line used to start a given container,
+# specify it by name using the --container argument.  --run can then be
+# used with this to actually execute docker to run the container.\n
+#
+# Other options listed allow you to modify this command line for
+# debugging purposes.  For example:
+#
+# docker-toool -c swift-proxy -r -e /bin/bash -u root -i -n test
+#
+# will run the swift proxy container as user root, executing /bin/bash,
+#
+# named 'test', and will run interactively (eg -ti).
+
+
+def parse_opts(argv):
+    parser = argparse.ArgumentParser("Tool to start docker containers via "
+                                     "TripleO configurations")
+    parser.add_argument('-f', '--config',
+                        help="""File to use as docker startup configuration data.""",
+                        default='/var/lib/docker-container-startup-configs.json')
+    parser.add_argument('-r', '--run',
+                        action='store_true',
+                        help="""Run the container as specified with --container.""",
+                        default=False)
+    parser.add_argument('-e', '--command',
+                        help="""Override the command used to run the container.""",
+                        default='')
+    parser.add_argument('-c', '--container',
+                        help="""Specify a container to run or show the command for.""",
+                        default='')
+    parser.add_argument('-u', '--user',
+                        help="""User to run container as.""",
+                        default='')
+    parser.add_argument('-n', '--name',
+                        help="""Name of container.""",
+                        default='')
+    parser.add_argument('-i', '--interactive',
+                        action='store_true',
+                        help="""Start docker container interactively (-ti).""",
+                        default=False)
+    opts = parser.parse_args(argv[1:])
+
+    return opts
+
+def docker_arg_map(key, value):
+    value = str(value).encode('ascii', 'ignore')
+    return {
+        'environment': "--env=%s" % value,
+        # 'image': value,
+        'net': "--net=%s" % value,
+        'pid': "--pid=%s" % value,
+        'privileged': "--privileged=%s" % value.lower(),
+        #'restart': "--restart=%s" % "false",
+        'user': "--user=%s" % value,
+        'volumes': "--volume=%s" % value,
+        'volumes_from': "--volumes-from=%s" % value,
+    }.get(key, None)
+
+def run_docker_container(opts, container_name):
+    container_found = False
+
+    with open(opts.config) as f:
+        json_data = json.load(f)
+
+    for step in (json_data or []):
+        if step is None:
+            continue
+        for container in (json_data[step] or []):
+            if container == container_name:
+                print('container found: %s' % container)
+                container_found = True
+                # A few positional arguments:
+                command = ''
+                image = ''
+
+                cmd = [
+                    docker_cmd,
+                    'run',
+                    '--name',
+                    opts.name or container
+                ]
+                for container_data in (json_data[step][container] or []):
+                    if container_data == "environment":
+                        for env in (json_data[step][container][container_data] or []):
+                            arg = docker_arg_map("environment", env)
+                            if arg:
+                                cmd.append(arg)
+                    elif container_data == "volumes":
+                        for volume in (json_data[step][container][container_data] or []):
+                            arg = docker_arg_map("volumes", volume)
+                            if arg:
+                                cmd.append(arg)
+                    elif container_data == "volumes_from":
+                        for volume in (json_data[step][container][container_data] or []):
+                            arg = docker_arg_map("volumes_from", volume)
+                            if arg:
+                                cmd.append(arg)
+                    elif container_data == 'command':
+                        command = json_data[step][container][container_data]
+                    elif container_data == 'image':
+                        image = json_data[step][container][container_data]
+                    else:
+                        # Only add a restart if we're not interactive
+                        if container_data == 'restart':
+                            if opts.interactive:
+                                continue
+                        if container_data == 'user':
+                            if opts.user:
+                                continue
+                        arg = docker_arg_map(container_data,
+                                json_data[step][container][container_data])
+                        if arg:
+                            cmd.append(arg)
+
+                if opts.user:
+                    cmd.append('--user')
+                    cmd.append(opts.user)
+                if opts.interactive:
+                    cmd.append('-ti')
+                    # May as well remove it when we're done too
+                    cmd.append('--rm')
+                cmd.append(image)
+                if opts.command:
+                    cmd.append(opts.command)
+                elif command:
+                    cmd.extend(command)
+
+                print ' '.join(cmd)
+
+                if opts.run:
+                    os.execl(docker_cmd, *cmd)
+
+    if not container_found:
+        print("Container '%s' not found!" % container_name)
+
+def list_docker_containers(opts):
+    print opts
+    with open(opts.config) as f:
+        json_data = json.load(f)
+
+    for step in (json_data or []):
+        if step is None:
+            continue
+        print step
+        for container in (json_data[step] or []):
+            print('\tcontainer: %s' % container)
+            for container_data in (json_data[step][container] or []):
+                #print('\t\tcontainer_data: %s' % container_data)
+                if container_data == "start_order":
+                    print('\t\tstart_order: %s' % json_data[step][container][container_data])
+
+opts = parse_opts(sys.argv)
+
+if opts.container:
+    run_docker_container(opts, opts.container)
+else:
+    list_docker_containers(opts)
+
diff --git a/docker/post.j2.yaml b/docker/post.j2.yaml
index e1154a62..65d0c4ee 100644
--- a/docker/post.j2.yaml
+++ b/docker/post.j2.yaml
@@ -189,6 +189,24 @@ resources:
             docker_config: {get_param: [role_data, {{role.name}}, docker_config]}
             docker_image: {get_param: [role_data, {{role.name}}, docker_image]}
 
+  # Here we are dumping all the docker container startup configuration data
+  # so that we can have access to how they are started outside of heat
+  # and docker-cmd.  This lets us create command line tools to start and
+  # test these containers.
+  {{role.name}}DockerConfigJsonStartupData:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: json-file
+      config:
+        /var/lib/docker-container-startup-configs.json:
+          {get_attr: [{{role.name}}DockerConfig, value]}
+
+  {{role.name}}DockerConfigJsonStartupDataDeployment:
+    type: OS::Heat::SoftwareDeploymentGroup
+    properties:
+      config: {get_resource: {{role.name}}DockerConfigJsonStartupData}
+      servers: {get_param: [servers, {{role.name}}]}
+
   {{role.name}}KollaJsonConfig:
     type: OS::Heat::StructuredConfig
     properties:
diff --git a/extraconfig/pre_deploy/rhel-registration/environment-rhel-registration.yaml b/extraconfig/pre_deploy/rhel-registration/environment-rhel-registration.yaml
index c388358a..24557517 100644
--- a/extraconfig/pre_deploy/rhel-registration/environment-rhel-registration.yaml
+++ b/extraconfig/pre_deploy/rhel-registration/environment-rhel-registration.yaml
@@ -21,3 +21,7 @@ parameter_defaults:
   rhel_reg_type: ""
   rhel_reg_method: ""
   rhel_reg_sat_repo: "rhel-7-server-satellite-tools-6.1-rpms"
+  rhel_reg_http_proxy_host: ""
+  rhel_reg_http_proxy_port: ""
+  rhel_reg_http_proxy_username: ""
+  rhel_reg_http_proxy_password: ""
diff --git a/extraconfig/pre_deploy/rhel-registration/rhel-registration.yaml b/extraconfig/pre_deploy/rhel-registration/rhel-registration.yaml
index fdf2e957..e8316c53 100644
--- a/extraconfig/pre_deploy/rhel-registration/rhel-registration.yaml
+++ b/extraconfig/pre_deploy/rhel-registration/rhel-registration.yaml
@@ -45,6 +45,14 @@ parameters:
     type: string
   rhel_reg_sat_repo:
     type: string
+  rhel_reg_http_proxy_host:
+    type: string
+  rhel_reg_http_proxy_port:
+    type: string
+  rhel_reg_http_proxy_username:
+    type: string
+  rhel_reg_http_proxy_password:
+    type: string
 
 resources:
 
@@ -71,6 +79,10 @@ resources:
         - name: REG_TYPE
         - name: REG_METHOD
         - name: REG_SAT_REPO
+        - name: REG_HTTP_PROXY_HOST
+        - name: REG_HTTP_PROXY_PORT
+        - name: REG_HTTP_PROXY_USERNAME
+        - name: REG_HTTP_PROXY_PASSWORD
       config: {get_file: scripts/rhel-registration}
 
   RHELRegistrationDeployment:
@@ -99,6 +111,10 @@ resources:
         REG_TYPE: {get_param: rhel_reg_type}
         REG_METHOD: {get_param: rhel_reg_method}
         REG_SAT_REPO: {get_param: rhel_reg_sat_repo}
+        REG_HTTP_PROXY_HOST: {get_param: rhel_reg_http_proxy_host}
+        REG_HTTP_PROXY_PORT: {get_param: rhel_reg_http_proxy_port}
+        REG_HTTP_PROXY_USERNAME: {get_param: rhel_reg_http_proxy_username}
+        REG_HTTP_PROXY_PASSWORD: {get_param: rhel_reg_http_proxy_password}
 
   RHELUnregistration:
     type: OS::Heat::SoftwareConfig
diff --git a/extraconfig/pre_deploy/rhel-registration/scripts/rhel-registration b/extraconfig/pre_deploy/rhel-registration/scripts/rhel-registration
index 2650a967..4c9e08ef 100644
--- a/extraconfig/pre_deploy/rhel-registration/scripts/rhel-registration
+++ b/extraconfig/pre_deploy/rhel-registration/scripts/rhel-registration
@@ -13,10 +13,18 @@ fi
 
 retryCount=0
 opts=
+config_opts=
 attach_opts=
 sat5_opts=
 repos="repos --enable rhel-7-server-rpms"
 satellite_repo=${REG_SAT_REPO}
+proxy_host=
+proxy_port=
+proxy_url=
+proxy_username=
+proxy_password=
+
+# process variables..
 if [ -n "${REG_AUTO_ATTACH:-}" ]; then
     opts="$opts --auto-attach"
 
@@ -97,6 +105,57 @@ if [ -n "${REG_TYPE:-}" ]; then
     opts="$opts --type=$REG_TYPE"
 fi
 
+# Proxy settings (host and port)
+if [ -n "${REG_HTTP_PROXY_HOST:-}" ]; then
+    proxy_host="${REG_HTTP_PROXY_HOST}"
+fi
+
+if [ -n "${REG_HTTP_PROXY_PORT:-}" ]; then
+    proxy_port="${REG_HTTP_PROXY_PORT}"
+fi
+
+# Proxy settings (user and password)
+if [ -n "${REG_HTTP_PROXY_USERNAME:-}" ]; then
+    proxy_username="${REG_HTTP_PROXY_USERNAME}"
+fi
+
+if [ -n "${REG_HTTP_PROXY_PASSWORD:-}" ]; then
+    proxy_password="${REG_HTTP_PROXY_PASSWORD}"
+fi
+
+# Sanity Checks for proxy host/port/user/password
+if [ -n "${REG_HTTP_PROXY_HOST:-}" ]; then
+    if [ -n "${REG_HTTP_PROXY_PORT:-}" ]; then
+        # Good both values are not empty
+        proxy_url="http://${proxy_host}:${proxy_port}"
+        config_opts="--server.proxy_hostname=${proxy_host} --server.proxy_port=${proxy_port}"
+        sat5_opts="${sat5_opts} --proxy_hostname=${proxy_url}"
+        echo "RHSM Proxy set to: ${proxy_url}"
+        if [ -n "${REG_HTTP_PROXY_USERNAME:-}" ]; then
+            if [ -n "${REG_HTTP_PROXY_PASSWORD:-}" ]; then
+                config_opts="${config_opts} --server.proxy_user=${proxy_username} --server.proxy_password=${proxy_password}"
+                sat5_opts="${sat5_opts} --proxyUser=${proxy_username} --proxyPassword=${proxy_password}"
+            else
+                echo "Warning: REG_HTTP_PROXY_PASSWORD cannot be null with non-empty REG_HTTP_PROXY_USERNAME! Skipping..."
+                proxy_username= ; proxy_password=
+            fi
+        else
+            if [ -n "${REG_HTTP_PROXY_PASSWORD:-}" ]; then
+                echo "Warning: REG_HTTP_PROXY_USERNAME cannot be null with non-empty REG_HTTP_PROXY_PASSWORD! Skipping..."
+                proxy_username= ; proxy_password=
+            fi
+        fi
+    else
+        echo "Warning: REG_HTTP_PROXY_PORT cannot be null with non-empty REG_HTTP_PROXY_HOST! Skipping..."
+        proxy_host= ; proxy_port= ; proxy_url= ; proxy_username= ; proxy_password=
+    fi
+else
+    if [ -n "${REG_HTTP_PROXY_PORT:-}" ]; then
+        echo "Warning: REG_HTTP_PROXY_HOST cannot be null with non-empty REG_HTTP_PROXY_PORT! Skipping..."
+        proxy_host= ; proxy_port= ; proxy_url= ; proxy_username= ; proxy_password=
+    fi
+fi
+
 function retry() {
   if [[ $retryCount < 3 ]]; then
     $@
@@ -127,6 +186,27 @@ function detect_satellite_version {
     fi
 }
 
+if [ "x${proxy_url}" != "x" ];then
+    # Config subscription-manager for proxy
+    subscription-manager config ${config_opts}
+
+    # Config yum for proxy..
+    sed -i -e '/^proxy=/d' /etc/yum.conf
+    echo "proxy=${proxy_url}" >> /etc/yum.conf
+
+    # Handle optional username/password
+    if [ -n "${proxy_username}" ]; then
+        sed -i -e '/^proxy_username=/d' /etc/yum.conf
+        echo "proxy_username=${proxy_username}" >> /etc/yum.conf
+    fi
+
+    if [ -n "${proxy_password}" ]; then
+        sed -i -e '/^proxy_password=/d' /etc/yum.conf
+        echo "proxy_password=${proxy_password}" >> /etc/yum.conf
+    fi
+
+fi
+
 case "${REG_METHOD:-}" in
     portal)
         retry subscription-manager register $opts
diff --git a/puppet/services/auditd.yaml b/puppet/services/auditd.yaml
index 639631e1..a0535ae7 100644
--- a/puppet/services/auditd.yaml
+++ b/puppet/services/auditd.yaml
@@ -32,3 +32,18 @@ outputs:
         auditd::rules: {get_param: AuditdRules}
       step_config: |
         include ::tripleo::profile::base::auditd
+      upgrade_tasks:
+        - name: Check if auditd is deployed
+          command: systemctl is-enabled auditd
+          tags: step0,validation
+          ignore_errors: True
+          register: auditd_enabled
+        - name: "PreUpgrade step0,validation: Check if auditd is running"
+          shell: >
+            /usr/bin/systemctl show 'auditd' --property ActiveState |
+            grep '\bactive\b'
+          when: auditd_enabled.rc == 0
+          tags: step0,validation
+        - name: Stop auditd service
+          tags: step2
+          service: name=auditd state=stopped
diff --git a/puppet/services/ceilometer-agent-central.yaml b/puppet/services/ceilometer-agent-central.yaml
index cf8a8a8e..9baca145 100644
--- a/puppet/services/ceilometer-agent-central.yaml
+++ b/puppet/services/ceilometer-agent-central.yaml
@@ -52,8 +52,9 @@ outputs:
         map_merge:
           - get_attr: [CeilometerServiceBase, role_data, config_settings]
           - ceilometer_redis_password: {get_param: RedisPassword}
+            central_namespace: true
       step_config: |
-        include ::tripleo::profile::base::ceilometer::agent::central
+        include ::tripleo::profile::base::ceilometer::agent::polling
       upgrade_tasks:
         - name: "PreUpgrade step0,validation: Check service openstack-ceilometer-central is running"
           shell: /usr/bin/systemctl show 'openstack-ceilometer-central' --property ActiveState | grep '\bactive\b'
diff --git a/puppet/services/ceilometer-agent-compute.yaml b/puppet/services/ceilometer-agent-compute.yaml
index 00042914..21f5ce63 100644
--- a/puppet/services/ceilometer-agent-compute.yaml
+++ b/puppet/services/ceilometer-agent-compute.yaml
@@ -46,8 +46,9 @@ outputs:
         map_merge:
           - get_attr: [CeilometerServiceBase, role_data, config_settings]
           - ceilometer::agent::compute::instance_discovery_method: {get_param: InstanceDiscoveryMethod}
+            compute_namespace: true
       step_config: |
-        include ::tripleo::profile::base::ceilometer::agent::compute
+        include ::tripleo::profile::base::ceilometer::agent::polling
       upgrade_tasks:
         - name: "PreUpgrade step0,validation: Check service openstack-ceilometer-compute is running"
           shell: /usr/bin/systemctl show 'openstack-ceilometer-compute' --property ActiveState | grep '\bactive\b'
diff --git a/puppet/services/congress.yaml b/puppet/services/congress.yaml
index 6855a838..7d0e9fae 100644
--- a/puppet/services/congress.yaml
+++ b/puppet/services/congress.yaml
@@ -65,9 +65,6 @@ outputs:
               - {get_param: [EndpointMap, MysqlInternal, host]}
               - '/congress'
               - '?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo'
-        congress::keystone::auth::tenant: 'service'
-        congress::keystone::auth::password: {get_param: CongressPassword}
-        congress::keystone::authtoken::auth_uri: {get_param: [EndpointMap, KeystoneInternal, uri]}
         congress::debug: {get_param: Debug}
         congress::rpc_backend: rabbit
         congress::rabbit_userid: {get_param: RabbitUserName}
@@ -76,6 +73,10 @@ outputs:
         congress::rabbit_port: {get_param: RabbitClientPort}
         congress::server::bind_host: {get_param: [ServiceNetMap, CongressApiNetwork]}
 
+        congress::keystone::authtoken::project_name: 'service'
+        congress::keystone::authtoken::auth_url: {get_param: [EndpointMap, KeystoneAdmin, uri_no_suffix]}
+        congress::keystone::authtoken::auth_uri: {get_param: [EndpointMap, KeystoneInternal, uri]}
+
         congress::db::mysql::password: {get_param: CongressPassword}
         congress::db::mysql::user: congress
         congress::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
@@ -84,6 +85,13 @@ outputs:
           - '%'
           - {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
 
+      service_config_settings:
+        keystone:
+          congress::keystone::auth::tenant: 'service'
+          congress::keystone::auth::password: {get_param: CongressPassword}
+          congress::keystone::auth::public_url: {get_param: [EndpointMap, CongressPublic, uri]}
+          congress::keystone::auth::internal_url: {get_param: [EndpointMap, CongressInternal, uri]}
+          congress::keystone::auth::admin_url: {get_param: [EndpointMap, CongressAdmin, uri]}
 
       step_config: |
         include ::tripleo::profile::base::congress
diff --git a/puppet/services/ec2-api.yaml b/puppet/services/ec2-api.yaml
index bb10140e..fefb3220 100644
--- a/puppet/services/ec2-api.yaml
+++ b/puppet/services/ec2-api.yaml
@@ -115,3 +115,19 @@ outputs:
           ec2api::db::mysql::allowed_hosts:
             - '%'
             - "%{hiera('mysql_bind_host')}"
+      upgrade_tasks:
+        - name: Check if ec2-api is deployed
+          command: systemctl is-enabled openstack-ec2-api
+          tags: step0,validation
+          ignore_errors: True
+          register: ec2-api_enabled
+        - name: "PreUpgrade step0,validation: Check if openstack-ec2-api is running"
+          shell: >
+            /usr/bin/systemctl show 'openstack-ec2-api' --property ActiveState |
+            grep '\bactive\b'
+          when: ec2-api_enabled.rc == 0
+          tags: step0,validation
+        - name: Stop openstack-ec2-api service
+          tags: step1
+          service: name=openstack-ec2-api state=stopped
+
diff --git a/puppet/services/horizon.yaml b/puppet/services/horizon.yaml
index 2111021b..77983854 100644
--- a/puppet/services/horizon.yaml
+++ b/puppet/services/horizon.yaml
@@ -96,3 +96,19 @@ outputs:
           - horizon::django_debug: {get_param: Debug}
       step_config: |
         include ::tripleo::profile::base::horizon
+      # Ansible tasks to handle upgrade
+      upgrade_tasks:
+        - name: Check if httpd is deployed
+          command: systemctl is-enabled httpd
+          tags: step0,validation
+          ignore_errors: True
+          register: httpd_enabled
+        - name: "PreUpgrade step0,validation: Check if httpd is running"
+          shell: >
+            /usr/bin/systemctl show 'httpd' --property ActiveState |
+            grep '\bactive\b'
+          when: httpd_enabled.rc == 0
+          tags: step0,validation
+        - name: Stop Horizon (under httpd)
+          tags: step2
+          service: name=httpd state=stopped
diff --git a/puppet/services/keystone.yaml b/puppet/services/keystone.yaml
index 9c4cc60f..5ac8436a 100644
--- a/puppet/services/keystone.yaml
+++ b/puppet/services/keystone.yaml
@@ -226,6 +226,7 @@ outputs:
             keystone::endpoint::internal_url: {get_param: [EndpointMap, KeystoneInternal, uri_no_suffix]}
             keystone::endpoint::admin_url: {get_param: [EndpointMap, KeystoneAdmin, uri_no_suffix]}
             keystone::endpoint::region: {get_param: KeystoneRegion}
+            keystone::endpoint::version: ''
             keystone_enable_db_purge: {get_param: KeystoneEnableDBPurge}
             keystone::rabbit_heartbeat_timeout_threshold: 60
             keystone::cron::token_flush::maxdelay: 3600
diff --git a/puppet/services/zaqar.yaml b/puppet/services/zaqar.yaml
index cb860fa8..bc3a51c2 100644
--- a/puppet/services/zaqar.yaml
+++ b/puppet/services/zaqar.yaml
@@ -64,3 +64,18 @@ outputs:
 
       step_config: |
         include ::tripleo::profile::base::zaqar
+      upgrade_tasks:
+        - name: Check if zaqar is deployed
+          command: systemctl is-enabled openstack-zaqar
+          tags: step0,validation
+          ignore_errors: True
+          register: zaqar_enabled
+        - name: "PreUpgrade step0,validation: Check if openstack-zaqar is running"
+          shell: >
+            /usr/bin/systemctl show 'openstack-zaqar' --property ActiveState |
+            grep '\bactive\b'
+          when: zaqar_enabled.rc == 0
+          tags: step0,validation
+        - name: Stop zaqar service
+          tags: step1
+          service: name=openstack-zaqar state=stopped