Merge "Add in roles data validation"

12 files changed, 152 insertions, 3 deletions

diff --git a/bindep.txt b/bindep.txt
index 4f9b4254..d42375eb 100644
--- a/bindep.txt
+++ b/bindep.txt
@@ -1,2 +1,4 @@
 # This is a cross-platform list tracking distribution packages needed by tests;
 # see http://docs.openstack.org/infra/bindep/ for additional information.
+libssl-dev [platform:dpkg]
+openssl-devel [platform:rpm]
diff --git a/releasenotes/notes/roles-data-validation-7845702b5ed85366.yaml b/releasenotes/notes/roles-data-validation-7845702b5ed85366.yaml
new file mode 100644
index 00000000..766b6581
--- /dev/null
+++ b/releasenotes/notes/roles-data-validation-7845702b5ed85366.yaml
@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    The roles_data.yaml and roles_data_undercloud.yaml can be generated with
+    tox using ``tox -e genrolesdata``.
+  - |
+    pep8 now checks that the roles_data.yaml and roles_data_undercloud.yaml
+    matches data generated from the roles/ files.
diff --git a/requirements.txt b/requirements.txt
index 4a9b7253..d4e343ec 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,3 +4,4 @@
 pbr!=2.1.0,>=2.0.0 # Apache-2.0
 Jinja2!=2.9.0,!=2.9.1,!=2.9.2,!=2.9.3,!=2.9.4,>=2.8 # BSD License (3 clause)
 six>=1.9.0 # MIT
+tripleo-common>=7.1.0 # Apache-2.0
diff --git a/roles/Controller.yaml b/roles/Controller.yaml
index 40e51aa8..e3af321e 100644
--- a/roles/Controller.yaml
+++ b/roles/Controller.yaml
@@ -44,9 +44,9 @@
     - OS::TripleO::Services::CinderHPELeftHandISCSI
     - OS::TripleO::Services::CinderScheduler
     - OS::TripleO::Services::CinderVolume
+    - OS::TripleO::Services::Clustercheck
     - OS::TripleO::Services::Collectd
     - OS::TripleO::Services::Congress
-    - OS::TripleO::Services::Clustercheck
     - OS::TripleO::Services::Docker
     - OS::TripleO::Services::Ec2Api
     - OS::TripleO::Services::Etcd
diff --git a/roles/Undercloud.yaml b/roles/Undercloud.yaml
index bcdedc71..f56749a9 100644
--- a/roles/Undercloud.yaml
+++ b/roles/Undercloud.yaml
@@ -18,6 +18,7 @@
     - OS::TripleO::Services::HeatEngine
     - OS::TripleO::Services::IronicApi
     - OS::TripleO::Services::IronicConductor
+    - OS::TripleO::Services::IronicInspector
     - OS::TripleO::Services::IronicPxe
     - OS::TripleO::Services::Iscsid
     - OS::TripleO::Services::Keystone
@@ -48,6 +49,7 @@
     - OS::TripleO::Services::UndercloudAodhListener
     - OS::TripleO::Services::UndercloudAodhNotifier
     - OS::TripleO::Services::UndercloudCeilometerAgentCentral
+    - OS::TripleO::Services::UndercloudCeilometerAgentIpmi
     - OS::TripleO::Services::UndercloudCeilometerAgentNotification
     - OS::TripleO::Services::UndercloudGnocchiApi
     - OS::TripleO::Services::UndercloudGnocchiMetricd
diff --git a/roles_data.yaml b/roles_data.yaml
index 7bb87c2e..fe24a423 100644
--- a/roles_data.yaml
+++ b/roles_data.yaml
@@ -1,5 +1,5 @@
 ###############################################################################
-# File generated by tripleoclient
+# File generated by TripleO
 ###############################################################################
 ###############################################################################
 # Role: Controller                                                            #
@@ -261,3 +261,4 @@
     - OS::TripleO::Services::Timezone
     - OS::TripleO::Services::TripleoFirewall
     - OS::TripleO::Services::TripleoPackages
+
diff --git a/roles_data_undercloud.yaml b/roles_data_undercloud.yaml
index 783df91d..2aa5a291 100644
--- a/roles_data_undercloud.yaml
+++ b/roles_data_undercloud.yaml
@@ -1,5 +1,5 @@
 ###############################################################################
-# File generated by tripleoclient
+# File generated by TripleO
 ###############################################################################
 ###############################################################################
 # Role: Undercloud                                                            #
@@ -59,3 +59,4 @@
     - OS::TripleO::Services::UndercloudGnocchiStatsd
     - OS::TripleO::Services::UndercloudPankoApi
     - OS::TripleO::Services::Zaqar
+
diff --git a/tools/roles-data-generate-samples.sh b/tools/roles-data-generate-samples.sh
new file mode 100755
index 00000000..cb370dcc
--- /dev/null
+++ b/tools/roles-data-generate-samples.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+set -e
+
+SCRIPT_DIR=$(cd `dirname $0` && pwd -P)
+OUTPUT_DIR=${OUTPUT_DIR:-$(cd "${SCRIPT_DIR}/../" && pwd -P)}
+
+echo "Generating ${OUTPUT_DIR}/roles_data.yaml"
+$SCRIPT_DIR/roles-data-generate.py Controller Compute BlockStorage ObjectStorage CephStorage > $OUTPUT_DIR/roles_data.yaml
+
+echo "Generating ${OUTPUT_DIR}/roles_data_undercloud.yaml"
+$SCRIPT_DIR/roles-data-generate.py Undercloud > $OUTPUT_DIR/roles_data_undercloud.yaml
diff --git a/tools/roles-data-generate.py b/tools/roles-data-generate.py
new file mode 100755
index 00000000..0b768744
--- /dev/null
+++ b/tools/roles-data-generate.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+#
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+import argparse
+import collections
+import os
+import sys
+
+from tripleo_common.utils import roles as rolesutils
+
+__tht_root_dir = os.path.dirname(os.path.dirname(__file__))
+__tht_roles_dir = os.path.join(__tht_root_dir, 'roles')
+
+
+def parse_opts(argv):
+    parser = argparse.ArgumentParser(
+        description='Generate roles_data.yaml for requested roles. NOTE: '
+                    'This is a stripped down version of what is provided by '
+                    'the tripleoclient. The tripleoclient should be used for '
+                    'additional functionality.')
+    parser.add_argument('--roles-path', metavar='<roles directory>',
+                        help="Filesystem path containing the roles yaml files",
+                        default=__tht_roles_dir)
+    parser.add_argument('roles', nargs="+", metavar='<role>',
+                        help='List of roles to use to generate the '
+                             'roles_data.yaml file')
+    opts = parser.parse_args(argv[1:])
+
+    return opts
+
+opts = parse_opts(sys.argv)
+
+roles = collections.OrderedDict.fromkeys(opts.roles)
+print(rolesutils.generate_roles_data_from_directory(opts.roles_path,
+                                                    roles.keys()))
diff --git a/tools/roles-data-validation.sh b/tools/roles-data-validation.sh
new file mode 100755
index 00000000..7c5786e1
--- /dev/null
+++ b/tools/roles-data-validation.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+set -e
+
+SCRIPT_DIR=$(cd `dirname $0` && pwd -P)
+THT_DIR=${OUTPUT_DIR:-$(cd "${SCRIPT_DIR}/../" && pwd -P)}
+TMPDIR=$(mktemp -d)
+
+function do_cleanup {
+  rm -rf $TMPDIR
+}
+trap do_cleanup EXIT
+
+function check_diff {
+  local thtfile=$1
+  local genfile=$2
+  echo -n "Performing diff on $thtfile $genfile... "
+  diff $thtfile $genfile > $TMPDIR/diff_results
+  if [ $? = 1 ]; then
+      echo "ERROR: Generated roles file not match the current ${thtfile}"
+      echo "Please make sure to update the appropriate roles/* files."
+      echo "Here is the diff ${thtfile} ${genfile}"
+      cat $TMPDIR/diff_results
+      exit 1
+  fi
+  echo "OK!"
+}
+
+OUTPUT_DIR=$TMPDIR
+source $SCRIPT_DIR/roles-data-generate-samples.sh
+
+set +e
+check_diff $THT_DIR/roles_data.yaml $TMPDIR/roles_data.yaml
+check_diff $THT_DIR/roles_data_undercloud.yaml $TMPDIR/roles_data_undercloud.yaml
diff --git a/tools/yaml-validate.py b/tools/yaml-validate.py
index f9e89db2..e9581a2d 100755
--- a/tools/yaml-validate.py
+++ b/tools/yaml-validate.py
@@ -337,6 +337,8 @@ param_map = {}
 for base_path in path_args:
     if os.path.isdir(base_path):
         for subdir, dirs, files in os.walk(base_path):
+            if '.tox' in dirs:
+                dirs.remove('.tox')
             for f in files:
                 if f.endswith('.yaml') and not f.endswith('.j2.yaml'):
                     file_path = os.path.join(subdir, f)
diff --git a/tox.ini b/tox.ini
index c87bf7be..da179e91 100644
--- a/tox.ini
+++ b/tox.ini
@@ -14,10 +14,13 @@ commands = python setup.py testr --slowest --testr-args='{posargs}'
 commands = {posargs}
 
 [testenv:pep8]
+whielist_externals =
+    bash
 commands =
     python ./tools/process-templates.py
     python ./network/endpoints/build_endpoint_map.py --check
     python ./tools/yaml-validate.py .
+    bash -c ./tools/roles-data-validation.sh
 
 [testenv:templates]
 commands = python ./tools/process-templates.py
@@ -32,3 +35,9 @@ commands = python setup.py test --coverage --coverage-package-name=tripleo_heat_
 commands =
            python ./tools/process-templates.py
            python ./tripleo_heat_templates/environment_generator.py sample-env-generator/
+
+[testenv:genroledata]
+whielist_externals =
+    bash
+commands =
+    bash -c tools/roles-data-generate-samples.sh