From 2eb1476b6e93279d4ee4988c856dba50ab168bc5 Mon Sep 17 00:00:00 2001 From: Alex Schultz Date: Fri, 9 Jun 2017 08:59:51 -0600 Subject: Add in roles data validation With the merging of Iad3e9b215c6f21ba761c8360bb7ed531e34520e6 the roles_data.yaml should be generated with tripleoclient rather than edited. This change adds in a pep8 task to verify that the appropriate role files in roles/ have been modified to match how our default roles_data.yaml is constructed. Additionally this change adds a new tox target called 'genrolesdata' that will all you to automatically generate roles_data.yaml and roles_data_undercloud.yaml Change-Id: I5eb15443a131a122d1a4abf6fc15a3ac3e15941b Related-Blueprint: example-custom-role-environments --- bindep.txt | 2 + .../roles-data-validation-7845702b5ed85366.yaml | 8 ++++ requirements.txt | 1 + roles/Controller.yaml | 2 +- roles/Undercloud.yaml | 2 + roles_data.yaml | 3 +- roles_data_undercloud.yaml | 3 +- tools/roles-data-generate-samples.sh | 26 ++++++++++++ tools/roles-data-generate.py | 49 ++++++++++++++++++++++ tools/roles-data-validation.sh | 48 +++++++++++++++++++++ tools/yaml-validate.py | 2 + tox.ini | 9 ++++ 12 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/roles-data-validation-7845702b5ed85366.yaml create mode 100755 tools/roles-data-generate-samples.sh create mode 100755 tools/roles-data-generate.py create mode 100755 tools/roles-data-validation.sh diff --git a/bindep.txt b/bindep.txt index 4f9b4254..d42375eb 100644 --- a/bindep.txt +++ b/bindep.txt @@ -1,2 +1,4 @@ # This is a cross-platform list tracking distribution packages needed by tests; # see http://docs.openstack.org/infra/bindep/ for additional information. +libssl-dev [platform:dpkg] +openssl-devel [platform:rpm] diff --git a/releasenotes/notes/roles-data-validation-7845702b5ed85366.yaml b/releasenotes/notes/roles-data-validation-7845702b5ed85366.yaml new file mode 100644 index 00000000..766b6581 --- /dev/null +++ b/releasenotes/notes/roles-data-validation-7845702b5ed85366.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + The roles_data.yaml and roles_data_undercloud.yaml can be generated with + tox using ``tox -e genrolesdata``. + - | + pep8 now checks that the roles_data.yaml and roles_data_undercloud.yaml + matches data generated from the roles/ files. diff --git a/requirements.txt b/requirements.txt index 4a9b7253..d4e343ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ pbr!=2.1.0,>=2.0.0 # Apache-2.0 Jinja2!=2.9.0,!=2.9.1,!=2.9.2,!=2.9.3,!=2.9.4,>=2.8 # BSD License (3 clause) six>=1.9.0 # MIT +tripleo-common>=7.1.0 # Apache-2.0 diff --git a/roles/Controller.yaml b/roles/Controller.yaml index 40e51aa8..e3af321e 100644 --- a/roles/Controller.yaml +++ b/roles/Controller.yaml @@ -44,9 +44,9 @@ - OS::TripleO::Services::CinderHPELeftHandISCSI - OS::TripleO::Services::CinderScheduler - OS::TripleO::Services::CinderVolume + - OS::TripleO::Services::Clustercheck - OS::TripleO::Services::Collectd - OS::TripleO::Services::Congress - - OS::TripleO::Services::Clustercheck - OS::TripleO::Services::Docker - OS::TripleO::Services::Ec2Api - OS::TripleO::Services::Etcd diff --git a/roles/Undercloud.yaml b/roles/Undercloud.yaml index bcdedc71..f56749a9 100644 --- a/roles/Undercloud.yaml +++ b/roles/Undercloud.yaml @@ -18,6 +18,7 @@ - OS::TripleO::Services::HeatEngine - OS::TripleO::Services::IronicApi - OS::TripleO::Services::IronicConductor + - OS::TripleO::Services::IronicInspector - OS::TripleO::Services::IronicPxe - OS::TripleO::Services::Iscsid - OS::TripleO::Services::Keystone @@ -48,6 +49,7 @@ - OS::TripleO::Services::UndercloudAodhListener - OS::TripleO::Services::UndercloudAodhNotifier - OS::TripleO::Services::UndercloudCeilometerAgentCentral + - OS::TripleO::Services::UndercloudCeilometerAgentIpmi - OS::TripleO::Services::UndercloudCeilometerAgentNotification - OS::TripleO::Services::UndercloudGnocchiApi - OS::TripleO::Services::UndercloudGnocchiMetricd diff --git a/roles_data.yaml b/roles_data.yaml index 7bb87c2e..fe24a423 100644 --- a/roles_data.yaml +++ b/roles_data.yaml @@ -1,5 +1,5 @@ ############################################################################### -# File generated by tripleoclient +# File generated by TripleO ############################################################################### ############################################################################### # Role: Controller # @@ -261,3 +261,4 @@ - OS::TripleO::Services::Timezone - OS::TripleO::Services::TripleoFirewall - OS::TripleO::Services::TripleoPackages + diff --git a/roles_data_undercloud.yaml b/roles_data_undercloud.yaml index 783df91d..2aa5a291 100644 --- a/roles_data_undercloud.yaml +++ b/roles_data_undercloud.yaml @@ -1,5 +1,5 @@ ############################################################################### -# File generated by tripleoclient +# File generated by TripleO ############################################################################### ############################################################################### # Role: Undercloud # @@ -59,3 +59,4 @@ - OS::TripleO::Services::UndercloudGnocchiStatsd - OS::TripleO::Services::UndercloudPankoApi - OS::TripleO::Services::Zaqar + diff --git a/tools/roles-data-generate-samples.sh b/tools/roles-data-generate-samples.sh new file mode 100755 index 00000000..cb370dcc --- /dev/null +++ b/tools/roles-data-generate-samples.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Copyright 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -e + +SCRIPT_DIR=$(cd `dirname $0` && pwd -P) +OUTPUT_DIR=${OUTPUT_DIR:-$(cd "${SCRIPT_DIR}/../" && pwd -P)} + +echo "Generating ${OUTPUT_DIR}/roles_data.yaml" +$SCRIPT_DIR/roles-data-generate.py Controller Compute BlockStorage ObjectStorage CephStorage > $OUTPUT_DIR/roles_data.yaml + +echo "Generating ${OUTPUT_DIR}/roles_data_undercloud.yaml" +$SCRIPT_DIR/roles-data-generate.py Undercloud > $OUTPUT_DIR/roles_data_undercloud.yaml diff --git a/tools/roles-data-generate.py b/tools/roles-data-generate.py new file mode 100755 index 00000000..0b768744 --- /dev/null +++ b/tools/roles-data-generate.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# +# Copyright 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +import argparse +import collections +import os +import sys + +from tripleo_common.utils import roles as rolesutils + +__tht_root_dir = os.path.dirname(os.path.dirname(__file__)) +__tht_roles_dir = os.path.join(__tht_root_dir, 'roles') + + +def parse_opts(argv): + parser = argparse.ArgumentParser( + description='Generate roles_data.yaml for requested roles. NOTE: ' + 'This is a stripped down version of what is provided by ' + 'the tripleoclient. The tripleoclient should be used for ' + 'additional functionality.') + parser.add_argument('--roles-path', metavar='', + help="Filesystem path containing the roles yaml files", + default=__tht_roles_dir) + parser.add_argument('roles', nargs="+", metavar='', + help='List of roles to use to generate the ' + 'roles_data.yaml file') + opts = parser.parse_args(argv[1:]) + + return opts + +opts = parse_opts(sys.argv) + +roles = collections.OrderedDict.fromkeys(opts.roles) +print(rolesutils.generate_roles_data_from_directory(opts.roles_path, + roles.keys())) diff --git a/tools/roles-data-validation.sh b/tools/roles-data-validation.sh new file mode 100755 index 00000000..7c5786e1 --- /dev/null +++ b/tools/roles-data-validation.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Copyright 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -e + +SCRIPT_DIR=$(cd `dirname $0` && pwd -P) +THT_DIR=${OUTPUT_DIR:-$(cd "${SCRIPT_DIR}/../" && pwd -P)} +TMPDIR=$(mktemp -d) + +function do_cleanup { + rm -rf $TMPDIR +} +trap do_cleanup EXIT + +function check_diff { + local thtfile=$1 + local genfile=$2 + echo -n "Performing diff on $thtfile $genfile... " + diff $thtfile $genfile > $TMPDIR/diff_results + if [ $? = 1 ]; then + echo "ERROR: Generated roles file not match the current ${thtfile}" + echo "Please make sure to update the appropriate roles/* files." + echo "Here is the diff ${thtfile} ${genfile}" + cat $TMPDIR/diff_results + exit 1 + fi + echo "OK!" +} + +OUTPUT_DIR=$TMPDIR +source $SCRIPT_DIR/roles-data-generate-samples.sh + +set +e +check_diff $THT_DIR/roles_data.yaml $TMPDIR/roles_data.yaml +check_diff $THT_DIR/roles_data_undercloud.yaml $TMPDIR/roles_data_undercloud.yaml diff --git a/tools/yaml-validate.py b/tools/yaml-validate.py index f9e89db2..e9581a2d 100755 --- a/tools/yaml-validate.py +++ b/tools/yaml-validate.py @@ -337,6 +337,8 @@ param_map = {} for base_path in path_args: if os.path.isdir(base_path): for subdir, dirs, files in os.walk(base_path): + if '.tox' in dirs: + dirs.remove('.tox') for f in files: if f.endswith('.yaml') and not f.endswith('.j2.yaml'): file_path = os.path.join(subdir, f) diff --git a/tox.ini b/tox.ini index c87bf7be..da179e91 100644 --- a/tox.ini +++ b/tox.ini @@ -14,10 +14,13 @@ commands = python setup.py testr --slowest --testr-args='{posargs}' commands = {posargs} [testenv:pep8] +whielist_externals = + bash commands = python ./tools/process-templates.py python ./network/endpoints/build_endpoint_map.py --check python ./tools/yaml-validate.py . + bash -c ./tools/roles-data-validation.sh [testenv:templates] commands = python ./tools/process-templates.py @@ -32,3 +35,9 @@ commands = python setup.py test --coverage --coverage-package-name=tripleo_heat_ commands = python ./tools/process-templates.py python ./tripleo_heat_templates/environment_generator.py sample-env-generator/ + +[testenv:genroledata] +whielist_externals = + bash +commands = + bash -c tools/roles-data-generate-samples.sh -- cgit 1.2.3-korg