[PDF] [SPEC] Add 'version: 1.0'

- add 'version: 1.0' to PDF spec;
- add 'version: 1.0' to all existing lab PDFs;
- extend schema with new property;
- add workaround for value-based decision-making in schema version
  selection via `validate-template.py`;
- add support for multiple schema versions;
- add versions for all schema blocks defined so far;
- fix PDF schema pattern for disk size decimals (e.g. '1.8T');

Change-Id: Ie8f768803ec19f1f9a7982fe5ca59df80764fc4a
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

5 files changed, 196 insertions, 136 deletions

diff --git a/config/pdf/pod1.encrypted.yaml b/config/pdf/pod1.encrypted.yaml
index c1e8f78c..b5c78b50 100644
--- a/config/pdf/pod1.encrypted.yaml
+++ b/config/pdf/pod1.encrypted.yaml
@@ -8,6 +8,8 @@
 ---
 ### POD descriptor file ###
 
+version: 1.0
+
 details:
   pod_owner: Lab Owner
   contact: email@address.com
diff --git a/config/pdf/pod1.schema.yaml b/config/pdf/pod1.schema.yaml
index a602b562..2a96d0b2 100644
--- a/config/pdf/pod1.schema.yaml
+++ b/config/pdf/pod1.schema.yaml
@@ -10,173 +10,216 @@ $schema: 'http://json-schema.org/schema#'
 $id: 'https://github.com/opnfv/pharos/blob/master/config/pdf/pod1.yaml'
 
 definitions:
+  ############################################################################
+  # Low-level, general purpose definitions, unversioned
   ip_address:
     type: 'string'  # NOTE: we don't validate this is a valid addr (yet)
   mac_address:
     type: 'string'
     pattern: '^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$'
-  # Common node schema for jumpserver, cluster nodes
+  ############################################################################
+  # Mid-level definitions, common schema for jumpserver & cluster nodes
   node:
-    type: 'object'
-    properties:
-      type:
-        type: 'string'
-        enum: ['baremetal', 'virtual']
-      vendor:
-        type: 'string'
-      model:
-        type: 'string'
-      arch:
-        type: 'string'
-        enum: ['aarch64', 'x86_64']
-      cpus:
-        type: 'number'
-      cpu_cflags:
-        type: 'string'
-      cores:
-        type: 'number'
-      memory:
-        type: 'string'
-        pattern: '^[0-9]+[GT]B?$'
-    required: ['type', 'vendor', 'model', 'arch', 'cpus', 'cpu_cflags', 'cores', 'memory']
-    additionalProperties: false
-  disks:
-    type: 'array'
-    items:
+    v1.0:
       type: 'object'
       properties:
-        name:
+        type:
+          type: 'string'
+          enum: ['baremetal', 'virtual']
+        vendor:
           type: 'string'
-          pattern: '^disk[0-9]+$'
-        disk_capacity:
+        model:
           type: 'string'
-          pattern: '^[0-9]+[MGT]B?$'
-        disk_type:
+        arch:
           type: 'string'
-          enum: ['hdd', 'ssd', 'cdrom', 'tape']
-        disk_interface:
+          enum: ['aarch64', 'x86_64']
+        cpus:
+          type: 'number'
+        cpu_cflags:
           type: 'string'
-          enum: ['sata', 'sas', 'ssd', 'nvme', 'scsi', 'iscsi']
-        disk_rotation:
+        cores:
           type: 'number'
-          enum: [0, 5400, 7200, 10000, 15000]
-      required: ['name', 'disk_capacity', 'disk_type', 'disk_interface', 'disk_rotation']
+        memory:
+          type: 'string'
+          pattern: '^[1-9]\d*[MGT]B?$'
+      required: ['type', 'vendor', 'model', 'arch', 'cpus', 'cpu_cflags', 'cores', 'memory']
       additionalProperties: false
+  disks:
+    v1.0:
+      type: 'array'
+      items:
+        type: 'object'
+        properties:
+          name:
+            type: 'string'
+            pattern: '^disk[0-9]+$'
+          disk_capacity:
+            type: 'string'
+            pattern: '^[1-9][\d\.]*[MGT]B?$'
+          disk_type:
+            type: 'string'
+            enum: ['hdd', 'ssd', 'cdrom', 'tape']
+          disk_interface:
+            type: 'string'
+            enum: ['sata', 'sas', 'ssd', 'nvme', 'scsi', 'iscsi']
+          disk_rotation:
+            type: 'number'
+            enum: [0, 5400, 7200, 10000, 15000]
+        required: ['name', 'disk_capacity', 'disk_type', 'disk_interface', 'disk_rotation']
+        additionalProperties: false
   remote_management:
-    type: 'object'
-    properties:
-      type:
-        type: 'string'
-        enum: ['ipmi', 'amt']
-      versions:
-        type: 'array'
-        items:
-          type: 'number'
-          enum: [1.0, 2.0]
-      user:
-        type: 'string'
-      pass:
-        type: 'string'
-      address:
-        $ref: '#/definitions/ip_address'
-      mac_address:
-        $ref: '#/definitions/mac_address'
-    required: ['type', 'versions', 'user', 'pass', 'address', 'mac_address']
-    additionalProperties: false
-  interfaces:
-    type: 'array'
-    items:
+    v1.0:
       type: 'object'
       properties:
-        name:
+        type:
+          type: 'string'
+          enum: ['ipmi', 'amt']
+        versions:
+          type: 'array'
+          items:
+            type: 'number'
+            enum: [1.0, 2.0]
+        user:
+          type: 'string'
+        pass:
           type: 'string'
-          pattern: '^nic[0-9]+$'
-        mac_address:
-          $ref: '#/definitions/mac_address'
-        # Optional
         address:
           $ref: '#/definitions/ip_address'
-        # Optional
-        vlan:
-          oneOf:
-            - type: 'string'
-              pattern: '^(native|[1-9][0-9]{0,3})(\|(native|[1-9][0-9]{0,3}))*$'
-            - type: 'integer'
-              mininum: 1
-              maximum: 4095
-        # Optional
-        speed:
-          type: 'string'
-          enum: ['1gb', '10gb', '25gb', '40gb']
-          # FIXME: mandatory for nodes?
-        # Optional
-        features:
-          type: ['string', 'null']
-          pattern: '^((dpdk|sriov)\|?)*$'
-          # FIXME: mandatory for nodes?
-      required: ['name', 'mac_address']
+        mac_address:
+          $ref: '#/definitions/mac_address'
+      required: ['type', 'versions', 'user', 'pass', 'address', 'mac_address']
       additionalProperties: false
-
-# Do not allow any properties not defined here. This lets us catch typos.
-additionalProperties: false
-
-properties:
+  interfaces:
+    v1.0:
+      type: 'array'
+      items:
+        type: 'object'
+        properties:
+          name:
+            type: 'string'
+            pattern: '^nic[0-9]+$'
+          mac_address:
+            $ref: '#/definitions/mac_address'
+          # Optional
+          address:
+            $ref: '#/definitions/ip_address'
+          # Optional
+          vlan:
+            oneOf:
+              - type: 'string'
+                pattern: '^(native|[1-9][0-9]{0,3})(\|(native|[1-9][0-9]{0,3}))*$'
+              - type: 'integer'
+                mininum: 1
+                maximum: 4095
+          # Optional
+          speed:
+            type: 'string'
+            enum: ['1gb', '10gb', '25gb', '40gb']
+            # FIXME: mandatory for nodes?
+          # Optional
+          features:
+            type: ['string', 'null']
+            pattern: '^((dpdk|sriov)\|?)*$'
+            # FIXME: mandatory for nodes?
+        required: ['name', 'mac_address']
+        additionalProperties: false
+  ############################################################################
+  # Top-level PDF blocks, versioned
   details:
-    type: 'object'
-    properties:
-      type:
-        type: 'string'
-        enum: ['production', 'development']
-      pod_owner:
-        type: 'string'
-      contact:
-        type: 'string'
-        pattern: '^([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+[,; ]*)+$'
-      lab:
-        type: 'string'
-      location:
-        type: 'string'
-      link:
-        type: 'string'
-    required: ['type', 'pod_owner', 'contact', 'lab', 'location', 'link']
-    additionalProperties: false
+    v1.0:
+      type: 'object'
+      properties:
+        type:
+          type: 'string'
+          enum: ['production', 'development']
+        pod_owner:
+          type: 'string'
+        contact:
+          type: 'string'
+          pattern: '^([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+[,; ]*)+$'
+        lab:
+          type: 'string'
+        location:
+          type: 'string'
+        link:
+          type: 'string'
+      required: ['type', 'pod_owner', 'contact', 'lab', 'location', 'link']
+      additionalProperties: false
   jumphost:
-    type: 'object'
-    properties:
-      name:
-        type: 'string'
-      node:
-        $ref: '#/definitions/node'
-      disks:
-        $ref: '#/definitions/disks'
-      os:
-        type: 'string'
-      remote_params:  # Optional YAML anchor, contents will be validated below
-        type: 'object'
-      remote_management:
-        $ref: '#/definitions/remote_management'
-      interfaces:
-        $ref: '#/definitions/interfaces'
-    required: ['name', 'node', 'disks', 'os', 'remote_management', 'interfaces']
-    additionalProperties: false
-  nodes:
-    type: 'array'
-    items:
+    v1.0:
       type: 'object'
       properties:
         name:
           type: 'string'
         node:
-          $ref: '#/definitions/node'
+          $ref: '#/definitions/node/v1.0'
         disks:
-          $ref: '#/definitions/disks'
+          $ref: '#/definitions/disks/v1.0'
         os:
           type: 'string'
-        remote_params:  # Optional YAML anchor, contents will be validated after inject
+        remote_params:  # Optional YAML anchor, contents will be validated below
           type: 'object'
         remote_management:
-          $ref: '#/definitions/remote_management'
+          $ref: '#/definitions/remote_management/v1.0'
         interfaces:
-          $ref: '#/definitions/interfaces'
-      required: ['name', 'node', 'disks', 'remote_management', 'interfaces']
+          $ref: '#/definitions/interfaces/v1.0'
+      required: ['name', 'node', 'disks', 'os', 'remote_management', 'interfaces']
       additionalProperties: false
+  nodes:
+    v1.0:
+      type: 'array'
+      items:
+        type: 'object'
+        properties:
+          name:
+            type: 'string'
+          node:
+            $ref: '#/definitions/node/v1.0'
+          disks:
+            $ref: '#/definitions/disks/v1.0'
+          os:
+            type: 'string'
+          remote_params:  # Optional YAML anchor, contents will be validated after inject
+            type: 'object'
+          remote_management:
+            $ref: '#/definitions/remote_management/v1.0'
+          interfaces:
+            $ref: '#/definitions/interfaces/v1.0'
+        required: ['name', 'node', 'disks', 'remote_management', 'interfaces']
+        additionalProperties: false
+
+##############################################################################
+# Top-level structure:
+# - define all possible top-level keys, without enforcing a schema for them,
+#   just so we can disallow additional properties;
+# - require 'version' and allow dynamically generated 'version_x.y' key;
+properties:
+  # version_x.y keys are auto-added by `validate_schema.py` based on 'version'
+  version_1.0:
+    type: 'boolean'
+  version:
+    type: 'number'
+    enum: [1.0]
+  details:
+    type: 'object'
+  jumphost:
+    type: 'object'
+  nodes:
+    type: 'array'
+required: ['version']
+additionalProperties: false
+
+##############################################################################
+# Schema versioning
+# - based on dynamically added 'version_x.y', require additional PDF blocks
+#   and validate them against the proper schema version;
+dependencies:
+  version_1.0:
+    properties:
+      details:
+        $ref: '#/definitions/details/v1.0'
+      jumphost:
+        $ref: '#/definitions/jumphost/v1.0'
+      nodes:
+        $ref: '#/definitions/nodes/v1.0'
+    required: ['details', 'jumphost', 'nodes']
diff --git a/config/pdf/pod1.yaml b/config/pdf/pod1.yaml
index c2d07349..d0cd7931 100644
--- a/config/pdf/pod1.yaml
+++ b/config/pdf/pod1.yaml
@@ -8,6 +8,8 @@
 ---
 ### POD descriptor file ###
 
+version: 1.0
+
 details:
   pod_owner: Lab Owner
   contact: email@address.com
diff --git a/config/utils/check-schema.sh b/config/utils/check-schema.sh
index 321c5ced..61bdec22 100755
--- a/config/utils/check-schema.sh
+++ b/config/utils/check-schema.sh
@@ -16,7 +16,7 @@ RC=0
 while IFS= read -r lab_config; do
     pdf_cmd="${VALIDATE_SCHEMA} -s ${PDF_SCHEMA} -y ${lab_config}"
     echo "###################### ${lab_config} ######################"
-    pdf_out=$(${pdf_cmd} |& sed 's|ENC\[PKCS.*\]|opnfv|g')
+    pdf_out=$(${pdf_cmd} |& sed 's|ENC\[PKCS.*\][\\n]*|opnfv|g')
     if [ -z "${pdf_out}" ]; then
         SUMMARY+=";${lab_config#labs/};OK;\n"
         echo "[PDF] [OK] ${pdf_cmd}"
diff --git a/config/utils/validate_schema.py b/config/utils/validate_schema.py
index cb404554..6bdc3bdb 100755
--- a/config/utils/validate_schema.py
+++ b/config/utils/validate_schema.py
@@ -22,6 +22,19 @@ with open(ARGS.yaml) as _:
 with open(ARGS.schema) as _:
     _SCHEMA = yaml.safe_load(_)
 
+# Draft 4 (latest supported by py-jsonschema) does not support value-based
+# decisions properly, see related github issue:
+# https://github.com/json-schema-org/json-schema-spec/issues/64
+# Workaround: build 'version_x.y: true' on the fly based on 'version: x.y'
+def schema_version_workaround(node):
+    """Traverse nested dictionaries and handle 'version' key where found."""
+    if 'version' in node:
+        node['version_{0}'.format(node['version'])] = True
+    for item in node.items():
+        if type(item) is dict:
+            schema_version_workaround(item)
+schema_version_workaround(_DICT)
+
 _VALIDATOR = jsonschema.Draft4Validator(_SCHEMA)
 for error in _VALIDATOR.iter_errors(_DICT):
     raise RuntimeError(str(error))