diff options
Diffstat (limited to 'config')
| -rw-r--r-- | config/installers/fuel/pod_config.yml.j2 | 261 | ||||
| -rw-r--r-- | config/pdf/idf-pod1.schema.yaml | 87 | ||||
| -rw-r--r-- | config/utils/gen_config_lib.py | 224 | ||||
| -rwxr-xr-x | config/utils/generate_config.py | 101 | ||||
| -rwxr-xr-x | config/utils/validate_schema.py | 18 |
5 files changed, 492 insertions, 199 deletions
diff --git a/config/installers/fuel/pod_config.yml.j2 b/config/installers/fuel/pod_config.yml.j2 index dfa9679d..9a6b4be1 100644 --- a/config/installers/fuel/pod_config.yml.j2 +++ b/config/installers/fuel/pod_config.yml.j2 @@ -5,158 +5,171 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -{%- set net_admin = conf.idf.net_config.admin.network %} -{%- set net_mgmt = conf.idf.net_config.mgmt.network %} -{%- set net_private = conf.idf.net_config.private.network %} -{%- set net_public = conf.idf.net_config.public.network %} -{%- set net_public_mask = conf.idf.net_config.public.mask %} -{%- set vlan_mgmt = conf.idf.net_config.mgmt.vlan %} -{%- set vlan_private = conf.idf.net_config.private.vlan %} -{%- if conf.idf.net_config.public.dns is defined %} - {%- set dns_public = conf.idf.net_config.public.dns %} +{%- set net = conf.idf.net_config %} +{%- set net_admin = [net.admin.network, net.admin.mask] | join("/") %} +{%- set net_mgmt = [net.mgmt.network, net.mgmt.mask] | join("/") %} +{%- set net_private = [net.private.network, net.private.mask] | join("/") %} +{%- set net_public = [net.public.network, net.public.mask] | join("/") %} +{%- set networks = [net_admin, net_mgmt, net_private, net_public] %} + +{%- set vlan_mgmt = net.mgmt.vlan %} +{%- set vlan_private = net.private.vlan %} +{%- set pxe_interface = net.admin.interface %} + +{%- if net.public.dns is defined %} + {%- set dns_public = net.public.dns %} +{%- else %} + {%- set dns_public = [ '8.8.8.8', '8.8.4.4' ] %} {%- endif %} -{%- set pxe_interface = conf.idf.net_config.admin.interface %} -{%- if conf.idf.net_config.public.gateway is defined %} - {%- set net_public_gw = conf.idf.net_config.public.gateway %} + +{%- if net.public.gateway is defined %} + {%- set net_public_gw = net.public.gateway %} {%- endif %} {%- if conf.idf.fuel.network.public_pool is defined %} {%- set net_public_pool_start = conf.idf.fuel.network.public_pool.start_ip %} {%- set net_public_pool_end = conf.idf.fuel.network.public_pool.end_ip %} {%- endif %} + {%- if conf.idf.fuel.maas is defined %} {%- set maas_timeout_comissioning = conf.idf.fuel.maas.timeout_comissioning %} {%- set maas_timeout_deploying = conf.idf.fuel.maas.timeout_deploying %} +{%- else %} + {%- set maas_timeout_comissioning = 10 %} + {%- set maas_timeout_deploying = 15 %} {%- endif %} -{%- if dns_public is not defined %} - {%- set dns_public = [ '8.8.8.8', '8.8.4.4' ] %} -{%- endif %} -{%- if net_public_gw is not defined %} - {%- set net_public_gw = net_public | ipaddr_index('1') %} -{%- endif %} +{%- set cmp_nodes = 3 %} + +{%- set net_admin_hosts = [ + 'opnfv_infra_config_pxe_address', + 'opnfv_infra_maas_node01_deploy_address', + 'opnfv_infra_maas_pxe_start_address'] %} + +{%- set net_mgmt_hosts = [ + 'opnfv_infra_config_address', + 'opnfv_infra_maas_node01_address', + 'opnfv_openstack_proxy_control_address', + 'opnfv_openstack_proxy_node01_control_address', + 'opnfv_openstack_proxy_node02_control_address', + 'opnfv_openstack_gateway_node01_address', + 'opnfv_openstack_gateway_node02_address', + 'opnfv_openstack_gateway_node03_address', + 'opnfv_infra_kvm_address', + 'opnfv_infra_kvm_node01_address', + 'opnfv_infra_kvm_node02_address', + 'opnfv_infra_kvm_node03_address', + 'opnfv_openstack_database_address', + 'opnfv_openstack_database_node01_address', + 'opnfv_openstack_database_node02_address', + 'opnfv_openstack_database_node03_address', + 'opnfv_openstack_message_queue_address', + 'opnfv_openstack_message_queue_node01_address', + 'opnfv_openstack_message_queue_node02_address', + 'opnfv_openstack_message_queue_node03_address', + 'opnfv_openstack_telemetry_address', + 'opnfv_openstack_telemetry_node01_address', + 'opnfv_openstack_telemetry_node02_address', + 'opnfv_openstack_telemetry_node03_address', + 'opnfv_openstack_control_address', + 'opnfv_openstack_control_node01_address', + 'opnfv_openstack_control_node02_address', + 'opnfv_openstack_control_node03_address', + 'opnfv_opendaylight_server_node01_single_address', + 'opnfv_stacklight_monitor_address', + 'opnfv_stacklight_monitor_node01_address', + 'opnfv_stacklight_monitor_node02_address', + 'opnfv_stacklight_monitor_node03_address', + 'opnfv_stacklight_log_address', + 'opnfv_stacklight_log_node01_address', + 'opnfv_stacklight_log_node02_address', + 'opnfv_stacklight_log_node03_address', + 'opnfv_stacklight_telemetry_address', + 'opnfv_stacklight_telemetry_node01_address', + 'opnfv_stacklight_telemetry_node02_address', + 'opnfv_stacklight_telemetry_node03_address'] %} + +{%- set net_public_hosts = [ + 'opnfv_openstack_proxy_address', + 'opnfv_openstack_proxy_node01_address', + 'opnfv_openstack_proxy_node02_address', + 'opnfv_openstack_gateway_node01_external_address', + 'opnfv_openstack_gateway_node02_external_address', + 'opnfv_openstack_gateway_node03_external_address', + 'opnfv_openstack_control_node01_external_address', + 'opnfv_openstack_control_node02_external_address', + 'opnfv_openstack_control_node03_external_address'] %} + +{%- set net_private_hosts = [ + 'opnfv_openstack_gateway_node01_tenant_address', + 'opnfv_openstack_gateway_node02_tenant_address', + 'opnfv_openstack_gateway_node03_tenant_address'] %} + +{%- set hosts = { + net_admin: net_admin_hosts, + net_mgmt: net_mgmt_hosts, + net_private: net_private_hosts, + net_public: net_public_hosts } %} + +{%- set start_ip = { + net_admin: 1, + net_mgmt: 1, + net_private: 1, + net_public: 1 } %} + +{%- set total_public_hosts = net_public_hosts | length + cmp_nodes %} {%- if net_public_pool_start is not defined or net_public_pool_end is not defined %} - {%- set net_public_pool_start = net_public | ipaddr_index('80') %} - {%- set net_public_pool_end = net_public | ipaddr_index('100') %} -{%- endif %} -{%- if maas_timeout_comissioning is not defined or maas_timeout_deploying is not defined %} - {%- set maas_timeout_comissioning = 10 %} - {%- set maas_timeout_deploying = 15 %} + {%- set net_public_pool_start = net_public | ipnet_hostaddr(total_public_hosts + start_ip[net_public] +1) %} + {%- set net_public_pool_end = net_public | ipnet_hostmax -1 %} {%- endif %} + --- parameters: _param: + opnfv_maas_timeout_comissioning: {{ maas_timeout_comissioning }} + opnfv_maas_timeout_deploying: {{ maas_timeout_deploying }} + opnfv_jump_bridge_admin: {{ conf.idf.fuel.jumphost.bridges.admin }} opnfv_jump_bridge_mgmt: {{ conf.idf.fuel.jumphost.bridges.mgmt }} opnfv_jump_bridge_private: {{ conf.idf.fuel.jumphost.bridges.private }} opnfv_jump_bridge_public: {{ conf.idf.fuel.jumphost.bridges.public }} - opnfv_infra_config_address: {{ net_mgmt | ipaddr_index(100) }} - opnfv_infra_config_pxe_address: {{ net_admin | ipaddr_index(2) }} - opnfv_infra_maas_node01_address: {{ net_mgmt | ipaddr_index(3) }} - opnfv_infra_maas_node01_deploy_address: {{ net_admin | ipaddr_index(3) }} - opnfv_infra_kvm_address: {{ net_mgmt | ipaddr_index(140) }} - opnfv_infra_kvm_node01_address: {{ net_mgmt | ipaddr_index(141) }} - opnfv_infra_kvm_node02_address: {{ net_mgmt | ipaddr_index(142) }} - opnfv_infra_kvm_node03_address: {{ net_mgmt | ipaddr_index(143) }} - - opnfv_infra_maas_pxe_network_address: {{ net_admin }} - opnfv_infra_maas_pxe_start_address: {{ net_admin | ipaddr_index(4) }} - opnfv_infra_maas_pxe_end_address: {{ net_admin | ipaddr_index(100) }} - - opnfv_openstack_gateway_node01_address: {{ net_mgmt | ipaddr_index(124) }} - opnfv_openstack_gateway_node02_address: {{ net_mgmt | ipaddr_index(125) }} - opnfv_openstack_gateway_node03_address: {{ net_mgmt | ipaddr_index(126) }} - opnfv_openstack_gateway_node01_tenant_address: {{ net_private | ipaddr_index(124) }} - opnfv_openstack_gateway_node02_tenant_address: {{ net_private | ipaddr_index(125) }} - opnfv_openstack_gateway_node03_tenant_address: {{ net_private | ipaddr_index(126) }} - opnfv_openstack_gateway_node01_external_address: {{ net_public | ipaddr_index(124) }} - opnfv_openstack_gateway_node02_external_address: {{ net_public | ipaddr_index(125) }} - opnfv_openstack_gateway_node03_external_address: {{ net_public | ipaddr_index(126) }} - opnfv_openstack_proxy_address: {{ net_public | ipaddr_index(103) }} - opnfv_openstack_proxy_node01_address: {{ net_public | ipaddr_index(104) }} - opnfv_openstack_proxy_node02_address: {{ net_public | ipaddr_index(105) }} - opnfv_openstack_proxy_control_address: {{ net_mgmt | ipaddr_index(103) }} - opnfv_openstack_proxy_node01_control_address: {{ net_mgmt | ipaddr_index(104) }} - opnfv_openstack_proxy_node02_control_address: {{ net_mgmt | ipaddr_index(105) }} - opnfv_openstack_control_address: {{ net_mgmt | ipaddr_index(10) }} - opnfv_openstack_control_node01_address: {{ net_mgmt | ipaddr_index(11) }} - opnfv_openstack_control_node02_address: {{ net_mgmt | ipaddr_index(12) }} - opnfv_openstack_control_node03_address: {{ net_mgmt | ipaddr_index(13) }} - opnfv_openstack_control_node01_external_address: {{ net_public | ipaddr_index(11) }} - opnfv_openstack_control_node02_external_address: {{ net_public | ipaddr_index(12) }} - opnfv_openstack_control_node03_external_address: {{ net_public | ipaddr_index(13) }} - opnfv_openstack_database_address: {{ net_mgmt | ipaddr_index(50) }} - opnfv_openstack_database_node01_address: {{ net_mgmt | ipaddr_index(51) }} - opnfv_openstack_database_node02_address: {{ net_mgmt | ipaddr_index(52) }} - opnfv_openstack_database_node03_address: {{ net_mgmt | ipaddr_index(53) }} - opnfv_openstack_message_queue_address: {{ net_mgmt | ipaddr_index(40) }} - opnfv_openstack_message_queue_node01_address: {{ net_mgmt | ipaddr_index(41) }} - opnfv_openstack_message_queue_node02_address: {{ net_mgmt | ipaddr_index(42) }} - opnfv_openstack_message_queue_node03_address: {{ net_mgmt | ipaddr_index(43) }} - opnfv_openstack_telemetry_address: {{ net_mgmt | ipaddr_index(75) }} - opnfv_openstack_telemetry_node01_address: {{ net_mgmt | ipaddr_index(76) }} - opnfv_openstack_telemetry_node02_address: {{ net_mgmt | ipaddr_index(77) }} - opnfv_openstack_telemetry_node03_address: {{ net_mgmt | ipaddr_index(78) }} - opnfv_openstack_compute_node01_single_address: {{ net_mgmt | ipaddr_index(101) }} - opnfv_openstack_compute_node02_single_address: {{ net_mgmt | ipaddr_index(102) }} - opnfv_openstack_compute_node03_single_address: {{ net_mgmt | ipaddr_index(103) }} - opnfv_openstack_compute_node01_control_address: {{ net_mgmt | ipaddr_index(101) }} - opnfv_openstack_compute_node02_control_address: {{ net_mgmt | ipaddr_index(102) }} - opnfv_openstack_compute_node03_control_address: {{ net_mgmt | ipaddr_index(103) }} - opnfv_openstack_compute_node01_tenant_address: {{ net_private | ipaddr_index(101) }} - opnfv_openstack_compute_node02_tenant_address: {{ net_private | ipaddr_index(102) }} - opnfv_openstack_compute_node03_tenant_address: {{ net_private | ipaddr_index(103) }} - opnfv_openstack_compute_node01_external_address: {{ net_public | ipaddr_index(101) }} - opnfv_openstack_compute_node02_external_address: {{ net_public | ipaddr_index(102) }} - - opnfv_opendaylight_server_node01_single_address: {{ net_mgmt | ipaddr_index(111) }} - - opnfv_net_public: {{ net_public }}/{{ net_public_mask }} - opnfv_net_public_mask: {{ net_public_mask | netmask }} + opnfv_infra_maas_pxe_network_address: {{ net.admin.network }} + opnfv_infra_maas_pxe_end_address: {{ net_admin | ipnet_hostmax }} + opnfv_net_public: {{ net_public }} + opnfv_net_public_mask: {{ net_public | ipnet_netmask }} opnfv_net_public_gw: {{ net_public_gw }} opnfv_net_public_pool_start: {{ net_public_pool_start }} opnfv_net_public_pool_end: {{ net_public_pool_end }} opnfv_name_servers: {{ dns_public }} opnfv_dns_server01: '{{ dns_public[0] }}' - opnfv_net_mgmt_vlan: {{ vlan_mgmt }} opnfv_net_tenant_vlan: {{ vlan_private }} - opnfv_maas_timeout_comissioning: {{ maas_timeout_comissioning }} - opnfv_maas_timeout_deploying: {{ maas_timeout_deploying }} +{%- for network in networks %} +{%- for key in hosts[network] %} +{%- set i = loop.index + start_ip[network] %} + {{key}}: {{ network | ipnet_hostaddr(i) }} +{%- endfor %} +{%- endfor %} + +{%- for cmp in range(1, cmp_nodes +1) %} + {%- set n = '%02d' | format(cmp) %} + {%- set mgmt = net_mgmt_hosts | length + start_ip[net_mgmt] + loop.index %} + {%- set pub = net_public_hosts | length + start_ip[net_public] + loop.index %} + {%- set pri = net_private_hosts | length + start_ip[net_private] + loop.index %} + opnfv_openstack_compute_node{{n}}_single_address: {{ net_mgmt | ipnet_hostaddr(mgmt) }} + opnfv_openstack_compute_node{{n}}_control_address: {{ net_mgmt | ipnet_hostaddr(mgmt) }} + opnfv_openstack_compute_node{{n}}_tenant_address: {{ net_private | ipnet_hostaddr(pri) }} + opnfv_openstack_compute_node{{n}}_external_address: {{ net_public | ipnet_hostaddr(pub) }} +{%- endfor %} - opnfv_maas_node01_architecture: '{{ conf.nodes.0.node.arch | dpkg_arch }}/generic' - opnfv_maas_node01_power_address: {{ conf.nodes.0.remote_management.address.rsplit('/')[0] }} - opnfv_maas_node01_power_type: {{ conf.nodes.0.remote_management.type }} - opnfv_maas_node01_power_user: {{ conf.nodes.0.remote_management.user }} - opnfv_maas_node01_power_password: {{ conf.nodes.0.remote_management.pass }} - opnfv_maas_node01_interface_mac: '{{ conf.nodes.0.interfaces[pxe_interface].mac_address }}' - - opnfv_maas_node02_architecture: '{{ conf.nodes.1.node.arch | dpkg_arch }}/generic' - opnfv_maas_node02_power_address: {{ conf.nodes.1.remote_management.address.rsplit('/')[0] }} - opnfv_maas_node02_power_type: {{ conf.nodes.1.remote_management.type }} - opnfv_maas_node02_power_user: {{ conf.nodes.1.remote_management.user }} - opnfv_maas_node02_power_password: {{ conf.nodes.1.remote_management.pass }} - opnfv_maas_node02_interface_mac: '{{ conf.nodes.1.interfaces[pxe_interface].mac_address }}' - - opnfv_maas_node03_architecture: '{{ conf.nodes.2.node.arch | dpkg_arch }}/generic' - opnfv_maas_node03_power_address: {{ conf.nodes.2.remote_management.address.rsplit('/')[0] }} - opnfv_maas_node03_power_type: {{ conf.nodes.2.remote_management.type }} - opnfv_maas_node03_power_user: {{ conf.nodes.2.remote_management.user }} - opnfv_maas_node03_power_password: {{ conf.nodes.2.remote_management.pass }} - opnfv_maas_node03_interface_mac: '{{ conf.nodes.2.interfaces[pxe_interface].mac_address }}' - - opnfv_maas_node04_architecture: '{{ conf.nodes.3.node.arch | dpkg_arch }}/generic' - opnfv_maas_node04_power_address: {{ conf.nodes.3.remote_management.address.rsplit('/')[0] }} - opnfv_maas_node04_power_type: {{ conf.nodes.3.remote_management.type }} - opnfv_maas_node04_power_user: {{ conf.nodes.3.remote_management.user }} - opnfv_maas_node04_power_password: {{ conf.nodes.3.remote_management.pass }} - opnfv_maas_node04_interface_mac: '{{ conf.nodes.3.interfaces[pxe_interface].mac_address }}' - - opnfv_maas_node05_architecture: '{{ conf.nodes.4.node.arch | dpkg_arch }}/generic' - opnfv_maas_node05_power_address: {{ conf.nodes.4.remote_management.address.rsplit('/')[0] }} - opnfv_maas_node05_power_type: {{ conf.nodes.4.remote_management.type }} - opnfv_maas_node05_power_user: {{ conf.nodes.4.remote_management.user }} - opnfv_maas_node05_power_password: {{ conf.nodes.4.remote_management.pass }} - opnfv_maas_node05_interface_mac: '{{ conf.nodes.4.interfaces[pxe_interface].mac_address }}' +{%- for node in conf.nodes %} + {%- set n = '%02d' | format(loop.index) %} + opnfv_maas_node{{n}}_architecture: '{{ node.node.arch | dpkg_arch }}/generic' + opnfv_maas_node{{n}}_power_address: {{ node.remote_management.address.rsplit('/')[0] }} + opnfv_maas_node{{n}}_power_type: {{ node.remote_management.type }} + opnfv_maas_node{{n}}_power_user: {{ node.remote_management.user }} + opnfv_maas_node{{n}}_power_password: {{ node.remote_management.pass }} + opnfv_maas_node{{n}}_interface_mac: '{{ node.interfaces[pxe_interface].mac_address }}' +{%- endfor %} diff --git a/config/pdf/idf-pod1.schema.yaml b/config/pdf/idf-pod1.schema.yaml index 0705fcc5..857749c4 100644 --- a/config/pdf/idf-pod1.schema.yaml +++ b/config/pdf/idf-pod1.schema.yaml @@ -14,14 +14,14 @@ definitions: v0.1: # NOTE: I hope this is going away soon, so I won't model it yet type: 'object' - daisy: - v0.1: - type: 'object' - # NOTE: To be properly modeled by Daisy maintainers compass: v0.1: type: 'object' # NOTE: To be properly modeled by Compass4NFV maintainers + daisy: + v0.1: + type: 'object' + # NOTE: To be properly modeled by Daisy maintainers fuel: v0.1: type: 'object' @@ -73,8 +73,67 @@ definitions: additionalProperties: false required: ['node'] additionalProperties: false + reclass: # Optional + type: 'object' + properties: + node: + type: 'array' + items: + type: 'object' + properties: + compute_params: + type: 'object' + properties: + common: # Optional + type: 'object' + properties: &compute_params_common_properties + nova_cpu_pinning: # Optional + type: 'string' + compute_hugepages_size: + type: 'string' + enum: ['2M', '1G'] + compute_hugepages_count: + type: 'number' + compute_hugepages_mount: + type: 'string' + compute_kernel_isolcpu: # Optional + type: 'string' + compute_ovs_pmd_cpu_mask: # Optional + type: ['string', 'number'] + compute_ovs_memory_channels: # Optional + type: ['string', 'number'] + required: ['compute_hugepages_size', 'compute_hugepages_count', + 'compute_hugepages_mount'] + additionalProperties: false + dpdk: # Optional + type: 'object' + properties: + <<: *compute_params_common_properties + compute_dpdk_driver: + type: 'string' + compute_ovs_dpdk_socket_mem: + type: ['string', 'number'] + compute_ovs_dpdk_lcore_mask: + type: ['string', 'number'] + dpdk0_driver: + type: 'string' + dpdk0_n_rxq: + type: 'number' + required: ['compute_dpdk_driver', 'dpdk0_driver', 'dpdk0_n_rxq', + 'compute_ovs_dpdk_socket_mem', + 'compute_ovs_dpdk_lcore_mask'] + additionalProperties: false + additionalProperties: false + required: ['compute_params'] + additionalProperties: false + required: ['node'] + additionalProperties: false required: ['jumphost', 'network'] additionalProperties: false + osa: + v0.1: + type: 'object' + # NOTE: To be properly modeled by XCI maintainers ############################################################################## # Top-level structure: @@ -94,14 +153,16 @@ properties: type: 'array' items: type: 'string' - enum: ['apex', 'compass4nfv', 'daisy', 'fuel', 'joid'] + enum: ['apex', 'compass4nfv', 'daisy', 'fuel', 'joid', 'osa'] net_config: type: 'object' - fuel: + compass: type: 'object' daisy: type: 'object' - compass: + fuel: + type: 'object' + osa: type: 'object' required: ['version'] additionalProperties: false @@ -114,16 +175,20 @@ properties: properties: net_config: $ref: '#/definitions/net_config/v0.1' - fuel: - $ref: '#/definitions/fuel/v0.1' - daisy: - $ref: '#/definitions/daisy/v0.1' compass: $ref: '#/definitions/compass/v0.1' + daisy: + $ref: '#/definitions/daisy/v0.1' + fuel: + $ref: '#/definitions/fuel/v0.1' + osa: + $ref: '#/definitions/osa/v0.1' fuel: required: ['net_config'] daisy: required: ['net_config'] + osa: + required: ['net_config'] # Do not allow any properties not defined here. This lets us catch typos. additionalProperties: false diff --git a/config/utils/gen_config_lib.py b/config/utils/gen_config_lib.py new file mode 100644 index 00000000..1e7229be --- /dev/null +++ b/config/utils/gen_config_lib.py @@ -0,0 +1,224 @@ +############################################################################## +# Copyright (c) 2018 OPNFV and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +"""Library for generate_config functions and custom jinja2 filters""" + +import logging +from ipaddress import IPv4Network, IPv4Address + + +def load_custom_filters(environment): + """Load all defined filters into the jinja2 enviroment""" + + # TODO deprecate ipaddr_index and netmask for the better ipnet ones + filter_list = { + 'dpkg_arch': filter_dpkg_arch, + 'storage_size_num': filter_storage_size_num, + 'ipnet_hostaddr': filter_ipnet_hostaddr, + 'ipnet_hostmin': filter_ipnet_hostmin, + 'ipnet_hostmax': filter_ipnet_hostmax, + 'ipnet_broadcast': filter_ipnet_broadcast, + 'ipnet_netmask': filter_ipnet_netmask, + 'ipnet_contains_ip': filter_ipnet_contains_ip, + 'ipnet_contains_iprange': filter_ipnet_contains_iprange, + 'ipnet_range_size': filter_ipnet_range_size, + 'ipaddr_index': filter_ipaddr_index, + 'netmask': filter_netmask + } + + for name, function in filter_list.items(): + environment.filters[name] = function + + +def filter_dpkg_arch(arch, to_dpkg=True): + """Convert DPKG-compatible from processor arch and vice-versa""" + + # Processor architecture (as reported by $(uname -m)) + # vs DPKG architecture mapping + dpkg_arch_table = { + 'aarch64': 'arm64', + 'x86_64': 'amd64', + } + arch_dpkg_table = dict( + zip(dpkg_arch_table.values(), dpkg_arch_table.keys())) + + if to_dpkg: + return dpkg_arch_table[arch] + else: + return arch_dpkg_table[arch] + + +def filter_storage_size_num(size_str): + """Convert human-readable size string to a string convertible to float""" + + # pattern: '^[1-9][\d\.]*[MGT]B?$', multiplier=1000 (not KiB) + if size_str.endswith('B'): + size_str = size_str[:-1] + try: + size_num = 1000000 + for multiplier in ['M', 'G', 'T']: + if size_str.endswith(multiplier): + return '{:.2f}'.format(size_num * float(size_str[:-1])) + size_num = size_num * 1000 + return '{:.2f}'.format(float(size_str)) + except ValueError as ex: + logging.error(size_str + " is not a valid size string") + raise + + +def filter_ipnet_hostaddr(network_cidr, index): + """Return the host IP address on given index from an IP network""" + try: + network_cidr_str = unicode(network_cidr) + except NameError as ex: + network_cidr_str = str(network_cidr) + try: + return IPv4Network(network_cidr_str)[index] + except ValueError as ex: + logging.error(network_cidr_str + " is not a valid network address") + raise + except IndexError as ex: + logging.error(network_cidr_str + " has not enough range for " + + str(index) + " host IPs.") + raise + + +def filter_ipnet_broadcast(network_cidr): + """Return broadcast IP address from given IP network""" + try: + network_cidr_str = unicode(network_cidr) + except NameError as ex: + network_cidr_str = str(network_cidr) + try: + return IPv4Network(network_cidr_str).broadcast_address + except ValueError as ex: + logging.error(network_cidr_str + " is not a valid network address") + raise + + +def filter_ipnet_hostmin(network_cidr): + """Return the first host IP address from given IP network""" + try: + network_cidr_str = unicode(network_cidr) + except NameError as ex: + network_cidr_str = str(network_cidr) + try: + return IPv4Network(network_cidr_str)[1] + except ValueError as ex: + logging.error(network_cidr_str + " is not a valid network address") + raise + + +def filter_ipnet_hostmax(network_cidr): + """Return the last host IP address from given IP network""" + try: + network_cidr_str = unicode(network_cidr) + except NameError as ex: + network_cidr_str = str(network_cidr) + try: + return IPv4Network(network_cidr_str)[-2] + except ValueError as ex: + logging.error(network_cidr_str + " is not a valid network address") + raise + + +def filter_ipnet_netmask(network_cidr): + """Return the IP netmask from given IP network""" + try: + network_cidr_str = unicode(network_cidr) + except NameError as ex: + network_cidr_str = str(network_cidr) + try: + return IPv4Network(network_cidr_str).netmask + except ValueError as ex: + logging.error(network_cidr_str + " is not a valid network address") + raise + + +def filter_ipnet_contains_ip(network_cidr, ip_address): + """Check if an IP network cointains a given range""" + try: + network_cidr_str = unicode(network_cidr) + ip_address_str = unicode(ip_address) + except NameError as ex: + network_cidr_str = str(network_cidr) + ip_address_str = str(ip_address) + try: + return IPv4Address(ip_address_str) in IPv4Network(network_cidr_str) + except ValueError as ex: + logging.error(network_cidr_str + " is not a valid network address") + raise + + +def filter_ipnet_contains_iprange(network_cidr, range_start, range_end): + """Check if an IP network cointains a given range""" + try: + network_cidr_str = unicode(network_cidr) + range_start_str = unicode(range_start) + range_end_str = unicode(range_end) + except NameError as ex: + network_cidr_str = str(network_cidr) + range_start_str = str(range_start) + range_end_str = str(range_end) + try: + ipnet = IPv4Network(network_cidr_str) + return (IPv4Address(range_start_str) in ipnet + and IPv4Address(range_end_str) in ipnet) + except ValueError as ex: + logging.error(network_cidr_str + " is not a valid network address") + raise + + +def filter_ipnet_range_size(network_cidr, range_start, range_end): + """Get the size of an IP range between two IP addresses""" + try: + network_cidr_str = unicode(network_cidr) + range_start_str = unicode(range_start) + range_end_str = unicode(range_end) + except NameError as ex: + network_cidr_str = str(network_cidr) + range_start_str = str(range_start) + range_end_str = str(range_end) + try: + ipnet = IPv4Network(network_cidr_str) + ip1 = IPv4Address(range_start_str) + ip2 = IPv4Address(range_end_str) + + if ip1 in ipnet and ip2 in ipnet: + index1 = list(ipnet.hosts()).index(ip1) + index2 = list(ipnet.hosts()).index(ip2) + ip_range_size = index2 - index1 + 1 + return ip_range_size + else: + raise ValueError + except ValueError as ex: + logging.error(range_start_str + " and " + range_end_str + + " are not valid IP addresses for range inside " + + network_cidr_str) + raise + + +# This filter is too simple and does not take network mask into account. +# TODO Deprecate for filter_ipnet_hostaddr +def filter_ipaddr_index(base_address, index): + """Return IP address in given network at given index""" + try: + base_address_str = unicode(base_address) + except NameError as ex: + base_address_str = str(base_address) + return IPv4Address(base_address_str) + int(index) + + +# TODO deprecate for filter_ipnet_netmask +def filter_netmask(prefix): + """Get netmask from prefix length integer""" + try: + prefix_str = unicode(prefix) + except NameError as ex: + prefix_str = str(prefix) + return IPv4Network("1.0.0.0/"+prefix_str).netmask diff --git a/config/utils/generate_config.py b/config/utils/generate_config.py index b2b52f0b..f45f7888 100755 --- a/config/utils/generate_config.py +++ b/config/utils/generate_config.py @@ -7,89 +7,76 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -"""This module does blah blah.""" +"""Generate configuration from PDF/IDF and jinja2 installer template""" + import argparse -import ipaddress import logging -import os +from os.path import abspath, exists, isfile, split +from subprocess import CalledProcessError, check_output +import gen_config_lib import yaml from jinja2 import Environment, FileSystemLoader -from subprocess import CalledProcessError, check_output + PARSER = argparse.ArgumentParser() PARSER.add_argument("--yaml", "-y", type=str, required=True) -PARSER.add_argument("--jinja2", "-j", type=str, required=True) +PARSER.add_argument("--jinja2", "-j", type=str, required=True, action='append') +PARSER.add_argument("--includesdir", "-i", action='append', default=['/']) +PARSER.add_argument("--batch", "-b", action='store_true') +PARSER.add_argument("--verbose", "-v", action='count') ARGS = PARSER.parse_args() -# Processor architecture vs DPKG architecture mapping -DPKG_ARCH_TABLE = { - 'aarch64': 'arm64', - 'x86_64': 'amd64', -} -ARCH_DPKG_TABLE = dict(zip(DPKG_ARCH_TABLE.values(), DPKG_ARCH_TABLE.keys())) +LOADER = yaml.CSafeLoader if yaml.__with_libyaml__ else yaml.SafeLoader +ARGS.jinja2 = [abspath(x) for x in ARGS.jinja2] -# Custom filter to allow simple IP address operations returning -# a new address from an upper or lower (negative) index -def ipaddr_index(base_address, index): - """Return IP address in given network at given index""" - try: - base_address_str = unicode(base_address) - #pylint: disable=unused-variable - except NameError as ex: - base_address_str = str(base_address) - return ipaddress.ip_address(base_address_str) + int(index) +logging.basicConfig() +LOGGER = logging.getLogger('generate_config') +if ARGS.verbose: + LOGGER.setLevel(logging.INFO) -# Custom filter to transform a prefix netmask to IP address format netmask -def netmask(prefix): - """Get netmask from prefix length integer""" - try: - prefix_str = unicode(prefix) - except NameError as ex: - prefix_str = str(prefix) - return ipaddress.IPv4Network("1.0.0.0/"+prefix_str).netmask - -# Custom filter to convert between processor architecture -# (as reported by $(uname -m)) and DPKG-style architecture -def dpkg_arch(arch, to_dpkg=True): - """Return DPKG-compatible from processor arch and vice-versa""" - if to_dpkg: - return DPKG_ARCH_TABLE[arch] - else: - return ARCH_DPKG_TABLE[arch] - -ENV = Environment(loader=FileSystemLoader(os.path.dirname(ARGS.jinja2))) -ENV.filters['ipaddr_index'] = ipaddr_index -ENV.filters['netmask'] = netmask -ENV.filters['dpkg_arch'] = dpkg_arch +ENV = Environment( + loader=FileSystemLoader(ARGS.includesdir), + extensions=['jinja2.ext.do'] +) +gen_config_lib.load_custom_filters(ENV) # Run `eyaml decrypt` on the whole file, but only if PDF data is encrypted # Note: eyaml return code is 0 even if keys are not available try: - if os.path.isfile(ARGS.yaml) and 'ENC[PKCS7' in open(ARGS.yaml).read(): - DICT = yaml.safe_load(check_output(['eyaml', 'decrypt', - '-f', ARGS.yaml])) + if isfile(ARGS.yaml) and 'ENC[PKCS7' in open(ARGS.yaml).read(): + DICT = yaml.load(check_output(['eyaml', 'decrypt', + '-f', ARGS.yaml]), Loader=LOADER) except CalledProcessError as ex: - logging.error('eyaml decryption failed! Fallback to raw data.') + LOGGER.error('eyaml decryption failed! Fallback to raw data.') except OSError as ex: - logging.warn('eyaml not found, skipping decryption. Fallback to raw data.') + LOGGER.warn('eyaml not found, skipping decryption. Fallback to raw data.') try: DICT['details'] except (NameError, TypeError) as ex: with open(ARGS.yaml) as _: - DICT = yaml.safe_load(_) + DICT = yaml.load(_, Loader=LOADER) # If an installer descriptor file (IDF) exists, include it (temporary) -IDF_PATH = '/idf-'.join(os.path.split(ARGS.yaml)) -if os.path.exists(IDF_PATH): +IDF_PATH = '/idf-'.join(split(ARGS.yaml)) +if exists(IDF_PATH): with open(IDF_PATH) as _: - IDF = yaml.safe_load(_) + IDF = yaml.load(_, Loader=LOADER) DICT['idf'] = IDF['idf'] # Print dictionary generated from yaml (uncomment for debug) # print(DICT) -# Render template and print generated conf to console -TEMPLATE = ENV.get_template(os.path.basename(ARGS.jinja2)) - -#pylint: disable=superfluous-parens -print(TEMPLATE.render(conf=DICT)) +for _j2 in ARGS.jinja2: + TEMPLATE = ENV.get_template(_j2) + OUTPUT = TEMPLATE.render(conf=DICT) + # Render template and write generated conf to file or stdout + if ARGS.batch: + if _j2.endswith('.j2'): + LOGGER.info('Parsing {}'.format(_j2)) + with open(_j2[:-3], 'w') as _: + _.write(OUTPUT) + else: + LOGGER.warn('Skipping {}, name does not end in ".j2"'.format(_j2)) + else: + # pylint: disable=superfluous-parens + print(OUTPUT) diff --git a/config/utils/validate_schema.py b/config/utils/validate_schema.py index 42f475d8..1676e15d 100755 --- a/config/utils/validate_schema.py +++ b/config/utils/validate_schema.py @@ -11,21 +11,20 @@ import argparse import jsonschema import yaml + PARSER = argparse.ArgumentParser() PARSER.add_argument("--yaml", "-y", type=str, required=True) PARSER.add_argument("--schema", "-s", type=str, required=True) ARGS = PARSER.parse_args() +LOADER = yaml.CSafeLoader if yaml.__with_libyaml__ else yaml.SafeLoader with open(ARGS.yaml) as _: - _DICT = yaml.safe_load(_) + _DICT = yaml.load(_, Loader=LOADER) with open(ARGS.schema) as _: - _SCHEMA = yaml.safe_load(_) + _SCHEMA = yaml.load(_, Loader=LOADER) + -# Draft 4 (latest supported by py-jsonschema) does not support value-based -# decisions properly, see related github issue: -# https://github.com/json-schema-org/json-schema-spec/issues/64 -# Workaround: build 'version_x.y: true' on the fly based on 'version: x.y' def schema_version_workaround(node): """Traverse nested dictionaries and handle 'version' key where found.""" if 'version' in node: @@ -33,9 +32,14 @@ def schema_version_workaround(node): for item in node.items(): if type(item) is dict: schema_version_workaround(item) + +# Draft 4 (latest supported by py-jsonschema) does not support value-based +# decisions properly, see related github issue: +# https://github.com/json-schema-org/json-schema-spec/issues/64 +# Workaround: build 'version_x.y: true' on the fly based on 'version: x.y' schema_version_workaround(_DICT) if 'idf' in _DICT: - schema_version_workaround(_DICT['idf']) + schema_version_workaround(_DICT['idf']) _VALIDATOR = jsonschema.Draft4Validator(_SCHEMA) for error in _VALIDATOR.iter_errors(_DICT): |