From 6d68ce08e193043e144533b1e38b66457af1e4db Mon Sep 17 00:00:00 2001 From: Dan Sneddon Date: Mon, 19 Jun 2017 11:34:05 -0700 Subject: Render isolated network templates using jinja2 This change adds templates that are used to create network and port definition templates for each network that is defined in network_data.yaml. In order to render the templates, additional fields have been added to the network_data.yaml file. If this optional data is present, it will be used to populate the default parameter values in the network template. The only required parameters in the network_data.yaml file is the network name. If the network will have IPv6 addresses, then ipv6: true must be set on the network. The existing networks have been modeled in the network_data.yaml, but until these templates are removed from the j2_excludes.yaml file they will not be generated on the fly. Any additional networks will have templates generated. This change also removes an unnecessary conditional from the networks.j2.yaml file, since InternalApiNetwork doesn't need to be reformatted as InternalNetwork (it's only used in this one file). A follow-up patch will remove the existing network definitions so all networks are created dynamically. Change-Id: If074f87494a46305c990a0ea332c7b576d3c6ed8 Depends-On: Iab8aca2f1fcaba0c8f109717a4b3068f629c9aab Partially-Implements: blueprint composable-networks --- j2_excludes.yaml | 36 +++++++++++ network/network.network.j2.yaml | 92 ++++++++++++++++++++++++++++ network/networks.j2.yaml | 11 ---- network/ports/port.network.j2.yaml | 72 ++++++++++++++++++++++ network/ports/port_from_pool.network.j2.yaml | 65 ++++++++++++++++++++ network_data.yaml | 37 +++++++++-- tools/process-templates.py | 41 ++++++++++++- 7 files changed, 337 insertions(+), 17 deletions(-) create mode 100644 network/network.network.j2.yaml create mode 100644 network/ports/port.network.j2.yaml create mode 100644 network/ports/port_from_pool.network.j2.yaml diff --git a/j2_excludes.yaml b/j2_excludes.yaml index 063e63d4..356068fc 100644 --- a/j2_excludes.yaml +++ b/j2_excludes.yaml @@ -8,3 +8,39 @@ name: - puppet/blockstorage-role.yaml - puppet/objectstorage-role.yaml - puppet/cephstorage-role.yaml + - network/internal_api.yaml + - network/external.yaml + - network/storage.yaml + - network/storage_mgmt.yaml + - network/tenant.yaml + - network/management.yaml + - network/internal_api_v6.yaml + - network/external_v6.yaml + - network/storage_v6.yaml + - network/storage_mgmt_v6.yaml + - network/tenant_v6.yaml + - network/management_v6.yaml + - network/ports/internal_api.yaml + - network/ports/external.yaml + - network/ports/storage.yaml + - network/ports/storage_mgmt.yaml + - network/ports/tenant.yaml + - network/ports/management.yaml + - network/ports/internal_api_v6.yaml + - network/ports/external_v6.yaml + - network/ports/storage_v6.yaml + - network/ports/storage_mgmt_v6.yaml + - network/ports/tenant_v6.yaml + - network/ports/management_v6.yaml + - network/ports/internal_api_from_pool.yaml + - network/ports/external_from_pool.yaml + - network/ports/storage_from_pool.yaml + - network/ports/storage_mgmt_from_pool.yaml + - network/ports/tenant_from_pool.yaml + - network/ports/management_from_pool.yaml + - network/ports/internal_api_from_pool_v6.yaml + - network/ports/external_from_pool_v6.yaml + - network/ports/storage_from_pool_v6.yaml + - network/ports/storage_mgmt_from_pool_v6.yaml + - network/ports/tenant_from_pool_v6.yaml + - network/ports/management_from_pool_v6.yaml diff --git a/network/network.network.j2.yaml b/network/network.network.j2.yaml new file mode 100644 index 00000000..2c223c16 --- /dev/null +++ b/network/network.network.j2.yaml @@ -0,0 +1,92 @@ +heat_template_version: pike + +description: > + {{network.name}} network definition (automatically generated). + +parameters: + # the defaults here work for static IP assignment (IPAM) only + {{network.name}}NetCidr: + default: {{network.ip_subnet|default("")}} + description: Cidr for the {{network.name_lower}} network. + type: string + {{network.name}}NetValueSpecs: + default: {'provider:physical_network': '{{network.name_lower}}', 'provider:network_type': 'flat'} + description: Value specs for the {{network.name_lower}} network. + type: json + {{network.name}}NetAdminStateUp: + default: false + description: This admin state of the network. + type: boolean + {{network.name}}NetEnableDHCP: + default: false + description: Whether to enable DHCP on the associated subnet. + type: boolean + {{network.name}}NetShared: + default: false + description: Whether this network is shared across all tenants. + type: boolean + {{network.name}}NetName: + default: {{network.name_lower}} + description: The name of the {{network.name_lower}} network. + type: string + {{network.name}}SubnetName: + default: {{network.name_lower}}_subnet + description: The name of the {{network.name_lower}} subnet in Neutron. + type: string + {{network.name}}AllocationPools: + default: {{network.allocation_pools|default([])}} + description: Ip allocation pool range for the {{network.name_lower}} network. + type: json + {{network.name}}InterfaceDefaultRoute: + default: {{network.gateway_ip|default("not_defined")}} + description: default route for the {{network.name_lower}} network + type: string +{%- if network.vlan %} + {{network.name}}NetworkVlanID: + default: {{network.vlan}} + description: Vlan ID for the {{network.name}} network traffic. + type: number +{%- endif %} +{%- if network.ipv6 %} + IPv6AddressMode: + default: dhcpv6-stateful + description: Neutron subnet IPv6 address mode + type: string + IPv6RAMode: + default: dhcpv6-stateful + description: Neutron subnet IPv6 router advertisement mode + type: string +{%- endif %} + +resources: + {{network.name}}Network: + type: OS::Neutron::Net + properties: + admin_state_up: {get_param: {{network.name}}NetAdminStateUp} + name: {get_param: {{network.name}}NetName} + shared: {get_param: {{network.name}}NetShared} + value_specs: {get_param: {{network.name}}NetValueSpecs} + + {{network.name}}Subnet: + type: OS::Neutron::Subnet + properties: + cidr: {get_param: {{network.name}}NetCidr} + name: {get_param: {{network.name}}SubnetName} + network: {get_resource: {{network.name}}Network} + allocation_pools: {get_param: {{network.name}}AllocationPools} + gateway_ip: {get_param: {{network.name}}InterfaceDefaultRoute} +{%- if network.ipv6 %} + ip_version: 6 + ipv6_address_mode: {get_param: IPv6AddressMode} + ipv6_ra_mode: {get_param: IPv6RAMode} +{%- else %} + enable_dhcp: {get_param: {{network.name}}NetEnableDHCP} +{%- endif %} + +outputs: + OS::stack_id: + description: {{network.name_lower}} network + value: {get_resource: {{network.name}}Network} + subnet_cidr: + value: {get_attr: {{network.name}}Subnet, cidr} + diff --git a/network/networks.j2.yaml b/network/networks.j2.yaml index 5aec597a..c790d370 100644 --- a/network/networks.j2.yaml +++ b/network/networks.j2.yaml @@ -5,11 +5,7 @@ description: Create networks to split out Overcloud traffic resources: {%- for network in networks %} - {%- if network.name != 'InternalApi' %} {{network.name}}Network: - {%- else %} - InternalNetwork: - {%- endif %} type: OS::TripleO::Network::{{network.name}} {%- endfor %} @@ -23,15 +19,8 @@ outputs: # NOTE(gfidente): we need to replace the null value with a # string to work around https://bugs.launchpad.net/heat/+bug/1700025 {%- for network in networks %} - {%- if network.name != 'InternalApi' %} {{network.name_lower}}: yaql: data: {get_attr: [{{network.name}}Network, subnet_cidr]} expression: str($.data).replace('null', 'disabled') - {%- else %} - {{network.name_lower}}: - yaql: - data: {get_attr: [InternalNetwork, subnet_cidr]} - expression: str($.data).replace('null', 'disabled') - {%- endif %} {%- endfor %} diff --git a/network/ports/port.network.j2.yaml b/network/ports/port.network.j2.yaml new file mode 100644 index 00000000..ded3e798 --- /dev/null +++ b/network/ports/port.network.j2.yaml @@ -0,0 +1,72 @@ +heat_template_version: pike + +description: > + Creates a port on the {{network.name}} network. The IP address will be chosen + automatically if FixedIPs is empty. + +parameters: + {{network.name}}NetName: + description: Name of the {{network.name_lower}} neutron network + default: {{network.name_lower|default(network.name|lower)}} + type: string + PortName: + description: Name of the port + default: '' + type: string + ControlPlaneIP: # Here for compatibility with noop.yaml + description: IP address on the control plane + default: '' + type: string + ControlPlaneNetwork: # Here for compatibility with ctlplane_vip.yaml + description: The name of the undercloud Neutron control plane + default: ctlplane + type: string + FixedIPs: + description: > + Control the IP allocation for the VIP port. E.g. + [{'ip_address':'1.2.3.4'}] + default: [] + type: json + IPPool: # Here for compatibility with from_pool.yaml + default: {} + type: json + NodeIndex: # Here for compatibility with from_pool.yaml + default: 0 + type: number + +resources: + + {{network.name}}Port: + type: OS::Neutron::Port + properties: + network: {get_param: {{network.name}}NetName} + name: {get_param: PortName} + fixed_ips: {get_param: FixedIPs} + replacement_policy: AUTO + +outputs: + ip_address: + description: {{network.name}} network IP + value: {get_attr: [{{network.name}}Port, fixed_ips, 0, ip_address]} + ip_address_uri: +{%- if network.ipv6 %} + description: {{network.name}} network IP (with brackets for IPv6 URLs) + value: + list_join: + - '' + - - '[' + - {get_attr: [{{network.name}}Port, fixed_ips, 0, ip_address]} + - ']' +{%- else %} + description: {{network.name}} network IP (for compatibility with IPv6 URLs) + value: {get_attr: [{{network.name}}Port, fixed_ips, 0, ip_address]} +{%- endif %} + ip_subnet: + description: IP/Subnet CIDR for the {{network.name}} network IP + value: + list_join: + - '' + - - {get_attr: [{{network.name}}Port, fixed_ips, 0, ip_address]} + - '/' + - {str_split: ['/', {get_attr: [{{network.name}}Port, subnets, 0, cidr]}, 1]} + diff --git a/network/ports/port_from_pool.network.j2.yaml b/network/ports/port_from_pool.network.j2.yaml new file mode 100644 index 00000000..9c08ec76 --- /dev/null +++ b/network/ports/port_from_pool.network.j2.yaml @@ -0,0 +1,65 @@ +heat_template_version: pike + +description: > + Creates a port on the {{network.name}} network, using a map of IPs per role. + Each role has a map of IPs in IPs parameters, with a list of IPs by + network (lower_name or lower case). For example: + ControllerIPs: + external: + - 1.2.3.4 # First controller + - 1.2.3.5 # Second controller + +parameters: + {{network.name}}NetName: + description: Name of the {{network.name}} neutron network + default: {{network.name_lower}} + type: string + PortName: + description: Name of the port + default: '' + type: string + ControlPlaneIP: # Here for compatibility with noop.yaml + description: IP address on the control plane + default: '' + type: string + ControlPlaneNetwork: # Here for compatibility with ctlplane_vip.yaml + description: The name of the undercloud Neutron control plane + default: ctlplane + type: string + IPPool: # Set in IPs map, see environments/ips-from-pool-all.yaml + default: {} + type: json + NodeIndex: # First node in the role will get first IP, and so on... + default: 0 + type: number + {{network.name}}NetCidr: + default: {{network.ip_subnet}} + description: Cidr for the {{network.name_lower}} network. + type: string + +outputs: + ip_address: + description: {{network.name}} network IP + value: {get_param: [IPPool, {get_param: {{network.name}}NetName}, {get_param: NodeIndex}]} + ip_address_uri: +{%- if network.ipv6 %} + description: {{network.name}} network IP (with brackets for IPv6 URLs) + value: + list_join: + - '' + - - '[' + - {get_param: [IPPool, {get_param: {{network.name}}NetName}, {get_param: NodeIndex}]} + - ']' +{%- else %} + description: {{network.name}} network IP (for compatibility with {{network.name_lower}}_v6.yaml) + value: {get_param: [IPPool, {get_param: {{network.name}}NetName}, {get_param: NodeIndex}]} +{%- endif %} + ip_subnet: + description: IP/Subnet CIDR for the {{network.name}} network IP + value: + list_join: + - '' + - - {get_param: [IPPool, {get_param: {{network.name}}NetName}, {get_param: NodeIndex}]} + - '/' + - {str_split: ['/', {get_param: {{network.name}}NetCidr}, 1]} + diff --git a/network_data.yaml b/network_data.yaml index 23c231f9..947769ae 100644 --- a/network_data.yaml +++ b/network_data.yaml @@ -5,30 +5,59 @@ # name: Name of the network (mandatory) # name_lower: lowercase version of name used for filenames # (optional, defaults to name.lower()) -# vlan: vlan for the network (optional) -# gateway: gateway for the network (optional) # enabled: Is the network enabled (optional, defaults to true) +# ipv6: Does this network use IPv6 IPs? (optional, defaults to false) +# (optional, may use parameter defaults in environment to set) +# vlan: vlan for the network (optional) # vip: Enable creation of a virtual IP on this network -# [TODO] (dsneddon@redhat.com) - Enable dynamic creation of VIP ports, to support -# VIPs on non-default networks. See https://bugs.launchpad.net/tripleo/+bug/1667104 +# [TODO] (dsneddon@redhat.com) - Enable dynamic creation of VIP ports, +# to support VIPs on non-default networks. +# See https://bugs.launchpad.net/tripleo/+bug/1667104 +# ip_subnet: IP/CIDR, e.g. '192.168.24.0/24' (optional, may use parameter defaults) +# allocation_pools: IP range list e.g. [{'start':'10.0.0.4', 'end':'10.0.0.250}] +# gateway_ip: gateway for the network (optional, may use parameter defaults) +# NOTE: IP-related values set parameter defaults in templates, may be overridden. +# +# Example: +# - name Example +# vip: false +# ip_subnet: '10.0.2.0/24' +# allocation_pools: [{'start': '10.0.2.4', 'end': '10.0.2.250'}] +# gateway_ip: '10.0.2.254' # +# TODO (dsneddon) remove existing templates from j2_excludes.yaml +# and generate all templates dynamically. + - name: External vip: true name_lower: external + ip_subnet: '10.0.0.0/24' + allocation_pools: [{'start': '10.0.0.4', 'end': '10.0.0.250'}] + gateway_ip: '10.0.0.1' - name: InternalApi name_lower: internal_api vip: true + ip_subnet: '172.16.2.0/24' + allocation_pools: [{'start': '172.16.2.4', 'end': '172.16.2.250'}] - name: Storage vip: true name_lower: storage + ip_subnet: '172.16.1.0/24' + allocation_pools: [{'start': '172.16.1.4', 'end': '172.16.1.250'}] - name: StorageMgmt name_lower: storage_mgmt vip: true + ip_subnet: '172.16.3.0/24' + allocation_pools: [{'start': '172.16.3.4', 'end': '172.16.3.250'}] - name: Tenant vip: false # Tenant network does not use VIPs name_lower: tenant + ip_subnet: '172.16.0.0/24' + allocation_pools: [{'start': '172.16.0.4', 'end': '172.16.0.250'}] - name: Management # Management network is disabled by default enabled: false vip: false # Management network does not use VIPs name_lower: management + ip_subnet: '10.0.1.0/24' + allocation_pools: [{'start': '10.0.1.4', 'end': '10.0.1.250'}] diff --git a/tools/process-templates.py b/tools/process-templates.py index badc1426..07c27bad 100755 --- a/tools/process-templates.py +++ b/tools/process-templates.py @@ -96,6 +96,16 @@ def process_templates(template_path, role_data_path, output_dir, r_map = {} for r in role_data: r_map[r.get('name')] = r + + n_map = {} + for n in network_data: + if (n.get('enabled') is not False): + n_map[n.get('name')] = n + if not n.get('name_lower'): + n_map[n.get('name')]['name_lower'] = n.get('name').lower() + else: + print("skipping %s network: network is disabled" % n.get('name')) + excl_templates = ['%s/%s' % (template_path, e) for e in j2_excludes.get('name')] @@ -126,10 +136,13 @@ def process_templates(template_path, role_data_path, output_dir, for f in files: file_path = os.path.join(subdir, f) - # We do two templating passes here: + # We do three templating passes here: # 1. *.role.j2.yaml - we template just the role name # and create multiple files (one per role) - # 2. *.j2.yaml - we template with all roles_data, + # 2 *.network.j2.yaml - we template the network name and + # data and create multiple files for networks and + # network ports (one per network) + # 3. *.j2.yaml - we template with all roles_data, # and create one file common to all roles if f.endswith('.role.j2.yaml'): print("jinja2 rendering role template %s" % f) @@ -167,6 +180,30 @@ def process_templates(template_path, role_data_path, output_dir, else: print('skipping rendering of %s' % out_f_path) + + elif f.endswith('.network.j2.yaml'): + print("jinja2 rendering network template %s" % f) + with open(file_path) as j2_template: + template_data = j2_template.read() + print("jinja2 rendering networks %s" % ",".join(n_map)) + for network in n_map: + j2_data = {'network': n_map[network]} + # Output file names in ".yaml" format + out_f = os.path.basename(f).replace('.network.j2.yaml', + '.yaml') + if os.path.dirname(file_path).endswith('ports'): + out_f = out_f.replace('port', + n_map[network]['name_lower']) + else: + out_f = out_f.replace('network', + n_map[network]['name_lower']) + out_f_path = os.path.join(out_dir, out_f) + if not (out_f_path in excl_templates): + _j2_render_to_file(template_data, j2_data, + out_f_path) + else: + print('skipping rendering of %s' % out_f_path) + elif f.endswith('.j2.yaml'): print("jinja2 rendering normal template %s" % f) with open(file_path) as j2_template: -- cgit 1.2.3-korg