diff options
| -rw-r--r-- | apex/builders/common_builder.py | 43 | ||||
| -rw-r--r-- | apex/overcloud/deploy.py | 3 | ||||
| -rw-r--r-- | build/csit-environment.yaml | 22 | ||||
| -rw-r--r-- | build/csit-queens-environment.yaml | 22 | ||||
| -rw-r--r-- | build/csit-rocky-environment.yaml | 22 | ||||
| -rw-r--r-- | config/deploy/common-patches.yaml | 8 | ||||
| -rw-r--r-- | lib/ansible/playbooks/patch_containers.yml | 13 | ||||
| -rw-r--r-- | lib/ansible/playbooks/prepare_overcloud_containers.yml | 11 |
8 files changed, 115 insertions, 29 deletions
diff --git a/apex/builders/common_builder.py b/apex/builders/common_builder.py index 7627ae3c..59af94cd 100644 --- a/apex/builders/common_builder.py +++ b/apex/builders/common_builder.py @@ -62,12 +62,13 @@ def project_to_path(project, patch=None): def project_to_docker_image(project, docker_url): """ Translates OpenStack project to OOO services that are containerized - :param project: name of OpenStack project + :param project: short name of OpenStack project :return: List of OOO docker service names """ # Fetch all docker containers in docker hub with tripleo and filter # based on project - + logging.info("Checking for docker images matching project: {}".format( + project)) hub_output = utils.open_webpage( urllib.parse.urljoin(docker_url, '?page_size=1024'), timeout=10) @@ -85,6 +86,8 @@ def project_to_docker_image(project, docker_url): for result in results: if result['name'].startswith("centos-binary-{}".format(project)): # add as docker image shortname (just service name) + logging.debug("Adding docker image {} for project {} for " + "patching".format(result['name'], project)) docker_images.append(result['name'].replace('centos-binary-', '')) return docker_images @@ -184,8 +187,16 @@ def add_upstream_patches(patches, image, tmp_dir, if docker_tag and 'python' in project_path: # Projects map to multiple THT services, need to check which # are supported - ooo_docker_services = project_to_docker_image(patch['project'], + project_short_name = os.path.basename(patch['project']) + ooo_docker_services = project_to_docker_image(project_short_name, docker_url) + if not ooo_docker_services: + logging.error("Did not find any matching docker containers " + "for project: {}".format(project_short_name)) + raise exc.ApexCommonBuilderException( + 'Unable to find docker services for python project in ' + 'patch') + # Just use the first image to see if patch was promoted into it docker_img = ooo_docker_services[0] else: ooo_docker_services = [] @@ -200,24 +211,38 @@ def add_upstream_patches(patches, image, tmp_dir, if patch_diff and not patch_promoted: patch_file = "{}.patch".format(patch['change-id']) + patch_file_paths = [] # If we found services, then we treat the patch like it applies to # docker only if ooo_docker_services: os_version = default_branch.replace('stable/', '') for service in ooo_docker_services: docker_services = docker_services.union({service}) + # We need to go root to be able to install patch and then + # switch back to previous user. Some containers that + # have the same name as the project do not necessarily + # contain the project code. For example + # novajoin-notifier does not contain nova package code. + # Therefore we must try to patch and unfortunately + # ignore failures until we have a better way of checking + # this docker_cmds = [ "WORKDIR {}".format(project_path), + "USER root", + "ARG REAL_USER", + "RUN yum -y install patch", "ADD {} {}".format(patch_file, project_path), - "RUN patch -p1 < {}".format(patch_file) + "RUN patch -p1 < {} || echo " + "'Patching failed'".format(patch_file), + "USER $REAL_USER" ] src_img_uri = "{}:8787/tripleo{}/centos-binary-{}:" \ "{}".format(uc_ip, os_version, service, docker_tag) oc_builder.build_dockerfile(service, tmp_dir, docker_cmds, src_img_uri) - patch_file_path = os.path.join(tmp_dir, 'containers', - patch_file) + patch_file_paths.append(os.path.join( + tmp_dir, "containers/{}".format(service), patch_file)) else: patch_file_path = os.path.join(tmp_dir, patch_file) virt_ops.extend([ @@ -227,8 +252,10 @@ def add_upstream_patches(patches, image, tmp_dir, project_path, patch_file)}]) logging.info("Adding patch {} to {}".format(patch_file, image)) - with open(patch_file_path, 'w') as fh: - fh.write(patch_diff) + patch_file_paths.append(patch_file_path) + for patch_fp in patch_file_paths: + with open(patch_fp, 'w') as fh: + fh.write(patch_diff) else: logging.info("Ignoring patch:\n{}".format(patch)) if len(virt_ops) > 1: diff --git a/apex/overcloud/deploy.py b/apex/overcloud/deploy.py index a4a96912..708a6625 100644 --- a/apex/overcloud/deploy.py +++ b/apex/overcloud/deploy.py @@ -448,6 +448,9 @@ def prep_image(ds, ns, img, tmp_dir, root_pw=None, docker_tag=None, {con.VIRT_RUN_CMD: "chmod 777 /glance"}, {con.VIRT_RUN_CMD: "chmod 777 /cinder"}, {con.VIRT_RUN_CMD: "chmod 777 /nova"}, + {con.VIRT_RUN_CMD: "chown nfsnobody:nfsnobody /glance"}, + {con.VIRT_RUN_CMD: "chown nfsnobody:nfsnobody /cinder"}, + {con.VIRT_RUN_CMD: "chown nfsnobody:nfsnobody /nova"}, {con.VIRT_RUN_CMD: "echo '/glance *(rw,sync," "no_root_squash,no_acl)' > /etc/exports"}, {con.VIRT_RUN_CMD: "echo '/cinder *(rw,sync," diff --git a/build/csit-environment.yaml b/build/csit-environment.yaml index 58676dc6..39486d32 100644 --- a/build/csit-environment.yaml +++ b/build/csit-environment.yaml @@ -15,16 +15,28 @@ parameter_defaults: tripleo::ringbuilder::build_ring: false nova::api::default_floating_pool: 'external' ControllerExtraConfig: - tripleo::firewall:firewall_rules: - '139 allow NFS': - dport: 2049 + tripleo::firewall::firewall_rules: + '139 allow NFS TCP': + dport: + - 2049 + - 111 + - 32765 + proto: tcp + action: accept + '140 allow NFS UDP': + dport: + - 2049 + - 111 + - 32765 + proto: udp + action: accept GlanceNfsEnabled: true GlanceNfsShare: overcloud-controller-0.opnfvlf.org:/glance GlanceNfsOptions: - 'rw,sync,nosharecache,context=system_u:object_r:glance_var_lib_t:s0' + 'rw,sync,context=system_u:object_r:glance_var_lib_t:s0' NovaNfsEnabled: true NovaNfsShare: overcloud-controller-0.opnfvlf.org:/nova - NovaNfsOptions: 'rw,sync,nosharecache,context=system_u:object_r:nfs_t:s0' + NovaNfsOptions: 'rw,sync,context=system_u:object_r:nfs_t:s0' DockerPuppetProcessCount: 10 NeutronNetworkVLANRanges: 'datacentre:500:525' SshServerOptions: diff --git a/build/csit-queens-environment.yaml b/build/csit-queens-environment.yaml index 2252bb02..12c994d1 100644 --- a/build/csit-queens-environment.yaml +++ b/build/csit-queens-environment.yaml @@ -15,16 +15,28 @@ parameter_defaults: tripleo::ringbuilder::build_ring: false nova::api::default_floating_pool: 'external' ControllerExtraConfig: - tripleo::firewall:firewall_rules: - '139 allow NFS': - dport: 2049 + tripleo::firewall::firewall_rules: + '139 allow NFS TCP': + dport: + - 2049 + - 111 + - 32765 + proto: tcp + action: accept + '140 allow NFS UDP': + dport: + - 2049 + - 111 + - 32765 + proto: udp + action: accept GlanceNfsEnabled: true GlanceNfsShare: overcloud-controller-0.opnfvlf.org:/glance GlanceNfsOptions: - 'rw,sync,nosharecache,context=system_u:object_r:glance_var_lib_t:s0' + 'rw,sync,context=system_u:object_r:glance_var_lib_t:s0' NovaNfsEnabled: true NovaNfsShare: overcloud-controller-0.opnfvlf.org:/nova - NovaNfsOptions: 'rw,sync,nosharecache,context=system_u:object_r:nfs_t:s0' + NovaNfsOptions: 'rw,sync,context=system_u:object_r:nfs_t:s0' DockerPuppetProcessCount: 10 NeutronNetworkVLANRanges: 'datacentre:500:525' SshServerOptions: diff --git a/build/csit-rocky-environment.yaml b/build/csit-rocky-environment.yaml index 58676dc6..39486d32 100644 --- a/build/csit-rocky-environment.yaml +++ b/build/csit-rocky-environment.yaml @@ -15,16 +15,28 @@ parameter_defaults: tripleo::ringbuilder::build_ring: false nova::api::default_floating_pool: 'external' ControllerExtraConfig: - tripleo::firewall:firewall_rules: - '139 allow NFS': - dport: 2049 + tripleo::firewall::firewall_rules: + '139 allow NFS TCP': + dport: + - 2049 + - 111 + - 32765 + proto: tcp + action: accept + '140 allow NFS UDP': + dport: + - 2049 + - 111 + - 32765 + proto: udp + action: accept GlanceNfsEnabled: true GlanceNfsShare: overcloud-controller-0.opnfvlf.org:/glance GlanceNfsOptions: - 'rw,sync,nosharecache,context=system_u:object_r:glance_var_lib_t:s0' + 'rw,sync,context=system_u:object_r:glance_var_lib_t:s0' NovaNfsEnabled: true NovaNfsShare: overcloud-controller-0.opnfvlf.org:/nova - NovaNfsOptions: 'rw,sync,nosharecache,context=system_u:object_r:nfs_t:s0' + NovaNfsOptions: 'rw,sync,context=system_u:object_r:nfs_t:s0' DockerPuppetProcessCount: 10 NeutronNetworkVLANRanges: 'datacentre:500:525' SshServerOptions: diff --git a/config/deploy/common-patches.yaml b/config/deploy/common-patches.yaml index bac6812c..7eb3f975 100644 --- a/config/deploy/common-patches.yaml +++ b/config/deploy/common-patches.yaml @@ -15,6 +15,8 @@ patches: project: openstack/puppet-tripleo - change-id: I93e3d355625508fdc42f44bdd358f3ba86fbd8d7 project: openstack/puppet-tripleo + - change-id: Id68aa27a8ab08d9c00655e5ed6b48d194aa8e6f6 + project: openstack/nova rocky: undercloud: - change-id: I2e0a40d7902f592e4b7bd727f57048111e0bea36 @@ -34,6 +36,9 @@ patches: - change-id: I93e3d355625508fdc42f44bdd358f3ba86fbd8d7 project: openstack/puppet-tripleo branch: master + - change-id: Id68aa27a8ab08d9c00655e5ed6b48d194aa8e6f6 + project: openstack/nova + branch: master queens: undercloud: - change-id: I966bf7f6f8d1cbc656abfad59e8bb927e1aa53c2 @@ -43,3 +48,6 @@ patches: project: openstack/puppet-tripleo - change-id: I93e3d355625508fdc42f44bdd358f3ba86fbd8d7 project: openstack/puppet-tripleo + - change-id: Id68aa27a8ab08d9c00655e5ed6b48d194aa8e6f6 + project: openstack/nova + branch: master diff --git a/lib/ansible/playbooks/patch_containers.yml b/lib/ansible/playbooks/patch_containers.yml new file mode 100644 index 00000000..bc4899ba --- /dev/null +++ b/lib/ansible/playbooks/patch_containers.yml @@ -0,0 +1,13 @@ +--- + - name: "Pull docker image to ensure it exists locally: {{ item }}" + shell: docker pull {{ undercloud_ip }}:8787/tripleo{{ os_version }}/centos-binary-{{ item }}:current-tripleo + - name: "Find docker image user {{ item }}" + shell: > + docker inspect --format='{{ '{{' }}.ContainerConfig.User{{ '}}' }}' + {{ undercloud_ip }}:8787/tripleo{{ os_version }}/centos-binary-{{ item }}:current-tripleo + register: user_result + - name: "Patch docker image {{ item }}" + shell: > + cd /home/stack/containers/{{ item }} && docker build + --build-arg REAL_USER={{ user_result.stdout }} + -t {{ undercloud_ip }}:8787/tripleo{{ os_version }}/centos-binary-{{ item }}:apex . diff --git a/lib/ansible/playbooks/prepare_overcloud_containers.yml b/lib/ansible/playbooks/prepare_overcloud_containers.yml index e2a4e134..db1bff89 100644 --- a/lib/ansible/playbooks/prepare_overcloud_containers.yml +++ b/lib/ansible/playbooks/prepare_overcloud_containers.yml @@ -28,14 +28,13 @@ url: http://{{ undercloud_ip }}:8787/v2/_catalog body_format: json register: response - - name: Patch Docker images - shell: > - cd /home/stack/containers/{{ item }} && docker build - -t {{ undercloud_ip }}:8787/tripleo{{ os_version }}/centos-binary-{{ item }}:apex . + - include_tasks: patch_containers.yml + with_items: "{{ patched_docker_services }}" + loop_control: + loop_var: item when: - patched_docker_services|length > 0 - item in (response.json)['repositories']|join(" ") - with_items: "{{ patched_docker_services }}" - name: Push patched docker images to local registry shell: docker push {{ undercloud_ip }}:8787/tripleo{{ os_version }}/centos-binary-{{ item }}:apex when: @@ -45,7 +44,7 @@ - name: Modify Images with Apex tag replace: path: "/home/stack/docker-images.yaml" - regexp: "(\\s*Docker.*?:.*?centos-binary-{{ item[1] }}):.*" + regexp: "(\\s*Docker.*?:.*?centos-binary-{{ item }}):.*" replace: '\1:apex' with_items: "{{ patched_docker_services }}" become: yes |