From 5c24cf417b08e0b427435d1bd5d27a4b03467092 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Thu, 24 Nov 2016 10:00:29 -0400 Subject: Cleanup ovs file and restart docker on every upgrade. In 3.3 one of our services lays down a systemd drop-in for configuring Docker networking to use lbr0. In 3.4, this has been changed but the file must be cleaned up manually by us. However, after removing the file docker requires a restart. This had big implications particularly in containerized environments where upgrade is a very fragile series of upgrading and service restarts. To avoid double docker restarts, and thus double service restarts in containerized environments, this change does the following: - Skip restart during docker upgrade, if it is required. We will restart on our own later. - Skip containerized service restarts when we upgrade the services themselves. - Clean shutdown of all containerized services. - Restart Docker. (always, previously this only happened if it needed an upgrade) - Ensure all containerized services are restarted. - Restart rpm node services. (always) - Mark node schedulable again. At the end of this process, docker0 should be back on the system. --- .../openshift-cluster/upgrades/upgrade_nodes.yml | 27 +++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) (limited to 'playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml') diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml b/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml index 53d670196..bb7955c45 100644 --- a/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml +++ b/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml @@ -44,8 +44,13 @@ {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --evacuate --force delegate_to: "{{ groups.oo_first_master.0 }}" when: inventory_hostname in groups.oo_nodes_to_upgrade + tasks: + - include: docker/upgrade.yml + vars: + # We will restart Docker ourselves after everything is ready: + skip_docker_restart: True when: l_docker_upgrade is defined and l_docker_upgrade | bool and not openshift.common.is_atomic | bool - include: "{{ node_config_hook }}" @@ -57,11 +62,31 @@ openshift_version: "{{ openshift_pkg_version | default('') }}" when: inventory_hostname in groups.oo_nodes_to_upgrade and not openshift.common.is_containerized | bool + - name: Remove obsolete docker-sdn-ovs.conf + file: path=/etc/systemd/system/docker.service.d/docker-sdn-ovs.conf state=absent + when: (deployment_type == 'openshift-enterprise' and openshift_release | version_compare('3.4', '>=')) or (deployment_type == 'origin' and openshift_release | version_compare('1.4', '>=')) + - include: containerized_node_upgrade.yml when: inventory_hostname in groups.oo_nodes_to_upgrade and openshift.common.is_containerized | bool - - meta: flush_handlers + - name: Ensure containerized services stopped before Docker restart + service: name={{ item }} state=stopped + with_items: + - etcd_container + - openvswitch + - "{{ openshift.common.service_type }}-master" + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + failed_when: false + when: openshift.common.is_containerized | bool + + # Mandatory Docker restart, ensure all containerized services are running: + - include: docker/restart.yml + - name: Restart rpm node service + service: name="{{ openshift.common.service_type }}-node" state=restarted + when: inventory_hostname in groups.oo_nodes_to_upgrade and not openshift.common.is_containerized | bool - name: Set node schedulability command: > {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --schedulable=true -- cgit v1.2.3