From 7dae2f94927b88ab96a6f447e1595a6f2ef88b9b Mon Sep 17 00:00:00 2001 From: Scott Dodson Date: Mon, 14 Nov 2016 14:18:37 -0500 Subject: Make etcd containerized upgrade stepwise --- .../upgrades/etcd/containerized_tasks.yml | 24 ++++++++---- .../openshift-cluster/upgrades/etcd/main.yml | 45 +++++++++++++++++----- 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/containerized_tasks.yml b/playbooks/common/openshift-cluster/upgrades/etcd/containerized_tasks.yml index 2cbe3b729..9ace7db25 100644 --- a/playbooks/common/openshift-cluster/upgrades/etcd/containerized_tasks.yml +++ b/playbooks/common/openshift-cluster/upgrades/etcd/containerized_tasks.yml @@ -2,13 +2,14 @@ - name: Verify cluster is healthy pre-upgrade command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health" -- name: Update systemd unit - openshift_facts: - role: etcd +- name: Get current image + shell: grep 'ExecStart=' /etc/systemd/system/etcd_container.service | awk '{print $NF}' + register: current_image -- name: Set etcd3 image name +- name: Set new_etcd_image set_fact: - new_etcd_image: "{{ openshift.etcd.etcd_image | regex_replace('/etcd$','/etcd3') }}" + new_etcd_image: "{{ current_image.stdout | regex_replace('/etcd.*$','/etcd3:' ~ upgrade_version ) if upgrade_version | version_compare('3.0','>=') + else current_image.stdout.split(':')[0] ~ ':' ~ upgrade_version }}" - name: Pull new etcd image command: "docker pull {{ new_etcd_image }}" @@ -16,15 +17,22 @@ - name: Update to latest etcd image replace: dest: /etc/systemd/system/etcd_container.service - regexp: "{{ openshift.etcd.etcd_image }}$" + regexp: "{{ current_image.stdout }}$" replace: "{{ new_etcd_image }}" -- name: Update etcd package not excluding etcd3 +- name: Restart etcd_container systemd: name: etcd_container daemon_reload: yes state: restarted +## TODO: probably should just move this into the backup playbooks, also this +## will fail on atomic host. We need to revisit how to do etcd backups there as +## the container may be newer than etcdctl on the host. Assumes etcd3 obsoletes etcd (7.3.1) +- name: Upgrade etcdctl to 3.0 if necessary and able + action: "{{ ansible_pkg_mgr }} name=etcd ensure=latest" + when: openshift.common.is_containerized | bool and not openshift.common.is_atomic | bool and upgrade_version | version_compare('3.0','>=') + - name: Verify cluster is healthy command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health" register: etcdctl @@ -36,4 +44,4 @@ openshift_facts: role: etcd local_facts: - - etcd_image: "{{ new_etcd_image }}" + etcd_image: "{{ new_etcd_image }}" diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/main.yml b/playbooks/common/openshift-cluster/upgrades/etcd/main.yml index 22af9e7e2..cce844403 100644 --- a/playbooks/common/openshift-cluster/upgrades/etcd/main.yml +++ b/playbooks/common/openshift-cluster/upgrades/etcd/main.yml @@ -33,11 +33,16 @@ - name: Determine etcd version hosts: etcd_hosts_to_upgrade tasks: - - name: Record etcd version + - name: Record RPM based etcd version command: rpm -qa --qf '%{version}' etcd\* register: etcd_installed_version failed_when: false when: not openshift.common.is_containerized | bool + - name: Record containerized etcd version + command: docker exec etcd_container rpm -qa --qf '%{version}' etcd\* + register: etcd_installed_version + failed_when: false + when: openshift.common.is_containerized | bool # I really dislike this copy/pasta but I wasn't able to find a way to get it to loop # through hosts, then loop through tasks only when appropriate @@ -50,7 +55,7 @@ - include: rhel_tasks.yml when: etcd_installed_version.stdout | default('99') | version_compare('2.1','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool -- name: Upgrade to 2.2 +- name: Upgrade RPM hosts to 2.2 hosts: etcd_hosts_to_upgrade serial: 1 vars: @@ -59,7 +64,16 @@ - include: rhel_tasks.yml when: etcd_installed_version.stdout | default('99') | version_compare('2.2','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool -- name: Upgrade to 2.3 +- name: Upgrade containerized hosts to 2.2.5 + hosts: etcd_hosts_to_upgrade + serial: 1 + vars: + upgrade_version: 2.2.5 + tasks: + - include: containerized_tasks.yml + when: etcd_installed_version.stdout | default('99') | version_compare('2.2','<') and openshift.common.is_containerized | bool + +- name: Upgrade RPM hosts to 2.3 hosts: etcd_hosts_to_upgrade serial: 1 vars: @@ -68,7 +82,16 @@ - include: rhel_tasks.yml when: etcd_installed_version.stdout | default('99') | version_compare('2.3','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool -- name: Upgrade to 3.0 +- name: Upgrade containerized hosts to 2.3.7 + hosts: etcd_hosts_to_upgrade + serial: 1 + vars: + upgrade_version: 2.3.7 + tasks: + - include: containerized_tasks.yml + when: etcd_installed_version.stdout | default('99') | version_compare('2.3','<') and openshift.common.is_containerized | bool + +- name: Upgrade RPM hosts to 3.0 hosts: etcd_hosts_to_upgrade serial: 1 vars: @@ -77,19 +100,21 @@ - include: rhel_tasks.yml when: etcd_installed_version.stdout | default('99') | version_compare('3.0','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool -- name: Upgrade fedora to latest +- name: Upgrade containerized hosts to etcd3 image hosts: etcd_hosts_to_upgrade serial: 1 + vars: + upgrade_version: 3.0.3 tasks: - - include: fedora_tasks.yml - when: ansible_distribution == 'Fedora' and not openshift.common.is_containerized | bool + - include: containerized_tasks.yml + when: etcd_installed_version.stdout | default('99') | version_compare('3.0','<') and openshift.common.is_containerized | bool -- name: Upgrade containerized hosts to etcd3 image +- name: Upgrade fedora to latest hosts: etcd_hosts_to_upgrade serial: 1 tasks: - - include: containerized_tasks.yml - when: openshift.common.is_containerized | bool + - include: fedora_tasks.yml + when: ansible_distribution == 'Fedora' and not openshift.common.is_containerized | bool - name: Backup etcd include: backup.yml -- cgit v1.2.3