From 1f7ff637a1e5413e2a8d8fe19ffb28b373243e6f Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Wed, 4 Jan 2017 15:40:54 -0400 Subject: Validate system restart policy during pre-upgrade. This was done far into the process potentially leaving the user in a difficult situation if they had now considered they were running the upgrade playbook on a host that would be restarted. Instead check configuration and what host we're running on in pre-upgrade and allow the user to abort before making any substantial changes. This is a step towards merging master upgrade into one serial process. --- .../openshift-cluster/upgrades/v3_3/upgrade.yml | 4 ++ .../upgrades/v3_3/upgrade_control_plane.yml | 4 ++ .../openshift-cluster/upgrades/v3_4/upgrade.yml | 4 ++ .../upgrades/v3_4/upgrade_control_plane.yml | 4 ++ playbooks/byo/openshift-master/restart.yml | 2 + playbooks/common/openshift-master/restart.yml | 67 ---------------------- .../common/openshift-master/restart_services.yml | 6 +- .../common/openshift-master/validate_restart.yml | 65 +++++++++++++++++++++ 8 files changed, 86 insertions(+), 70 deletions(-) create mode 100644 playbooks/common/openshift-master/validate_restart.yml diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_3/upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/v3_3/upgrade.yml index 4ce815271..84a5a026f 100644 --- a/playbooks/byo/openshift-cluster/upgrades/v3_3/upgrade.yml +++ b/playbooks/byo/openshift-cluster/upgrades/v3_3/upgrade.yml @@ -66,6 +66,10 @@ tags: - pre_upgrade +- include: ../../../../common/openshift-master/validate_restart.yml + tags: + - pre_upgrade + - include: ../../../../common/openshift-cluster/upgrades/pre/verify_upgrade_targets.yml tags: - pre_upgrade diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_3/upgrade_control_plane.yml b/playbooks/byo/openshift-cluster/upgrades/v3_3/upgrade_control_plane.yml index d6af71827..7717c95e4 100644 --- a/playbooks/byo/openshift-cluster/upgrades/v3_3/upgrade_control_plane.yml +++ b/playbooks/byo/openshift-cluster/upgrades/v3_3/upgrade_control_plane.yml @@ -71,6 +71,10 @@ tags: - pre_upgrade +- include: ../../../../common/openshift-master/validate_restart.yml + tags: + - pre_upgrade + - include: ../../../../common/openshift-cluster/upgrades/pre/verify_upgrade_targets.yml tags: - pre_upgrade diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_4/upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/v3_4/upgrade.yml index d6115e7a5..59dd802ee 100644 --- a/playbooks/byo/openshift-cluster/upgrades/v3_4/upgrade.yml +++ b/playbooks/byo/openshift-cluster/upgrades/v3_4/upgrade.yml @@ -66,6 +66,10 @@ tags: - pre_upgrade +- include: ../../../../common/openshift-master/validate_restart.yml + tags: + - pre_upgrade + - include: ../../../../common/openshift-cluster/upgrades/pre/verify_upgrade_targets.yml tags: - pre_upgrade diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_4/upgrade_control_plane.yml b/playbooks/byo/openshift-cluster/upgrades/v3_4/upgrade_control_plane.yml index 8cde2ac88..5a7937aed 100644 --- a/playbooks/byo/openshift-cluster/upgrades/v3_4/upgrade_control_plane.yml +++ b/playbooks/byo/openshift-cluster/upgrades/v3_4/upgrade_control_plane.yml @@ -71,6 +71,10 @@ tags: - pre_upgrade +- include: ../../../../common/openshift-master/validate_restart.yml + tags: + - pre_upgrade + - include: ../../../../common/openshift-cluster/upgrades/pre/verify_upgrade_targets.yml tags: - pre_upgrade diff --git a/playbooks/byo/openshift-master/restart.yml b/playbooks/byo/openshift-master/restart.yml index 0a163526a..b6cebe829 100644 --- a/playbooks/byo/openshift-master/restart.yml +++ b/playbooks/byo/openshift-master/restart.yml @@ -15,4 +15,6 @@ tasks: - include_vars: ../../byo/openshift-cluster/cluster_hosts.yml +- include: ../../common/openshift-cluster/evaluate_groups.yml +- include: ../../common/openshift-master/validate_restart.yml - include: ../../common/openshift-master/restart.yml diff --git a/playbooks/common/openshift-master/restart.yml b/playbooks/common/openshift-master/restart.yml index 7b340887a..6fb35f25c 100644 --- a/playbooks/common/openshift-master/restart.yml +++ b/playbooks/common/openshift-master/restart.yml @@ -1,71 +1,4 @@ --- -- include: ../openshift-cluster/evaluate_groups.yml - -- name: Validate configuration for rolling restart - hosts: oo_masters_to_config - roles: - - openshift_facts - tasks: - - fail: - msg: "openshift_rolling_restart_mode must be set to either 'services' or 'system'" - when: openshift_rolling_restart_mode is defined and openshift_rolling_restart_mode not in ["services", "system"] - - openshift_facts: - role: "{{ item.role }}" - local_facts: "{{ item.local_facts }}" - with_items: - - role: common - local_facts: - rolling_restart_mode: "{{ openshift_rolling_restart_mode | default('services') }}" - - role: master - local_facts: - cluster_method: "{{ openshift_master_cluster_method | default(None) }}" - -# Creating a temp file on localhost, we then check each system that will -# be rebooted to see if that file exists, if so we know we're running -# ansible on a machine that needs a reboot, and we need to error out. -- name: Create temp file on localhost - hosts: localhost - connection: local - become: no - gather_facts: no - tasks: - - local_action: command mktemp - register: mktemp - changed_when: false - -- name: Check if temp file exists on any masters - hosts: oo_masters_to_config - tasks: - - stat: path="{{ hostvars.localhost.mktemp.stdout }}" - register: exists - changed_when: false - -- name: Cleanup temp file on localhost - hosts: localhost - connection: local - become: no - gather_facts: no - tasks: - - file: path="{{ hostvars.localhost.mktemp.stdout }}" state=absent - changed_when: false - -- name: Warn if restarting the system where ansible is running - hosts: oo_masters_to_config - tasks: - - pause: - prompt: > - Warning: Running playbook from a host that will be restarted! - Press CTRL+C and A to abort playbook execution. You may - continue by pressing ENTER but the playbook will stop - executing after this system has been restarted and services - must be verified manually. To only restart services, set - openshift_master_rolling_restart_mode=services in host - inventory and relaunch the playbook. - when: exists.stat.exists and openshift.common.rolling_restart_mode == 'system' - - set_fact: - current_host: "{{ exists.stat.exists }}" - when: openshift.common.rolling_restart_mode == 'system' - - name: Restart masters hosts: oo_masters_to_config vars: diff --git a/playbooks/common/openshift-master/restart_services.yml b/playbooks/common/openshift-master/restart_services.yml index b40c32669..508b5a3ac 100644 --- a/playbooks/common/openshift-master/restart_services.yml +++ b/playbooks/common/openshift-master/restart_services.yml @@ -8,14 +8,14 @@ service: name: "{{ openshift.common.service_type }}-master-api" state: restarted - when: openshift_master_ha | bool and openshift.master.cluster_method != 'pacemaker' + when: openshift_master_ha | bool - name: Wait for master API to come back online wait_for: host: "{{ openshift.common.hostname }}" state: started delay: 10 port: "{{ openshift.master.api_port }}" - when: openshift_master_ha | bool and openshift.master.cluster_method != 'pacemaker' + when: openshift_master_ha | bool - name: Restart master controllers service: name: "{{ openshift.common.service_type }}-master-controllers" @@ -23,4 +23,4 @@ # Ignore errrors since it is possible that type != simple for # pre-3.1.1 installations. ignore_errors: true - when: openshift_master_ha | bool and openshift.master.cluster_method != 'pacemaker' + when: openshift_master_ha | bool diff --git a/playbooks/common/openshift-master/validate_restart.yml b/playbooks/common/openshift-master/validate_restart.yml new file mode 100644 index 000000000..5dbb21502 --- /dev/null +++ b/playbooks/common/openshift-master/validate_restart.yml @@ -0,0 +1,65 @@ +--- +- name: Validate configuration for rolling restart + hosts: oo_masters_to_config + roles: + - openshift_facts + tasks: + - fail: + msg: "openshift_rolling_restart_mode must be set to either 'services' or 'system'" + when: openshift_rolling_restart_mode is defined and openshift_rolling_restart_mode not in ["services", "system"] + - openshift_facts: + role: "{{ item.role }}" + local_facts: "{{ item.local_facts }}" + with_items: + - role: common + local_facts: + rolling_restart_mode: "{{ openshift_rolling_restart_mode | default('services') }}" + - role: master + local_facts: + cluster_method: "{{ openshift_master_cluster_method | default(None) }}" + +# Creating a temp file on localhost, we then check each system that will +# be rebooted to see if that file exists, if so we know we're running +# ansible on a machine that needs a reboot, and we need to error out. +- name: Create temp file on localhost + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - local_action: command mktemp + register: mktemp + changed_when: false + +- name: Check if temp file exists on any masters + hosts: oo_masters_to_config + tasks: + - stat: path="{{ hostvars.localhost.mktemp.stdout }}" + register: exists + changed_when: false + +- name: Cleanup temp file on localhost + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - file: path="{{ hostvars.localhost.mktemp.stdout }}" state=absent + changed_when: false + +- name: Warn if restarting the system where ansible is running + hosts: oo_masters_to_config + tasks: + - pause: + prompt: > + Warning: Running playbook from a host that will be restarted! + Press CTRL+C and A to abort playbook execution. You may + continue by pressing ENTER but the playbook will stop + executing after this system has been restarted and services + must be verified manually. To only restart services, set + openshift_master_rolling_restart_mode=services in host + inventory and relaunch the playbook. + when: exists.stat.exists and openshift.common.rolling_restart_mode == 'system' + - set_fact: + current_host: "{{ exists.stat.exists }}" + when: openshift.common.rolling_restart_mode == 'system' -- cgit v1.2.3 From 362a4eafc1ae9122f98adb5b96ad86ca39529c96 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Fri, 6 Jan 2017 15:14:19 -0400 Subject: Perform master upgrades in a single play serially. --- playbooks/byo/openshift-master/restart.yml | 12 ++++++++- .../upgrades/upgrade_control_plane.yml | 29 ++++++++++------------ playbooks/common/openshift-master/restart.yml | 11 -------- 3 files changed, 24 insertions(+), 28 deletions(-) delete mode 100644 playbooks/common/openshift-master/restart.yml diff --git a/playbooks/byo/openshift-master/restart.yml b/playbooks/byo/openshift-master/restart.yml index b6cebe829..b60807a71 100644 --- a/playbooks/byo/openshift-master/restart.yml +++ b/playbooks/byo/openshift-master/restart.yml @@ -17,4 +17,14 @@ - include: ../../common/openshift-cluster/evaluate_groups.yml - include: ../../common/openshift-master/validate_restart.yml -- include: ../../common/openshift-master/restart.yml + +- name: Restart masters + hosts: oo_masters_to_config + vars: + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + serial: 1 + tasks: + - include: restart_hosts.yml + when: openshift.common.rolling_restart_mode == 'system' + - include: restart_services.yml + when: openshift.common.rolling_restart_mode == 'services' diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml index 6950b6166..7f738ea0f 100644 --- a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml +++ b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml @@ -30,14 +30,6 @@ - name: Upgrade and backup etcd include: ./etcd/main.yml -- name: Upgrade master packages - hosts: oo_masters_to_config - roles: - - openshift_facts - tasks: - - include: rpm_upgrade.yml component=master - when: not openshift.common.is_containerized | bool - # Create service signer cert when missing. Service signer certificate # is added to master config in the master config hook for v3_3. - name: Determine if service signer cert must be created @@ -59,14 +51,20 @@ roles: - openshift_master_facts -- name: Upgrade master config and systemd units +- name: Upgrade master hosts: oo_masters_to_config + vars: + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + serial: 1 handlers: - include: ../../../../roles/openshift_master/handlers/main.yml static: yes roles: - openshift_facts post_tasks: + - include: rpm_upgrade.yml component=master + when: not openshift.common.is_containerized | bool + - include_vars: ../../../../roles/openshift_master_facts/vars/main.yml - include: upgrade_scheduler.yml @@ -104,9 +102,12 @@ state: link when: ca_crt_stat.stat.isreg and not ca_bundle_stat.stat.exists -- name: Set master update status to complete - hosts: oo_masters_to_config - tasks: + - include: ../../openshift-master/restart_hosts.yml + when: openshift.common.rolling_restart_mode == 'system' + + - include: ../../openshift-master/restart_services.yml + when: openshift.common.rolling_restart_mode == 'services' + - set_fact: master_update_complete: True @@ -128,10 +129,6 @@ msg: "Upgrade cannot continue. The following masters did not finish updating: {{ master_update_failed | join(',') }}" when: master_update_failed | length > 0 -# We are now ready to restart master services (or entire system -# depending on openshift_rolling_restart_mode): -- include: ../../openshift-master/restart.yml - ############################################################################### # Reconcile Cluster Roles, Cluster Role Bindings and Security Context Constraints ############################################################################### diff --git a/playbooks/common/openshift-master/restart.yml b/playbooks/common/openshift-master/restart.yml deleted file mode 100644 index 6fb35f25c..000000000 --- a/playbooks/common/openshift-master/restart.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Restart masters - hosts: oo_masters_to_config - vars: - openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" - serial: 1 - tasks: - - include: restart_hosts.yml - when: openshift.common.rolling_restart_mode == 'system' - - include: restart_services.yml - when: openshift.common.rolling_restart_mode == 'services' -- cgit v1.2.3