From 1f7ff637a1e5413e2a8d8fe19ffb28b373243e6f Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Wed, 4 Jan 2017 15:40:54 -0400 Subject: Validate system restart policy during pre-upgrade. This was done far into the process potentially leaving the user in a difficult situation if they had now considered they were running the upgrade playbook on a host that would be restarted. Instead check configuration and what host we're running on in pre-upgrade and allow the user to abort before making any substantial changes. This is a step towards merging master upgrade into one serial process. --- .../openshift-cluster/upgrades/v3_3/upgrade.yml | 4 ++ .../upgrades/v3_3/upgrade_control_plane.yml | 4 ++ .../openshift-cluster/upgrades/v3_4/upgrade.yml | 4 ++ .../upgrades/v3_4/upgrade_control_plane.yml | 4 ++ playbooks/byo/openshift-master/restart.yml | 2 + playbooks/common/openshift-master/restart.yml | 67 ---------------------- .../common/openshift-master/restart_services.yml | 6 +- .../common/openshift-master/validate_restart.yml | 65 +++++++++++++++++++++ 8 files changed, 86 insertions(+), 70 deletions(-) create mode 100644 playbooks/common/openshift-master/validate_restart.yml diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_3/upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/v3_3/upgrade.yml index 4ce815271..84a5a026f 100644 --- a/playbooks/byo/openshift-cluster/upgrades/v3_3/upgrade.yml +++ b/playbooks/byo/openshift-cluster/upgrades/v3_3/upgrade.yml @@ -66,6 +66,10 @@ tags: - pre_upgrade +- include: ../../../../common/openshift-master/validate_restart.yml + tags: + - pre_upgrade + - include: ../../../../common/openshift-cluster/upgrades/pre/verify_upgrade_targets.yml tags: - pre_upgrade diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_3/upgrade_control_plane.yml b/playbooks/byo/openshift-cluster/upgrades/v3_3/upgrade_control_plane.yml index d6af71827..7717c95e4 100644 --- a/playbooks/byo/openshift-cluster/upgrades/v3_3/upgrade_control_plane.yml +++ b/playbooks/byo/openshift-cluster/upgrades/v3_3/upgrade_control_plane.yml @@ -71,6 +71,10 @@ tags: - pre_upgrade +- include: ../../../../common/openshift-master/validate_restart.yml + tags: + - pre_upgrade + - include: ../../../../common/openshift-cluster/upgrades/pre/verify_upgrade_targets.yml tags: - pre_upgrade diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_4/upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/v3_4/upgrade.yml index d6115e7a5..59dd802ee 100644 --- a/playbooks/byo/openshift-cluster/upgrades/v3_4/upgrade.yml +++ b/playbooks/byo/openshift-cluster/upgrades/v3_4/upgrade.yml @@ -66,6 +66,10 @@ tags: - pre_upgrade +- include: ../../../../common/openshift-master/validate_restart.yml + tags: + - pre_upgrade + - include: ../../../../common/openshift-cluster/upgrades/pre/verify_upgrade_targets.yml tags: - pre_upgrade diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_4/upgrade_control_plane.yml b/playbooks/byo/openshift-cluster/upgrades/v3_4/upgrade_control_plane.yml index 8cde2ac88..5a7937aed 100644 --- a/playbooks/byo/openshift-cluster/upgrades/v3_4/upgrade_control_plane.yml +++ b/playbooks/byo/openshift-cluster/upgrades/v3_4/upgrade_control_plane.yml @@ -71,6 +71,10 @@ tags: - pre_upgrade +- include: ../../../../common/openshift-master/validate_restart.yml + tags: + - pre_upgrade + - include: ../../../../common/openshift-cluster/upgrades/pre/verify_upgrade_targets.yml tags: - pre_upgrade diff --git a/playbooks/byo/openshift-master/restart.yml b/playbooks/byo/openshift-master/restart.yml index 0a163526a..b6cebe829 100644 --- a/playbooks/byo/openshift-master/restart.yml +++ b/playbooks/byo/openshift-master/restart.yml @@ -15,4 +15,6 @@ tasks: - include_vars: ../../byo/openshift-cluster/cluster_hosts.yml +- include: ../../common/openshift-cluster/evaluate_groups.yml +- include: ../../common/openshift-master/validate_restart.yml - include: ../../common/openshift-master/restart.yml diff --git a/playbooks/common/openshift-master/restart.yml b/playbooks/common/openshift-master/restart.yml index 7b340887a..6fb35f25c 100644 --- a/playbooks/common/openshift-master/restart.yml +++ b/playbooks/common/openshift-master/restart.yml @@ -1,71 +1,4 @@ --- -- include: ../openshift-cluster/evaluate_groups.yml - -- name: Validate configuration for rolling restart - hosts: oo_masters_to_config - roles: - - openshift_facts - tasks: - - fail: - msg: "openshift_rolling_restart_mode must be set to either 'services' or 'system'" - when: openshift_rolling_restart_mode is defined and openshift_rolling_restart_mode not in ["services", "system"] - - openshift_facts: - role: "{{ item.role }}" - local_facts: "{{ item.local_facts }}" - with_items: - - role: common - local_facts: - rolling_restart_mode: "{{ openshift_rolling_restart_mode | default('services') }}" - - role: master - local_facts: - cluster_method: "{{ openshift_master_cluster_method | default(None) }}" - -# Creating a temp file on localhost, we then check each system that will -# be rebooted to see if that file exists, if so we know we're running -# ansible on a machine that needs a reboot, and we need to error out. -- name: Create temp file on localhost - hosts: localhost - connection: local - become: no - gather_facts: no - tasks: - - local_action: command mktemp - register: mktemp - changed_when: false - -- name: Check if temp file exists on any masters - hosts: oo_masters_to_config - tasks: - - stat: path="{{ hostvars.localhost.mktemp.stdout }}" - register: exists - changed_when: false - -- name: Cleanup temp file on localhost - hosts: localhost - connection: local - become: no - gather_facts: no - tasks: - - file: path="{{ hostvars.localhost.mktemp.stdout }}" state=absent - changed_when: false - -- name: Warn if restarting the system where ansible is running - hosts: oo_masters_to_config - tasks: - - pause: - prompt: > - Warning: Running playbook from a host that will be restarted! - Press CTRL+C and A to abort playbook execution. You may - continue by pressing ENTER but the playbook will stop - executing after this system has been restarted and services - must be verified manually. To only restart services, set - openshift_master_rolling_restart_mode=services in host - inventory and relaunch the playbook. - when: exists.stat.exists and openshift.common.rolling_restart_mode == 'system' - - set_fact: - current_host: "{{ exists.stat.exists }}" - when: openshift.common.rolling_restart_mode == 'system' - - name: Restart masters hosts: oo_masters_to_config vars: diff --git a/playbooks/common/openshift-master/restart_services.yml b/playbooks/common/openshift-master/restart_services.yml index b40c32669..508b5a3ac 100644 --- a/playbooks/common/openshift-master/restart_services.yml +++ b/playbooks/common/openshift-master/restart_services.yml @@ -8,14 +8,14 @@ service: name: "{{ openshift.common.service_type }}-master-api" state: restarted - when: openshift_master_ha | bool and openshift.master.cluster_method != 'pacemaker' + when: openshift_master_ha | bool - name: Wait for master API to come back online wait_for: host: "{{ openshift.common.hostname }}" state: started delay: 10 port: "{{ openshift.master.api_port }}" - when: openshift_master_ha | bool and openshift.master.cluster_method != 'pacemaker' + when: openshift_master_ha | bool - name: Restart master controllers service: name: "{{ openshift.common.service_type }}-master-controllers" @@ -23,4 +23,4 @@ # Ignore errrors since it is possible that type != simple for # pre-3.1.1 installations. ignore_errors: true - when: openshift_master_ha | bool and openshift.master.cluster_method != 'pacemaker' + when: openshift_master_ha | bool diff --git a/playbooks/common/openshift-master/validate_restart.yml b/playbooks/common/openshift-master/validate_restart.yml new file mode 100644 index 000000000..5dbb21502 --- /dev/null +++ b/playbooks/common/openshift-master/validate_restart.yml @@ -0,0 +1,65 @@ +--- +- name: Validate configuration for rolling restart + hosts: oo_masters_to_config + roles: + - openshift_facts + tasks: + - fail: + msg: "openshift_rolling_restart_mode must be set to either 'services' or 'system'" + when: openshift_rolling_restart_mode is defined and openshift_rolling_restart_mode not in ["services", "system"] + - openshift_facts: + role: "{{ item.role }}" + local_facts: "{{ item.local_facts }}" + with_items: + - role: common + local_facts: + rolling_restart_mode: "{{ openshift_rolling_restart_mode | default('services') }}" + - role: master + local_facts: + cluster_method: "{{ openshift_master_cluster_method | default(None) }}" + +# Creating a temp file on localhost, we then check each system that will +# be rebooted to see if that file exists, if so we know we're running +# ansible on a machine that needs a reboot, and we need to error out. +- name: Create temp file on localhost + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - local_action: command mktemp + register: mktemp + changed_when: false + +- name: Check if temp file exists on any masters + hosts: oo_masters_to_config + tasks: + - stat: path="{{ hostvars.localhost.mktemp.stdout }}" + register: exists + changed_when: false + +- name: Cleanup temp file on localhost + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - file: path="{{ hostvars.localhost.mktemp.stdout }}" state=absent + changed_when: false + +- name: Warn if restarting the system where ansible is running + hosts: oo_masters_to_config + tasks: + - pause: + prompt: > + Warning: Running playbook from a host that will be restarted! + Press CTRL+C and A to abort playbook execution. You may + continue by pressing ENTER but the playbook will stop + executing after this system has been restarted and services + must be verified manually. To only restart services, set + openshift_master_rolling_restart_mode=services in host + inventory and relaunch the playbook. + when: exists.stat.exists and openshift.common.rolling_restart_mode == 'system' + - set_fact: + current_host: "{{ exists.stat.exists }}" + when: openshift.common.rolling_restart_mode == 'system' -- cgit v1.2.3