From 8774f4995654715629be47a8cd5814bdb1962ec9 Mon Sep 17 00:00:00 2001 From: Andrew Butcher Date: Mon, 11 Jan 2016 11:24:27 -0500 Subject: Add wait for API before starting controllers w/ native ha install. --- roles/openshift_master/tasks/main.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml index 3b46a0df4..fbbcd2a8f 100644 --- a/roles/openshift_master/tasks/main.yml +++ b/roles/openshift_master/tasks/main.yml @@ -287,6 +287,22 @@ master_api_service_status_changed: "{{ start_result | changed }}" when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' +- name: Wait for API to become available + # Using curl here since the uri module requires python-httplib2 and + # wait_for port doesn't provide health information. + command: > + curl -k --head --silent {{ openshift.master.api_url }} + register: api_available_output + until: api_available_output.stdout.find("200 OK") != -1 + retries: 120 + delay: 1 + changed_when: false + +- fail: + msg: > + Unable to contact master API at {{ openshift.master.api_url }} + when: api_available_output.stdout.find("200 OK") == -1 + - name: Start and enable master controller service: name={{ openshift.common.service_type }}-master-controllers enabled=yes state=started when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' -- cgit v1.2.3 From 25e213f79ba5e25bf51d584971064e26d3537b49 Mon Sep 17 00:00:00 2001 From: Scott Dodson Date: Thu, 7 Jan 2016 21:59:46 -0500 Subject: Add a Verify API Server handler that waits for the API server to become available --- playbooks/common/openshift-node/config.yml | 11 ++++++++++- roles/openshift_master/handlers/main.yml | 9 +++++++++ roles/openshift_master/tasks/main.yml | 2 ++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml index 483a7768c..fbaf64300 100644 --- a/playbooks/common/openshift-node/config.yml +++ b/playbooks/common/openshift-node/config.yml @@ -215,6 +215,15 @@ | oo_collect('openshift.common.hostname') }}" openshift_node_vars: "{{ hostvars | oo_select_keys(groups['oo_nodes_to_config']) }}" pre_tasks: - + # Necessary because when you're on a node that's also a master the master will be + # restarted after the node restarts docker and it will take up to 60 seconds for + # systemd to start the master again + - name: Wait for master to become available before proceeding + wait_for: + host: "{{ hostvars[groups.oo_first_master.0].openshift.common.ip }}" + port: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_port }}" + state: started + timeout: 180 + when: openshift.common.is_containerized | bool roles: - openshift_manage_node diff --git a/roles/openshift_master/handlers/main.yml b/roles/openshift_master/handlers/main.yml index e1b95eda4..523ba8ee4 100644 --- a/roles/openshift_master/handlers/main.yml +++ b/roles/openshift_master/handlers/main.yml @@ -2,11 +2,20 @@ - name: restart master service: name={{ openshift.common.service_type }}-master state=restarted when: (not openshift_master_ha | bool) and (not (master_service_status_changed | default(false) | bool)) + notify: Verify API Server - name: restart master api service: name={{ openshift.common.service_type }}-master-api state=restarted when: (openshift_master_ha | bool) and (not (master_api_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native' + notify: Verify API Server - name: restart master controllers service: name={{ openshift.common.service_type }}-master-controllers state=restarted when: (openshift_master_ha | bool) and (not (master_controllers_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native' + +- name: Verify API Server + wait_for: + host: "{{ openshift.common.ip }}" + port: "{{ openshift.master.api_port }}" + state: started + timeout: 180 diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml index fbbcd2a8f..b43e9177e 100644 --- a/roles/openshift_master/tasks/main.yml +++ b/roles/openshift_master/tasks/main.yml @@ -269,6 +269,7 @@ service: name={{ openshift.common.service_type }}-master enabled=yes state=started when: not openshift_master_ha | bool register: start_result + notify: Verify API Server - name: Stop and disable non HA master when running HA service: name={{ openshift.common.service_type }}-master enabled=no state=stopped @@ -282,6 +283,7 @@ service: name={{ openshift.common.service_type }}-master-api enabled=yes state=started when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' register: start_result + notify: Verify API Server - set_fact: master_api_service_status_changed: "{{ start_result | changed }}" -- cgit v1.2.3 From 609469eb8d25baeee30cda96377c9b3fda6e499d Mon Sep 17 00:00:00 2001 From: Andrew Butcher Date: Tue, 12 Jan 2016 17:18:18 -0500 Subject: Update api verification. --- playbooks/common/openshift-node/config.yml | 20 ++++++++++++++------ roles/openshift_master/handlers/main.yml | 14 +++++++++----- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml index fbaf64300..336cbed5e 100644 --- a/playbooks/common/openshift-node/config.yml +++ b/playbooks/common/openshift-node/config.yml @@ -218,12 +218,20 @@ # Necessary because when you're on a node that's also a master the master will be # restarted after the node restarts docker and it will take up to 60 seconds for # systemd to start the master again - - name: Wait for master to become available before proceeding - wait_for: - host: "{{ hostvars[groups.oo_first_master.0].openshift.common.ip }}" - port: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_port }}" - state: started - timeout: 180 + - name: Wait for master API to become available before proceeding + # Using curl here since the uri module requires python-httplib2 and + # wait_for port doesn't provide health information. + command: > + curl -k --head --silent {{ openshift.master.api_url }} + register: api_available_output + until: api_available_output.stdout.find("200 OK") != -1 + retries: 120 + delay: 1 + changed_when: false when: openshift.common.is_containerized | bool + - fail: + msg: > + Unable to contact master API at {{ openshift.master.api_url }} + when: openshift.common.is_containerized | bool and api_available_output.stdout.find("200 OK") == -1 roles: - openshift_manage_node diff --git a/roles/openshift_master/handlers/main.yml b/roles/openshift_master/handlers/main.yml index 523ba8ee4..6b9992eea 100644 --- a/roles/openshift_master/handlers/main.yml +++ b/roles/openshift_master/handlers/main.yml @@ -14,8 +14,12 @@ when: (openshift_master_ha | bool) and (not (master_controllers_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native' - name: Verify API Server - wait_for: - host: "{{ openshift.common.ip }}" - port: "{{ openshift.master.api_port }}" - state: started - timeout: 180 + # Using curl here since the uri module requires python-httplib2 and + # wait_for port doesn't provide health information. + command: > + curl -k --head --silent {{ openshift.master.api_url }} + register: api_available_output + until: api_available_output.stdout.find("200 OK") != -1 + retries: 120 + delay: 1 + changed_when: false -- cgit v1.2.3 From 2e3e0ebe0d98f5374fbfb3a95145a9665d57fe69 Mon Sep 17 00:00:00 2001 From: Andrew Butcher Date: Wed, 13 Jan 2016 10:16:43 -0500 Subject: Add wait in between api and controllers start for native ha. --- playbooks/common/openshift-master/config.yml | 1 + roles/openshift_master/tasks/main.yml | 9 +++------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml index 4ecdf2a0c..0df03f194 100644 --- a/playbooks/common/openshift-master/config.yml +++ b/playbooks/common/openshift-master/config.yml @@ -313,6 +313,7 @@ - name: Configure master instances hosts: oo_masters_to_config + any_errors_fatal: true serial: 1 vars: sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml index b43e9177e..150b76fc8 100644 --- a/roles/openshift_master/tasks/main.yml +++ b/roles/openshift_master/tasks/main.yml @@ -283,12 +283,13 @@ service: name={{ openshift.common.service_type }}-master-api enabled=yes state=started when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' register: start_result - notify: Verify API Server - set_fact: master_api_service_status_changed: "{{ start_result | changed }}" when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' +# A separate wait is required here for native HA since notifies will +# be resolved after all tasks in the role. - name: Wait for API to become available # Using curl here since the uri module requires python-httplib2 and # wait_for port doesn't provide health information. @@ -299,11 +300,7 @@ retries: 120 delay: 1 changed_when: false - -- fail: - msg: > - Unable to contact master API at {{ openshift.master.api_url }} - when: api_available_output.stdout.find("200 OK") == -1 + when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' and master_api_service_status_changed | bool - name: Start and enable master controller service: name={{ openshift.common.service_type }}-master-controllers enabled=yes state=started -- cgit v1.2.3