diff options
Diffstat (limited to 'playbooks/common/openshift-master')
7 files changed, 310 insertions, 36 deletions
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml index becd68dbe..6f86703d6 100644 --- a/playbooks/common/openshift-master/config.yml +++ b/playbooks/common/openshift-master/config.yml @@ -43,6 +43,7 @@ api_port: "{{ openshift_master_api_port | default(None) }}" api_url: "{{ openshift_master_api_url | default(None) }}" api_use_ssl: "{{ openshift_master_api_use_ssl | default(None) }}" + controllers_port: "{{ openshift_master_controllers_port | default(None) }}" public_api_url: "{{ openshift_master_public_api_url | default(None) }}" cluster_hostname: "{{ openshift_master_cluster_hostname | default(None) }}" cluster_public_hostname: "{{ openshift_master_cluster_public_hostname | default(None) }}" @@ -51,6 +52,7 @@ console_url: "{{ openshift_master_console_url | default(None) }}" console_use_ssl: "{{ openshift_master_console_use_ssl | default(None) }}" public_console_url: "{{ openshift_master_public_console_url | default(None) }}" + portal_net: "{{ openshift_master_portal_net | default(None) }}" - name: Check status of external etcd certificatees stat: path: "{{ openshift.common.config_base }}/master/{{ item }}" @@ -70,7 +72,7 @@ - name: Create temp directory for syncing certs hosts: localhost connection: local - sudo: false + become: no gather_facts: no tasks: - name: Create local temp directory for syncing certs @@ -84,6 +86,7 @@ etcd_generated_certs_dir: /etc/etcd/generated_certs etcd_needing_client_certs: "{{ hostvars | oo_select_keys(groups['oo_masters_to_config']) + | default([]) | oo_filter_list(filter_attr='etcd_client_certs_missing') }}" sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" roles: @@ -161,6 +164,11 @@ | list ) }}" master_cert_subdir: master-{{ openshift.common.hostname }} master_cert_config_dir: "{{ openshift.common.config_base }}/master" + - set_fact: + openshift_infra_nodes: "{{ hostvars | oo_select_keys(groups['nodes']) + | oo_nodes_with_label('region', 'infra') + | oo_collect('inventory_hostname') }}" + when: openshift_infra_nodes is not defined - name: Configure master certificates hosts: oo_first_master @@ -207,7 +215,7 @@ - name: Compute haproxy_backend_servers hosts: localhost connection: local - sudo: false + become: no gather_facts: no tasks: - set_fact: @@ -217,6 +225,7 @@ hosts: oo_lb_to_config vars: sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" + haproxy_frontend_port: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_port }}" haproxy_frontends: - name: atomic-openshift-api mode: tcp @@ -232,42 +241,41 @@ balance: source servers: "{{ hostvars.localhost.haproxy_backend_servers }}" roles: + - role: openshift_facts - role: haproxy when: groups.oo_masters_to_config | length > 1 -- name: Generate master session keys +- name: Check for cached session secrets hosts: oo_first_master + roles: + - role: openshift_facts + post_tasks: + - openshift_facts: + role: master + local_facts: + session_auth_secrets: "{{ openshift_master_session_auth_secrets | default(openshift.master.session_auth_secrets | default(None)) }}" + session_encryption_secrets: "{{ openshift_master_session_encryption_secrets | default(openshift.master.session_encryption_secrets | default(None)) }}" + +- name: Generate master session secrets + hosts: oo_first_master + vars: + g_session_secrets_present: "{{ (openshift.master.session_auth_secrets | default([]) and openshift.master.session_encryption_secrets | default([])) | length > 0 }}" + g_session_auth_secrets: "{{ [ 24 | oo_generate_secret ] }}" + g_session_encryption_secrets: "{{ [ 24 | oo_generate_secret ] }}" + roles: + - role: openshift_facts tasks: - - fail: - msg: "Both openshift_master_session_auth_secrets and openshift_master_session_encryption_secrets must be provided if either variable is set" - when: (openshift_master_session_auth_secrets is defined and openshift_master_session_encryption_secrets is not defined) or (openshift_master_session_encryption_secrets is defined and openshift_master_session_auth_secrets is not defined) - - fail: - msg: "openshift_master_session_auth_secrets and openshift_master_encryption_secrets must be equal length" - when: (openshift_master_session_auth_secrets is defined and openshift_master_session_encryption_secrets is defined) and (openshift_master_session_auth_secrets | length != openshift_master_session_encryption_secrets | length) - - name: Install OpenSSL package - action: "{{ansible_pkg_mgr}} pkg=openssl state=present" - - name: Generate session authentication key - command: /usr/bin/openssl rand -base64 24 - register: session_auth_output - with_sequence: count=1 - when: openshift_master_session_auth_secrets is undefined - - name: Generate session encryption key - command: /usr/bin/openssl rand -base64 24 - register: session_encryption_output - with_sequence: count=1 - when: openshift_master_session_encryption_secrets is undefined - - set_fact: - session_auth_secret: "{{ openshift_master_session_auth_secrets - | default(session_auth_output.results - | oo_collect(attribute='stdout') - | list) }}" - session_encryption_secret: "{{ openshift_master_session_encryption_secrets - | default(session_encryption_output.results - | oo_collect(attribute='stdout') - | list) }}" + - openshift_facts: + role: master + local_facts: + session_auth_secrets: "{{ g_session_auth_secrets }}" + session_encryption_secrets: "{{ g_session_encryption_secrets }}" + when: not g_session_secrets_present | bool - name: Parse named certificates hosts: localhost + connection: local + become: no vars: internal_hostnames: "{{ hostvars[groups.oo_first_master.0].openshift.common.internal_hostnames }}" named_certificates: "{{ hostvars[groups.oo_first_master.0].openshift_master_named_certificates | default([]) }}" @@ -313,13 +321,14 @@ - name: Configure master instances hosts: oo_masters_to_config + any_errors_fatal: true serial: 1 vars: sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" openshift_master_count: "{{ groups.oo_masters_to_config | length }}" - openshift_master_session_auth_secrets: "{{ hostvars[groups['oo_first_master'][0]]['session_auth_secret'] }}" - openshift_master_session_encryption_secrets: "{{ hostvars[groups['oo_first_master'][0]]['session_encryption_secret'] }}" + openshift_master_session_auth_secrets: "{{ hostvars[groups.oo_first_master.0].openshift.master.session_auth_secrets }}" + openshift_master_session_encryption_secrets: "{{ hostvars[groups.oo_first_master.0].openshift.master.session_encryption_secrets }}" pre_tasks: - name: Ensure certificate directory exists file: @@ -336,6 +345,8 @@ - role: nickhammond.logrotate - role: fluentd_master when: openshift.common.use_fluentd | bool + - role: nuage_master + when: openshift.common.use_nuage | bool post_tasks: - name: Create group for deployment type group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }} @@ -349,7 +360,8 @@ roles: - role: openshift_master_cluster when: openshift_master_ha | bool and openshift.master.cluster_method == "pacemaker" - - openshift_examples + - role: openshift_examples + when: openshift.common.install_examples | bool - role: openshift_cluster_metrics when: openshift.common.use_cluster_metrics | bool - role: openshift_manageiq @@ -361,7 +373,7 @@ cockpit_plugins: "{{ osm_cockpit_plugins | default(['cockpit-kubernetes']) }}" roles: - role: cockpit - when: ( deployment_type in ['atomic-enterprise','openshift-enterprise'] ) and + when: not openshift.common.is_atomic and ( deployment_type in ['atomic-enterprise','openshift-enterprise'] ) and (osm_use_cockpit | bool or osm_use_cockpit is undefined ) - name: Configure flannel @@ -382,7 +394,7 @@ - name: Delete temporary directory on localhost hosts: localhost connection: local - sudo: false + become: no gather_facts: no tasks: - file: name={{ g_master_mktemp.stdout }} state=absent @@ -399,7 +411,15 @@ - name: Create services hosts: oo_first_master + vars: + attach_registry_volume: "{{ groups.oo_nfs_to_config | length > 0 }}" + pre_tasks: + - set_fact: + nfs_host: "{{ groups.oo_nfs_to_config.0 }}" + registry_volume_path: "{{ hostvars[groups.oo_nfs_to_config.0].openshift.nfs.exports_dir + '/' + hostvars[groups.oo_nfs_to_config.0].openshift.nfs.registry_volume }}" + when: attach_registry_volume | bool roles: - role: openshift_router when: openshift.master.infra_nodes is defined - #- role: openshift_registry + - role: openshift_registry + when: openshift.master.infra_nodes is defined and attach_registry_volume | bool diff --git a/playbooks/common/openshift-master/restart.yml b/playbooks/common/openshift-master/restart.yml new file mode 100644 index 000000000..02449e40d --- /dev/null +++ b/playbooks/common/openshift-master/restart.yml @@ -0,0 +1,151 @@ +--- +- include: ../openshift-cluster/evaluate_groups.yml + +- name: Validate configuration for rolling restart + hosts: oo_masters_to_config + roles: + - openshift_facts + tasks: + - fail: + msg: "openshift_rolling_restart_mode must be set to either 'services' or 'system'" + when: openshift_rolling_restart_mode is defined and openshift_rolling_restart_mode not in ["services", "system"] + - openshift_facts: + role: "{{ item.role }}" + local_facts: "{{ item.local_facts }}" + with_items: + - role: common + local_facts: + rolling_restart_mode: "{{ openshift_rolling_restart_mode | default('services') }}" + - role: master + local_facts: + cluster_method: "{{ openshift_master_cluster_method | default(None) }}" + +# Creating a temp file on localhost, we then check each system that will +# be rebooted to see if that file exists, if so we know we're running +# ansible on a machine that needs a reboot, and we need to error out. +- name: Create temp file on localhost + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - local_action: command mktemp + register: mktemp + changed_when: false + +- name: Check if temp file exists on any masters + hosts: oo_masters_to_config + tasks: + - stat: path="{{ hostvars.localhost.mktemp.stdout }}" + register: exists + changed_when: false + +- name: Cleanup temp file on localhost + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - file: path="{{ hostvars.localhost.mktemp.stdout }}" state=absent + changed_when: false + +- name: Warn if restarting the system where ansible is running + hosts: oo_masters_to_config + tasks: + - pause: + prompt: > + Warning: Running playbook from a host that will be restarted! + Press CTRL+C and A to abort playbook execution. You may + continue by pressing ENTER but the playbook will stop + executing after this system has been restarted and services + must be verified manually. To only restart services, set + openshift_master_rolling_restart_mode=services in host + inventory and relaunch the playbook. + when: exists.stat.exists and openshift.common.rolling_restart_mode == 'system' + - set_fact: + current_host: "{{ exists.stat.exists }}" + when: openshift.common.rolling_restart_mode == 'system' + +- name: Determine which masters are currently active + hosts: oo_masters_to_config + any_errors_fatal: true + tasks: + - name: Check master service status + command: > + systemctl is-active {{ openshift.common.service_type }}-master + register: active_check_output + when: openshift.master.cluster_method | default(None) == 'pacemaker' + failed_when: false + changed_when: false + - set_fact: + is_active: "{{ active_check_output.stdout == 'active' }}" + when: openshift.master.cluster_method | default(None) == 'pacemaker' + +- name: Evaluate master groups + hosts: localhost + become: no + tasks: + - fail: + msg: > + Did not receive active status from any masters. Please verify pacemaker cluster. + when: "{{ hostvars[groups.oo_first_master.0].openshift.master.cluster_method | default(None) == 'pacemaker' and 'True' not in (hostvars + | oo_select_keys(groups['oo_masters_to_config']) + | oo_collect('is_active') + | list) }}" + - name: Evaluate oo_active_masters + add_host: + name: "{{ item }}" + groups: oo_active_masters + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_sudo: "{{ g_sudo | default(omit) }}" + with_items: "{{ groups.oo_masters_to_config | default([]) }}" + when: (hostvars[item]['is_active'] | default(false)) | bool + - name: Evaluate oo_current_masters + add_host: + name: "{{ item }}" + groups: oo_current_masters + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_sudo: "{{ g_sudo | default(omit) }}" + with_items: "{{ groups.oo_masters_to_config | default([]) }}" + when: (hostvars[item]['current_host'] | default(false)) | bool + +- name: Validate pacemaker cluster + hosts: oo_active_masters + tasks: + - name: Retrieve pcs status + command: pcs status + register: pcs_status_output + changed_when: false + - fail: + msg: > + Pacemaker cluster validation failed. One or more nodes are not online. + when: not (pcs_status_output.stdout | validate_pcs_cluster(groups.oo_masters_to_config)) | bool + +- name: Restart masters + hosts: oo_masters_to_config:!oo_active_masters:!oo_current_masters + vars: + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + serial: 1 + tasks: + - include: restart_hosts.yml + when: openshift.common.rolling_restart_mode == 'system' + - include: restart_services.yml + when: openshift.common.rolling_restart_mode == 'services' + +- name: Restart active masters + hosts: oo_active_masters + serial: 1 + tasks: + - include: restart_hosts_pacemaker.yml + when: openshift.common.rolling_restart_mode == 'system' + - include: restart_services_pacemaker.yml + when: openshift.common.rolling_restart_mode == 'services' + +- name: Restart current masters + hosts: oo_current_masters + serial: 1 + tasks: + - include: restart_hosts.yml + when: openshift.common.rolling_restart_mode == 'system' + - include: restart_services.yml + when: openshift.common.rolling_restart_mode == 'services' diff --git a/playbooks/common/openshift-master/restart_hosts.yml b/playbooks/common/openshift-master/restart_hosts.yml new file mode 100644 index 000000000..ff206f5a2 --- /dev/null +++ b/playbooks/common/openshift-master/restart_hosts.yml @@ -0,0 +1,39 @@ +- name: Restart master system + # https://github.com/ansible/ansible/issues/10616 + shell: sleep 2 && shutdown -r now "OpenShift Ansible master rolling restart" + async: 1 + poll: 0 + ignore_errors: true + become: yes +# When cluster_method != pacemaker we can ensure the api_port is +# available. +- name: Wait for master API to come back online + become: no + local_action: + module: wait_for + host="{{ inventory_hostname }}" + state=started + delay=10 + port="{{ openshift.master.api_port }}" + when: openshift.master.cluster_method != 'pacemaker' +- name: Wait for master to start + become: no + local_action: + module: wait_for + host="{{ inventory_hostname }}" + state=started + delay=10 + port=22 + when: openshift.master.cluster_method == 'pacemaker' +- name: Wait for master to become available + command: pcs status + register: pcs_status_output + until: pcs_status_output.stdout | validate_pcs_cluster([inventory_hostname]) | bool + retries: 15 + delay: 2 + changed_when: false + when: openshift.master.cluster_method == 'pacemaker' +- fail: + msg: > + Pacemaker cluster validation failed {{ inventory hostname }} is not online. + when: openshift.master.cluster_method == 'pacemaker' and not (pcs_status_output.stdout | validate_pcs_cluster([inventory_hostname])) | bool diff --git a/playbooks/common/openshift-master/restart_hosts_pacemaker.yml b/playbooks/common/openshift-master/restart_hosts_pacemaker.yml new file mode 100644 index 000000000..c9219e8de --- /dev/null +++ b/playbooks/common/openshift-master/restart_hosts_pacemaker.yml @@ -0,0 +1,25 @@ +- name: Fail over master resource + command: > + pcs resource move master {{ hostvars | oo_select_keys(groups['oo_masters_to_config']) | oo_collect('openshift.common.hostname', {'is_active': 'False'}) | list | first }} +- name: Wait for master API to come back online + become: no + local_action: + module: wait_for + host="{{ openshift.master.cluster_hostname }}" + state=started + delay=10 + port="{{ openshift.master.api_port }}" +- name: Restart master system + # https://github.com/ansible/ansible/issues/10616 + shell: sleep 2 && shutdown -r now "OpenShift Ansible master rolling restart" + async: 1 + poll: 0 + ignore_errors: true + become: yes +- name: Wait for master to start + become: no + local_action: + module: wait_for + host="{{ inventory_hostname }}" + state=started + delay=10 diff --git a/playbooks/common/openshift-master/restart_services.yml b/playbooks/common/openshift-master/restart_services.yml new file mode 100644 index 000000000..5e539cd65 --- /dev/null +++ b/playbooks/common/openshift-master/restart_services.yml @@ -0,0 +1,27 @@ +- name: Restart master + service: + name: "{{ openshift.common.service_type }}-master" + state: restarted + when: not openshift_master_ha | bool +- name: Restart master API + service: + name: "{{ openshift.common.service_type }}-master-api" + state: restarted + when: openshift_master_ha | bool and openshift.master.cluster_method != 'pacemaker' +- name: Wait for master API to come back online + become: no + local_action: + module: wait_for + host="{{ inventory_hostname }}" + state=started + delay=10 + port="{{ openshift.master.api_port }}" + when: openshift_master_ha | bool and openshift.master.cluster_method != 'pacemaker' +- name: Restart master controllers + service: + name: "{{ openshift.common.service_type }}-master-controllers" + state: restarted + # Ignore errrors since it is possible that type != simple for + # pre-3.1.1 installations. + ignore_errors: true + when: openshift_master_ha | bool and openshift.master.cluster_method != 'pacemaker' diff --git a/playbooks/common/openshift-master/restart_services_pacemaker.yml b/playbooks/common/openshift-master/restart_services_pacemaker.yml new file mode 100644 index 000000000..e738f3fb6 --- /dev/null +++ b/playbooks/common/openshift-master/restart_services_pacemaker.yml @@ -0,0 +1,10 @@ +- name: Restart master services + command: pcs resource restart master +- name: Wait for master API to come back online + become: no + local_action: + module: wait_for + host="{{ openshift.master.cluster_hostname }}" + state=started + delay=10 + port="{{ openshift.master.api_port }}" diff --git a/playbooks/common/openshift-master/service.yml b/playbooks/common/openshift-master/service.yml index 27e1e66f9..f60c5a2b5 100644 --- a/playbooks/common/openshift-master/service.yml +++ b/playbooks/common/openshift-master/service.yml @@ -2,6 +2,8 @@ - name: Populate g_service_masters host group if needed hosts: localhost gather_facts: no + connection: local + become: no tasks: - fail: msg="new_cluster_state is required to be injected in this playbook" when: new_cluster_state is not defined |