diff options
Diffstat (limited to 'playbooks/openshift-etcd')
27 files changed, 848 insertions, 0 deletions
diff --git a/playbooks/openshift-etcd/certificates.yml b/playbooks/openshift-etcd/certificates.yml new file mode 100644 index 000000000..c06e3b575 --- /dev/null +++ b/playbooks/openshift-etcd/certificates.yml @@ -0,0 +1,6 @@ +--- +- import_playbook: ../init/main.yml + +- import_playbook: private/ca.yml + +- import_playbook: private/certificates.yml diff --git a/playbooks/openshift-etcd/config.yml b/playbooks/openshift-etcd/config.yml new file mode 100644 index 000000000..c7814207c --- /dev/null +++ b/playbooks/openshift-etcd/config.yml @@ -0,0 +1,4 @@ +--- +- import_playbook: ../init/main.yml + +- import_playbook: private/config.yml diff --git a/playbooks/openshift-etcd/embedded2external.yml b/playbooks/openshift-etcd/embedded2external.yml new file mode 100644 index 000000000..7d090fa9b --- /dev/null +++ b/playbooks/openshift-etcd/embedded2external.yml @@ -0,0 +1,4 @@ +--- +- import_playbook: ../init/main.yml + +- import_playbook: private/embedded2external.yml diff --git a/playbooks/openshift-etcd/migrate.yml b/playbooks/openshift-etcd/migrate.yml new file mode 100644 index 000000000..0340b74a5 --- /dev/null +++ b/playbooks/openshift-etcd/migrate.yml @@ -0,0 +1,4 @@ +--- +- import_playbook: ../init/main.yml + +- import_playbook: private/migrate.yml diff --git a/playbooks/openshift-etcd/private/ca.yml b/playbooks/openshift-etcd/private/ca.yml new file mode 100644 index 000000000..72c39d546 --- /dev/null +++ b/playbooks/openshift-etcd/private/ca.yml @@ -0,0 +1,16 @@ +--- +- name: Generate new etcd CA + hosts: oo_first_etcd + roles: + - role: openshift_clock + - role: openshift_etcd_facts + tasks: + - import_role: + name: etcd + tasks_from: ca.yml + vars: + etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" + when: + - etcd_ca_setup | default(True) | bool diff --git a/playbooks/openshift-etcd/private/certificates-backup.yml b/playbooks/openshift-etcd/private/certificates-backup.yml new file mode 100644 index 000000000..2f9bef799 --- /dev/null +++ b/playbooks/openshift-etcd/private/certificates-backup.yml @@ -0,0 +1,19 @@ +--- +- name: Backup and remove generated etcd certificates + hosts: oo_first_etcd + any_errors_fatal: true + tasks: + - import_role: + name: etcd + tasks_from: backup_generated_certificates.yml + - import_role: + name: etcd + tasks_from: remove_generated_certificates.yml + +- name: Backup deployed etcd certificates + hosts: oo_etcd_to_config + any_errors_fatal: true + tasks: + - import_role: + name: etcd + tasks_from: backup_server_certificates.yml diff --git a/playbooks/openshift-etcd/private/certificates.yml b/playbooks/openshift-etcd/private/certificates.yml new file mode 100644 index 000000000..7c45938c1 --- /dev/null +++ b/playbooks/openshift-etcd/private/certificates.yml @@ -0,0 +1,4 @@ +--- +- import_playbook: server_certificates.yml + +- import_playbook: master_etcd_certificates.yml diff --git a/playbooks/openshift-etcd/private/config.yml b/playbooks/openshift-etcd/private/config.yml new file mode 100644 index 000000000..35407969e --- /dev/null +++ b/playbooks/openshift-etcd/private/config.yml @@ -0,0 +1,39 @@ +--- +- name: etcd Install Checkpoint Start + hosts: all + gather_facts: false + tasks: + - name: Set etcd install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_etcd: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" + +- import_playbook: ca.yml + +- import_playbook: certificates.yml + +- name: Configure etcd + hosts: oo_etcd_to_config + any_errors_fatal: true + roles: + - role: openshift_clock + - role: openshift_etcd + etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" + - role: nickhammond.logrotate + +- name: etcd Install Checkpoint End + hosts: all + gather_facts: false + tasks: + - name: Set etcd install 'Complete' + run_once: true + set_stats: + data: + installer_phase_etcd: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/openshift-etcd/private/embedded2external.yml b/playbooks/openshift-etcd/private/embedded2external.yml new file mode 100644 index 000000000..b71eaacd0 --- /dev/null +++ b/playbooks/openshift-etcd/private/embedded2external.yml @@ -0,0 +1,162 @@ +--- +- name: Pre-migrate checks + hosts: localhost + tasks: + # Check there is only one etcd host + - assert: + that: groups.oo_etcd_to_config | default([]) | length == 1 + msg: "[etcd] group must contain only one host" + # Check there is only one master + - assert: + that: groups.oo_masters_to_config | default([]) | length == 1 + msg: "[master] group must contain only one host" + +# 1. stop a master +- name: Prepare masters for etcd data migration + hosts: oo_first_master + roles: + - role: openshift_facts + tasks: + - name: Check the master API is ready + import_role: + name: openshift_master + tasks_from: check_master_api_is_ready.yml + - set_fact: + master_service: "{{ openshift_service_type + '-master' }}" + embedded_etcd_backup_suffix: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" + - debug: + msg: "master service name: {{ master_service }}" + - name: Stop master + service: + name: "{{ master_service }}" + state: stopped + # 2. backup embedded etcd + # Can't use with_items with import_role: https://github.com/ansible/ansible/issues/21285 + - import_role: + name: etcd + tasks_from: backup.yml + vars: + r_etcd_common_backup_tag: pre-migrate + r_etcd_common_embedded_etcd: "{{ true }}" + r_etcd_common_backup_sufix_name: "{{ embedded_etcd_backup_suffix }}" + + - import_role: + name: etcd + tasks_from: backup.archive.yml + vars: + r_etcd_common_backup_tag: pre-migrate + r_etcd_common_embedded_etcd: "{{ true }}" + r_etcd_common_backup_sufix_name: "{{ embedded_etcd_backup_suffix }}" + +# 3. deploy certificates (for etcd and master) +- import_playbook: ca.yml + +- import_playbook: server_certificates.yml + +- name: Backup etcd client certificates for master host + hosts: oo_first_master + tasks: + - import_role: + name: etcd + tasks_from: backup_master_etcd_certificates.yml + +- name: Redeploy master etcd certificates + import_playbook: master_etcd_certificates.yml + vars: + etcd_certificates_redeploy: "{{ true }}" + +# 4. deploy external etcd +- import_playbook: config.yml + +# 5. stop external etcd +- name: Cleanse etcd + hosts: oo_etcd_to_config[0] + gather_facts: no + pre_tasks: + - import_role: + name: etcd + tasks_from: disable_etcd.yml + - import_role: + name: etcd + tasks_from: clean_data.yml + +# 6. copy the embedded etcd backup to the external host +# TODO(jchaloup): if the etcd and first master are on the same host, just copy the directory +- name: Copy embedded etcd backup to the external host + hosts: localhost + tasks: + - name: Create local temp directory for syncing etcd backup + local_action: command mktemp -d /tmp/etcd_backup-XXXXXXX + register: g_etcd_client_mktemp + changed_when: False + become: no + + - import_role: + name: etcd + tasks_from: backup.fetch.yml + vars: + etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}" + r_etcd_common_backup_tag: pre-migrate + r_etcd_common_embedded_etcd: "{{ true }}" + r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}" + delegate_to: "{{ groups.oo_first_master[0] }}" + + - import_role: + name: etcd + tasks_from: backup.copy.yml + vars: + etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}" + r_etcd_common_backup_tag: pre-migrate + r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}" + delegate_to: "{{ groups.oo_etcd_to_config[0] }}" + + - debug: + msg: "etcd_backup_dest_directory: {{ g_etcd_client_mktemp.stdout }}" + + - name: Delete temporary directory + local_action: file path="{{ g_etcd_client_mktemp.stdout }}" state=absent + changed_when: False + become: no + +# 7. force new cluster from the backup +- name: Force new etcd cluster + hosts: oo_etcd_to_config[0] + tasks: + - import_role: + name: etcd + tasks_from: backup.unarchive.yml + vars: + r_etcd_common_backup_tag: pre-migrate + r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}" + + - import_role: + name: etcd + tasks_from: backup.force_new_cluster.yml + vars: + r_etcd_common_backup_tag: pre-migrate + r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}" + etcd_peer: "{{ openshift.common.ip }}" + etcd_url_scheme: "https" + etcd_peer_url_scheme: "https" + +# 8. re-configure master to use the external etcd +- name: Configure master to use external etcd + hosts: oo_first_master + tasks: + - import_role: + name: openshift_master + tasks_from: configure_external_etcd.yml + vars: + etcd_peer_url_scheme: "https" + etcd_ip: "{{ hostvars[groups.oo_etcd_to_config.0].openshift.common.ip }}" + etcd_peer_port: 2379 + + # 9. start the master + - name: Start master + service: + name: "{{ master_service }}" + state: started + register: service_status + until: service_status.state is defined and service_status.state == "started" + retries: 5 + delay: 10 diff --git a/playbooks/openshift-etcd/private/master_etcd_certificates.yml b/playbooks/openshift-etcd/private/master_etcd_certificates.yml new file mode 100644 index 000000000..d98470db2 --- /dev/null +++ b/playbooks/openshift-etcd/private/master_etcd_certificates.yml @@ -0,0 +1,13 @@ +--- +- name: Create etcd client certificates for master hosts + hosts: oo_masters_to_config + any_errors_fatal: true + roles: + - role: openshift_etcd_facts + - role: openshift_etcd_client_certificates + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}" + etcd_cert_config_dir: "{{ openshift.common.config_base }}/master" + etcd_cert_prefix: "master.etcd-" + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config diff --git a/playbooks/openshift-etcd/private/migrate.yml b/playbooks/openshift-etcd/private/migrate.yml new file mode 100644 index 000000000..0a2ac7f1a --- /dev/null +++ b/playbooks/openshift-etcd/private/migrate.yml @@ -0,0 +1,162 @@ +--- +- name: Check if the master has embedded etcd + hosts: localhost + connection: local + become: no + gather_facts: no + tags: + - always + tasks: + - fail: + msg: "Migration of an embedded etcd is not supported. Please, migrate the embedded etcd into an external etcd first." + when: + - groups.oo_etcd_to_config | default([]) | length == 0 + +- name: Run pre-checks + hosts: oo_etcd_to_migrate + tasks: + - import_role: + name: etcd + tasks_from: migrate.pre_check.yml + vars: + etcd_peer: "{{ ansible_default_ipv4.address }}" + +# TODO: This will be different for release-3.6 branch +- name: Prepare masters for etcd data migration + hosts: oo_masters_to_config + tasks: + - set_fact: + master_services: + - "{{ openshift_service_type + '-master-controllers' }}" + - "{{ openshift_service_type + '-master-api' }}" + - debug: + msg: "master service name: {{ master_services }}" + - name: Stop masters + service: + name: "{{ item }}" + state: stopped + with_items: "{{ master_services }}" + +- name: Backup v2 data + hosts: oo_etcd_to_migrate + gather_facts: no + roles: + - role: openshift_facts + post_tasks: + - import_role: + name: etcd + tasks_from: backup.yml + vars: + r_etcd_common_backup_tag: pre-migration + r_etcd_common_backup_sufix_name: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" + +- name: Gate on etcd backup + hosts: localhost + connection: local + become: no + tasks: + - set_fact: + etcd_backup_completed: "{{ hostvars + | lib_utils_oo_select_keys(groups.oo_etcd_to_migrate) + | lib_utils_oo_collect('inventory_hostname', {'r_etcd_common_backup_complete': true}) }}" + - set_fact: + etcd_backup_failed: "{{ groups.oo_etcd_to_migrate | difference(etcd_backup_completed) | list }}" + - fail: + msg: "Migration cannot continue. The following hosts did not complete etcd backup: {{ etcd_backup_failed | join(',') }}" + when: + - etcd_backup_failed | length > 0 + +- name: Stop etcd + hosts: oo_etcd_to_migrate + gather_facts: no + pre_tasks: + - import_role: + name: etcd + tasks_from: disable_etcd.yml + +- name: Migrate data on first etcd + hosts: oo_etcd_to_migrate[0] + gather_facts: no + tasks: + - import_role: + name: etcd + tasks_from: migrate.yml + vars: + etcd_peer: "{{ openshift.common.ip }}" + etcd_url_scheme: "https" + etcd_peer_url_scheme: "https" + +- name: Clean data stores on remaining etcd hosts + hosts: oo_etcd_to_migrate[1:] + gather_facts: no + tasks: + - import_role: + name: etcd + tasks_from: clean_data.yml + vars: + etcd_peer: "{{ openshift.common.ip }}" + etcd_url_scheme: "https" + etcd_peer_url_scheme: "https" + - name: Add etcd hosts + delegate_to: localhost + add_host: + name: "{{ item }}" + groups: oo_new_etcd_to_config + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_become: "{{ g_sudo | default(omit) }}" + with_items: "{{ groups.oo_etcd_to_migrate[1:] | default([]) }}" + changed_when: no + - name: Set success + set_fact: + r_etcd_migrate_success: true + +- import_playbook: scaleup.yml + +- name: Gate on etcd migration + hosts: oo_masters_to_config + gather_facts: no + tasks: + - set_fact: + etcd_migration_completed: "{{ hostvars + | lib_utils_oo_select_keys(groups.oo_etcd_to_migrate) + | lib_utils_oo_collect('inventory_hostname', {'r_etcd_migrate_success': true}) }}" + - set_fact: + etcd_migration_failed: "{{ groups.oo_etcd_to_migrate | difference(etcd_migration_completed) | list }}" + +- name: Add TTLs on the first master + hosts: oo_first_master[0] + tasks: + - import_role: + name: etcd + tasks_from: migrate.add_ttls.yml + vars: + etcd_peer: "{{ hostvars[groups.oo_etcd_to_migrate.0].openshift.common.ip }}" + etcd_url_scheme: "https" + etcd_peer_url_scheme: "https" + when: etcd_migration_failed | length == 0 + +- name: Configure masters if etcd data migration is succesfull + hosts: oo_masters_to_config + tasks: + - import_role: + name: etcd + tasks_from: migrate.configure_master.yml + when: etcd_migration_failed | length == 0 + - debug: + msg: "Skipping master re-configuration since migration failed." + when: + - etcd_migration_failed | length > 0 + - name: Start master services + service: + name: "{{ item }}" + state: started + register: service_status + # Sometimes the master-api, resp. master-controllers fails to start for the first time + until: service_status.state is defined and service_status.state == "started" + retries: 5 + delay: 10 + with_items: "{{ master_services[::-1] }}" + - fail: + msg: "Migration failed. The following hosts were not properly migrated: {{ etcd_migration_failed | join(',') }}" + when: + - etcd_migration_failed | length > 0 diff --git a/playbooks/openshift-etcd/private/redeploy-ca.yml b/playbooks/openshift-etcd/private/redeploy-ca.yml new file mode 100644 index 000000000..7b0d99255 --- /dev/null +++ b/playbooks/openshift-etcd/private/redeploy-ca.yml @@ -0,0 +1,101 @@ +--- +- name: Check cert expirys + hosts: oo_etcd_to_config:oo_masters_to_config + vars: + openshift_certificate_expiry_show_all: yes + roles: + # Sets 'check_results' per host which contains health status for + # etcd, master and node certificates. We will use 'check_results' + # to determine if any certificates were expired prior to running + # this playbook. Service restarts will be skipped if any + # certificates were previously expired. + - role: openshift_certificate_expiry + +- name: Backup existing etcd CA certificate directories + hosts: oo_etcd_to_config + tasks: + - import_role: + name: etcd + tasks_from: backup_ca_certificates.yml + - import_role: + name: etcd + tasks_from: remove_ca_certificates.yml + +- import_playbook: ca.yml + +- name: Create temp directory for syncing certs + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - name: Create local temp directory for syncing certs + local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX + register: g_etcd_mktemp + changed_when: false + +- name: Distribute etcd CA to etcd hosts + hosts: oo_etcd_to_config + tasks: + - import_role: + name: etcd + tasks_from: distribute_ca.yml + vars: + etcd_sync_cert_dir: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}" + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + +- import_playbook: restart.yml + # Do not restart etcd when etcd certificates were previously expired. + when: ('expired' not in (hostvars + | lib_utils_oo_select_keys(groups['etcd']) + | lib_utils_oo_collect('check_results.check_results.etcd') + | lib_utils_oo_collect('health'))) + +- name: Retrieve etcd CA certificate + hosts: oo_first_etcd + tasks: + - import_role: + name: etcd + tasks_from: retrieve_ca_certificates.yml + vars: + etcd_sync_cert_dir: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}" + +- name: Distribute etcd CA to masters + hosts: oo_masters_to_config + vars: + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + tasks: + - name: Deploy etcd CA + copy: + src: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}/ca.crt" + dest: "{{ openshift.common.config_base }}/master/master.etcd-ca.crt" + when: groups.oo_etcd_to_config | default([]) | length > 0 + +- name: Delete temporary directory on localhost + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - file: + name: "{{ g_etcd_mktemp.stdout }}" + state: absent + changed_when: false + +- import_playbook: ../../openshift-master/private/restart.yml + # Do not restart masters when master or etcd certificates were previously expired. + when: + # masters + - ('expired' not in hostvars + | lib_utils_oo_select_keys(groups['oo_masters_to_config']) + | lib_utils_oo_collect('check_results.check_results.ocp_certs') + | lib_utils_oo_collect('health', {'path':hostvars[groups.oo_first_master.0].openshift.common.config_base ~ "/master/master.server.crt"})) + - ('expired' not in hostvars + | lib_utils_oo_select_keys(groups['oo_masters_to_config']) + | lib_utils_oo_collect('check_results.check_results.ocp_certs') + | lib_utils_oo_collect('health', {'path':hostvars[groups.oo_first_master.0].openshift.common.config_base ~ "/master/ca-bundle.crt"})) + # etcd + - ('expired' not in (hostvars + | lib_utils_oo_select_keys(groups['etcd']) + | lib_utils_oo_collect('check_results.check_results.etcd') + | lib_utils_oo_collect('health'))) diff --git a/playbooks/openshift-etcd/private/redeploy-certificates.yml b/playbooks/openshift-etcd/private/redeploy-certificates.yml new file mode 100644 index 000000000..1c8eb27ac --- /dev/null +++ b/playbooks/openshift-etcd/private/redeploy-certificates.yml @@ -0,0 +1,18 @@ +--- +- name: Check cert expirys + hosts: oo_etcd_to_config + vars: + openshift_certificate_expiry_show_all: yes + roles: + # Sets 'check_results' per host which contains health status for + # etcd, master and node certificates. We will use 'check_results' + # to determine if any certificates were expired prior to running + # this playbook. Service restarts will be skipped if any + # certificates were previously expired. + - role: openshift_certificate_expiry + +- import_playbook: certificates-backup.yml + +- import_playbook: certificates.yml + vars: + etcd_certificates_redeploy: true diff --git a/playbooks/openshift-etcd/private/restart.yml b/playbooks/openshift-etcd/private/restart.yml new file mode 100644 index 000000000..a2a53651b --- /dev/null +++ b/playbooks/openshift-etcd/private/restart.yml @@ -0,0 +1,19 @@ +--- +- name: Restart etcd + hosts: oo_etcd_to_config + serial: 1 + tasks: + - import_role: + name: etcd + tasks_from: restart.yml + when: + - not g_etcd_certificates_expired | default(false) | bool + +- name: Restart etcd + hosts: oo_etcd_to_config + tasks: + - import_role: + name: etcd + tasks_from: restart.yml + when: + - g_etcd_certificates_expired | default(false) | bool diff --git a/playbooks/openshift-etcd/private/roles b/playbooks/openshift-etcd/private/roles new file mode 120000 index 000000000..20c4c58cf --- /dev/null +++ b/playbooks/openshift-etcd/private/roles @@ -0,0 +1 @@ +../../../roles
\ No newline at end of file diff --git a/playbooks/openshift-etcd/private/scaleup.yml b/playbooks/openshift-etcd/private/scaleup.yml new file mode 100644 index 000000000..8a9811a25 --- /dev/null +++ b/playbooks/openshift-etcd/private/scaleup.yml @@ -0,0 +1,81 @@ +--- +- name: Gather facts + hosts: oo_etcd_to_config:oo_new_etcd_to_config + roles: + - openshift_etcd_facts + post_tasks: + - set_fact: + etcd_hostname: "{{ etcd_hostname }}" + etcd_ip: "{{ etcd_ip }}" + +- name: Configure etcd + hosts: oo_new_etcd_to_config + serial: 1 + any_errors_fatal: true + vars: + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + pre_tasks: + - name: Add new etcd members to cluster + command: > + /usr/bin/etcdctl --cert-file {{ etcd_peer_cert_file }} + --key-file {{ etcd_peer_key_file }} + --ca-file {{ etcd_peer_ca_file }} + -C {{ etcd_peer_url_scheme }}://{{ hostvars[etcd_ca_host].etcd_ip }}:{{ etcd_client_port }} + member add {{ etcd_hostname }} {{ etcd_peer_url_scheme }}://{{ etcd_ip }}:{{ etcd_peer_port }} + delegate_to: "{{ etcd_ca_host }}" + failed_when: + - etcd_add_check.rc == 1 + - ("peerURL exists" not in etcd_add_check.stderr) + register: etcd_add_check + retries: 3 + delay: 10 + until: etcd_add_check.rc == 0 + - import_role: + name: etcd + tasks_from: server_certificates.yml + vars: + etcd_peers: "{{ groups.oo_new_etcd_to_config | default([], true) }}" + etcd_certificates_etcd_hosts: "{{ groups.oo_new_etcd_to_config | default([], true) }}" + roles: + - role: os_firewall + when: etcd_add_check.rc == 0 + - role: openshift_etcd + when: etcd_add_check.rc == 0 + etcd_peers: "{{ groups.oo_etcd_to_config | union(groups.oo_new_etcd_to_config)| default([], true) }}" + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" + etcd_initial_cluster_state: "existing" + etcd_initial_cluster: "{{ etcd_add_check.stdout_lines[3] | regex_replace('ETCD_INITIAL_CLUSTER=','') | regex_replace('\"','') }}" + etcd_ca_setup: False + - role: nickhammond.logrotate + when: etcd_add_check.rc == 0 + post_tasks: + - name: Verify cluster is stable + command: > + /usr/bin/etcdctl --cert-file {{ etcd_peer_cert_file }} + --key-file {{ etcd_peer_key_file }} + --ca-file {{ etcd_peer_ca_file }} + -C {{ etcd_peer_url_scheme }}://{{ hostvars[etcd_ca_host].etcd_hostname }}:{{ etcd_client_port }} + cluster-health + register: scaleup_health + retries: 3 + delay: 30 + until: scaleup_health.rc == 0 + +- name: Update master etcd client urls + hosts: oo_masters_to_config + serial: 1 + vars: + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + openshift_master_etcd_hosts: "{{ hostvars + | lib_utils_oo_select_keys(groups['oo_etcd_to_config'] | union(groups['oo_new_etcd_to_config'] | default([]) )) + | lib_utils_oo_collect('openshift.common.hostname') + | default(none, true) }}" + openshift_master_etcd_port: "{{ (etcd_client_port | default('2379')) if (groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config) else none }}" + roles: + - role: openshift_master_facts + post_tasks: + - import_role: + name: openshift_master + tasks_from: update_etcd_client_urls.yml diff --git a/playbooks/openshift-etcd/private/server_certificates.yml b/playbooks/openshift-etcd/private/server_certificates.yml new file mode 100644 index 000000000..ebcf4a5ff --- /dev/null +++ b/playbooks/openshift-etcd/private/server_certificates.yml @@ -0,0 +1,14 @@ +--- +- name: Create etcd server certificates for etcd hosts + hosts: oo_etcd_to_config + any_errors_fatal: true + roles: + - role: openshift_etcd_facts + post_tasks: + - import_role: + name: etcd + tasks_from: server_certificates.yml + vars: + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" + etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" diff --git a/playbooks/openshift-etcd/private/upgrade_backup.yml b/playbooks/openshift-etcd/private/upgrade_backup.yml new file mode 100644 index 000000000..97b6edba5 --- /dev/null +++ b/playbooks/openshift-etcd/private/upgrade_backup.yml @@ -0,0 +1,27 @@ +--- +- name: Backup etcd + hosts: oo_etcd_hosts_to_backup + roles: + - role: openshift_etcd_facts + post_tasks: + - import_role: + name: etcd + tasks_from: backup.yml + vars: + r_etcd_common_backup_tag: "{{ etcd_backup_tag }}" + r_etcd_common_backup_sufix_name: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" + +- name: Gate on etcd backup + hosts: localhost + connection: local + become: no + tasks: + - set_fact: + etcd_backup_completed: "{{ hostvars + | lib_utils_oo_select_keys(groups.oo_etcd_hosts_to_backup) + | lib_utils_oo_collect('inventory_hostname', {'r_etcd_common_backup_complete': true}) }}" + - set_fact: + etcd_backup_failed: "{{ groups.oo_etcd_hosts_to_backup | difference(etcd_backup_completed) | list }}" + - fail: + msg: "Upgrade cannot continue. The following hosts did not complete etcd backup: {{ etcd_backup_failed | join(',') }}" + when: etcd_backup_failed | length > 0 diff --git a/playbooks/openshift-etcd/private/upgrade_image_members.yml b/playbooks/openshift-etcd/private/upgrade_image_members.yml new file mode 100644 index 000000000..f9e50e748 --- /dev/null +++ b/playbooks/openshift-etcd/private/upgrade_image_members.yml @@ -0,0 +1,17 @@ +--- +# INPUT etcd_upgrade_version +# INPUT etcd_container_version +# INPUT openshift_is_containerized +- name: Upgrade containerized hosts to {{ etcd_upgrade_version }} + hosts: oo_etcd_hosts_to_upgrade + serial: 1 + tasks: + - import_role: + name: etcd + tasks_from: upgrade_image.yml + vars: + r_etcd_upgrade_version: "{{ etcd_upgrade_version }}" + etcd_peer: "{{ openshift.common.hostname }}" + when: + - etcd_container_version | default('99') is version_compare(etcd_upgrade_version,'<') + - openshift_is_containerized | bool diff --git a/playbooks/openshift-etcd/private/upgrade_main.yml b/playbooks/openshift-etcd/private/upgrade_main.yml new file mode 100644 index 000000000..8997680f9 --- /dev/null +++ b/playbooks/openshift-etcd/private/upgrade_main.yml @@ -0,0 +1,29 @@ +--- +# For 1.4/3.4 we want to upgrade everyone to etcd-3.0. etcd docs say to +# upgrade from 2.0.x to 2.1.x to 2.2.x to 2.3.x to 3.0.x. While this is a tedius +# task for RHEL and CENTOS it's simply not possible in Fedora unless you've +# mirrored packages on your own because only the GA and latest versions are +# available in the repos. So for Fedora we'll simply skip this, sorry. + +- name: Backup etcd before upgrading anything + import_playbook: upgrade_backup.yml + vars: + etcd_backup_tag: "pre-upgrade-" + when: openshift_etcd_backup | default(true) | bool + +- name: Drop etcdctl profiles + hosts: oo_etcd_hosts_to_upgrade + tasks: + - import_role: + name: etcd + tasks_from: drop_etcdctl.yml + +- name: Perform etcd upgrade + import_playbook: upgrade_step.yml + when: openshift_etcd_upgrade | default(true) | bool + +- name: Backup etcd + import_playbook: upgrade_backup.yml + vars: + etcd_backup_tag: "post-3.0-" + when: openshift_etcd_backup | default(true) | bool diff --git a/playbooks/openshift-etcd/private/upgrade_rpm_members.yml b/playbooks/openshift-etcd/private/upgrade_rpm_members.yml new file mode 100644 index 000000000..e78cc5826 --- /dev/null +++ b/playbooks/openshift-etcd/private/upgrade_rpm_members.yml @@ -0,0 +1,18 @@ +--- +# INPUT etcd_upgrade_version +# INPUT etcd_rpm_version +# INPUT openshift_is_containerized +- name: Upgrade to {{ etcd_upgrade_version }} + hosts: oo_etcd_hosts_to_upgrade + serial: 1 + tasks: + - import_role: + name: etcd + tasks_from: upgrade_rpm.yml + vars: + r_etcd_upgrade_version: "{{ etcd_upgrade_version }}" + etcd_peer: "{{ openshift.common.hostname }}" + when: + - etcd_rpm_version.stdout | default('99') is version_compare(etcd_upgrade_version, '<') + - ansible_distribution == 'RedHat' + - not openshift_is_containerized | bool diff --git a/playbooks/openshift-etcd/private/upgrade_step.yml b/playbooks/openshift-etcd/private/upgrade_step.yml new file mode 100644 index 000000000..6aec838d4 --- /dev/null +++ b/playbooks/openshift-etcd/private/upgrade_step.yml @@ -0,0 +1,64 @@ +--- +- name: Determine etcd version + hosts: oo_etcd_hosts_to_upgrade + tasks: + - import_role: + name: etcd + tasks_from: version_detect.yml + +- import_playbook: upgrade_rpm_members.yml + vars: + etcd_upgrade_version: '2.1' + +- import_playbook: upgrade_rpm_members.yml + vars: + etcd_upgrade_version: '2.2' + +- import_playbook: upgrade_image_members.yml + vars: + etcd_upgrade_version: '2.2.5' + +- import_playbook: upgrade_rpm_members.yml + vars: + etcd_upgrade_version: '2.3' + +- import_playbook: upgrade_image_members.yml + vars: + etcd_upgrade_version: '2.3.7' + +- import_playbook: upgrade_rpm_members.yml + vars: + etcd_upgrade_version: '3.0' + +- import_playbook: upgrade_image_members.yml + vars: + etcd_upgrade_version: '3.0.15' + +- import_playbook: upgrade_rpm_members.yml + vars: + etcd_upgrade_version: '3.1' + +- import_playbook: upgrade_image_members.yml + vars: + etcd_upgrade_version: '3.1.3' + +- import_playbook: upgrade_rpm_members.yml + vars: + etcd_upgrade_version: '3.2' + +- import_playbook: upgrade_image_members.yml + vars: + etcd_upgrade_version: '3.2.7' + +- name: Upgrade fedora to latest + hosts: oo_etcd_hosts_to_upgrade + serial: 1 + tasks: + - import_role: + name: etcd + tasks_from: upgrade_image.yml + vars: + etcd_peer: "{{ openshift.common.hostname }}" + when: + - ansible_distribution == 'Fedora' + - not openshift_is_containerized | bool diff --git a/playbooks/openshift-etcd/redeploy-ca.yml b/playbooks/openshift-etcd/redeploy-ca.yml new file mode 100644 index 000000000..769d694ba --- /dev/null +++ b/playbooks/openshift-etcd/redeploy-ca.yml @@ -0,0 +1,4 @@ +--- +- import_playbook: ../init/main.yml + +- import_playbook: private/redeploy-ca.yml diff --git a/playbooks/openshift-etcd/redeploy-certificates.yml b/playbooks/openshift-etcd/redeploy-certificates.yml new file mode 100644 index 000000000..8ea1994f7 --- /dev/null +++ b/playbooks/openshift-etcd/redeploy-certificates.yml @@ -0,0 +1,10 @@ +--- +- import_playbook: ../init/main.yml + +- import_playbook: private/redeploy-certificates.yml + +- import_playbook: private/restart.yml + vars: + g_etcd_certificates_expired: "{{ ('expired' in (hostvars | lib_utils_oo_select_keys(groups['etcd']) | lib_utils_oo_collect('check_results.check_results.etcd') | lib_utils_oo_collect('health'))) | bool }}" + +- import_playbook: ../openshift-master/private/restart.yml diff --git a/playbooks/openshift-etcd/restart.yml b/playbooks/openshift-etcd/restart.yml new file mode 100644 index 000000000..041c1384d --- /dev/null +++ b/playbooks/openshift-etcd/restart.yml @@ -0,0 +1,4 @@ +--- +- import_playbook: ../init/main.yml + +- import_playbook: private/restart.yml diff --git a/playbooks/openshift-etcd/scaleup.yml b/playbooks/openshift-etcd/scaleup.yml new file mode 100644 index 000000000..7e9ab6834 --- /dev/null +++ b/playbooks/openshift-etcd/scaleup.yml @@ -0,0 +1,4 @@ +--- +- import_playbook: ../init/main.yml + +- import_playbook: private/scaleup.yml diff --git a/playbooks/openshift-etcd/upgrade.yml b/playbooks/openshift-etcd/upgrade.yml new file mode 100644 index 000000000..ccc797527 --- /dev/null +++ b/playbooks/openshift-etcd/upgrade.yml @@ -0,0 +1,4 @@ +--- +- import_playbook: ../init/evaluate_groups.yml + +- import_playbook: private/upgrade_main.yml |