diff options
Diffstat (limited to 'roles/etcd/tasks')
22 files changed, 173 insertions, 81 deletions
diff --git a/roles/etcd/tasks/auxiliary/clean_data.yml b/roles/etcd/tasks/auxiliary/clean_data.yml index 95a0e7c0a..1ed2db5bc 100644 --- a/roles/etcd/tasks/auxiliary/clean_data.yml +++ b/roles/etcd/tasks/auxiliary/clean_data.yml @@ -1,5 +1,5 @@ --- - name: Remove member data file: - path: /var/lib/etcd/member + path: "{{ etcd_data_dir }}/member" state: absent diff --git a/roles/etcd/tasks/auxiliary/disable_etcd.yml b/roles/etcd/tasks/auxiliary/disable_etcd.yml new file mode 100644 index 000000000..7c6d0409d --- /dev/null +++ b/roles/etcd/tasks/auxiliary/disable_etcd.yml @@ -0,0 +1,5 @@ +--- +- name: Disable etcd members + service: + name: "{{ etcd_service }}" + state: stopped diff --git a/roles/etcd/tasks/auxiliary/force_new_cluster.yml b/roles/etcd/tasks/auxiliary/force_new_cluster.yml new file mode 100644 index 000000000..ae8a36130 --- /dev/null +++ b/roles/etcd/tasks/auxiliary/force_new_cluster.yml @@ -0,0 +1,31 @@ +--- +- name: Set ETCD_FORCE_NEW_CLUSTER=true on first etcd host + lineinfile: + line: "ETCD_FORCE_NEW_CLUSTER=true" + dest: /etc/etcd/etcd.conf + backup: true + +- name: Start etcd + systemd: + name: "{{ etcd_service }}" + state: started + +- name: Wait for cluster to become healthy after bringing up first member + command: > + etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoint https://{{ etcd_peer }}:{{ etcd_client_port }} cluster-health + register: l_etcd_migrate_health + until: l_etcd_migrate_health.rc == 0 + retries: 3 + delay: 30 + +- name: Unset ETCD_FORCE_NEW_CLUSTER=true on first etcd host + lineinfile: + line: "ETCD_FORCE_NEW_CLUSTER=true" + dest: /etc/etcd/etcd.conf + state: absent + backup: true + +- name: Restart first etcd host + systemd: + name: "{{ etcd_service }}" + state: restarted diff --git a/roles/etcd/tasks/backup.archive.yml b/roles/etcd/tasks/backup.archive.yml new file mode 100644 index 000000000..6daa6dc51 --- /dev/null +++ b/roles/etcd/tasks/backup.archive.yml @@ -0,0 +1,3 @@ +--- +- include: backup/vars.yml +- include: backup/archive.yml diff --git a/roles/etcd/tasks/backup.copy.yml b/roles/etcd/tasks/backup.copy.yml new file mode 100644 index 000000000..cc540cbca --- /dev/null +++ b/roles/etcd/tasks/backup.copy.yml @@ -0,0 +1,3 @@ +--- +- include: backup/vars.yml +- include: backup/copy.yml diff --git a/roles/etcd/tasks/backup.fetch.yml b/roles/etcd/tasks/backup.fetch.yml new file mode 100644 index 000000000..26ec15043 --- /dev/null +++ b/roles/etcd/tasks/backup.fetch.yml @@ -0,0 +1,3 @@ +--- +- include: backup/vars.yml +- include: backup/fetch.yml diff --git a/roles/etcd/tasks/backup.force_new_cluster.yml b/roles/etcd/tasks/backup.force_new_cluster.yml new file mode 100644 index 000000000..d2e866416 --- /dev/null +++ b/roles/etcd/tasks/backup.force_new_cluster.yml @@ -0,0 +1,12 @@ +--- +- include: backup/vars.yml + +- name: Move content of etcd backup under the etcd data directory + command: > + mv "{{ l_etcd_backup_dir }}/member" "{{ etcd_data_dir }}" + +- name: Set etcd group for the etcd data directory + command: > + chown -R etcd:etcd "{{ etcd_data_dir }}" + +- include: auxiliary/force_new_cluster.yml diff --git a/roles/etcd/tasks/backup.unarchive.yml b/roles/etcd/tasks/backup.unarchive.yml new file mode 100644 index 000000000..77a637360 --- /dev/null +++ b/roles/etcd/tasks/backup.unarchive.yml @@ -0,0 +1,3 @@ +--- +- include: backup/vars.yml +- include: backup/unarchive.yml diff --git a/roles/etcd/tasks/backup/archive.yml b/roles/etcd/tasks/backup/archive.yml new file mode 100644 index 000000000..f6aa68a6e --- /dev/null +++ b/roles/etcd/tasks/backup/archive.yml @@ -0,0 +1,5 @@ +--- +- name: Archive backup + archive: + path: "{{ l_etcd_backup_dir }}" + dest: "{{ l_etcd_backup_dir }}.tgz" diff --git a/roles/etcd/tasks/backup/backup.yml b/roles/etcd/tasks/backup/backup.yml index 42d27c081..ca0d29155 100644 --- a/roles/etcd/tasks/backup/backup.yml +++ b/roles/etcd/tasks/backup/backup.yml @@ -1,25 +1,9 @@ --- -# set the etcd backup directory name here in case the tag or sufix consists of dynamic value that changes over time -# e.g. openshift-backup-{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }} value will change every second so if the date changes -# right after setting l_etcd_incontainer_backup_dir and before l_etcd_backup_dir facts, the backup directory name is different -- set_fact: - l_backup_dir_name: "openshift-backup-{{ r_etcd_common_backup_tag }}{{ r_etcd_common_backup_sufix_name }}" - -- set_fact: - l_etcd_data_dir: "{{ etcd_data_dir }}{{ '/etcd.etcd' if r_etcd_common_etcd_runtime == 'runc' else '' }}" - -- set_fact: - l_etcd_incontainer_data_dir: "{{ etcd_data_dir }}" - -- set_fact: - l_etcd_incontainer_backup_dir: "{{ l_etcd_incontainer_data_dir }}/{{ l_backup_dir_name }}" - -- set_fact: - l_etcd_backup_dir: "{{ l_etcd_data_dir }}/{{ l_backup_dir_name }}" +- include: vars.yml # TODO: replace shell module with command and update later checks - name: Check available disk space for etcd backup - shell: df --output=avail -k {{ l_etcd_data_dir }} | tail -n 1 + shell: df --output=avail -k {{ etcd_data_dir }} | tail -n 1 register: l_avail_disk # AUDIT:changed_when: `false` because we are only inspecting # state, not manipulating anything @@ -27,7 +11,7 @@ # TODO: replace shell module with command and update later checks - name: Check current etcd disk usage - shell: du --exclude='*openshift-backup*' -k {{ l_etcd_data_dir }} | tail -n 1 | cut -f1 + shell: du --exclude='*openshift-backup*' -k {{ etcd_data_dir }} | tail -n 1 | cut -f1 register: l_etcd_disk_usage # AUDIT:changed_when: `false` because we are only inspecting # state, not manipulating anything @@ -60,17 +44,17 @@ - r_etcd_common_embedded_etcd | bool - not l_ostree_booted.stat.exists | bool -- name: Check selinux label of '{{ l_etcd_data_dir }}' +- name: Check selinux label of '{{ etcd_data_dir }}' command: > - stat -c '%C' {{ l_etcd_data_dir }} + stat -c '%C' {{ etcd_data_dir }} register: l_etcd_selinux_labels - debug: msg: "{{ l_etcd_selinux_labels }}" -- name: Make sure the '{{ l_etcd_data_dir }}' has the proper label +- name: Make sure the '{{ etcd_data_dir }}' has the proper label command: > - chcon -t svirt_sandbox_file_t "{{ l_etcd_data_dir }}" + chcon -t svirt_sandbox_file_t "{{ etcd_data_dir }}" when: - l_etcd_selinux_labels.rc == 0 - "'svirt_sandbox_file_t' not in l_etcd_selinux_labels.stdout" @@ -84,12 +68,12 @@ # https://github.com/openshift/openshift-docs/commit/b38042de02d9780842dce95cfa0ef45d53b58bc6 - name: Check for v3 data store stat: - path: "{{ l_etcd_data_dir }}/member/snap/db" + path: "{{ etcd_data_dir }}/member/snap/db" register: l_v3_db - name: Copy etcd v3 data store command: > - cp -a {{ l_etcd_data_dir }}/member/snap/db + cp -a {{ etcd_data_dir }}/member/snap/db {{ l_etcd_backup_dir }}/member/snap/ when: l_v3_db.stat.exists diff --git a/roles/etcd/tasks/backup/copy.yml b/roles/etcd/tasks/backup/copy.yml new file mode 100644 index 000000000..967e5ee66 --- /dev/null +++ b/roles/etcd/tasks/backup/copy.yml @@ -0,0 +1,5 @@ +--- +- name: Copy etcd backup + copy: + src: "{{ etcd_backup_sync_directory }}/{{ l_backup_dir_name }}.tgz" + dest: "{{ etcd_data_dir }}" diff --git a/roles/etcd/tasks/backup/fetch.yml b/roles/etcd/tasks/backup/fetch.yml new file mode 100644 index 000000000..610ce1960 --- /dev/null +++ b/roles/etcd/tasks/backup/fetch.yml @@ -0,0 +1,8 @@ +--- +- name: Fetch etcd backup + fetch: + src: "{{ l_etcd_backup_dir }}.tgz" + dest: "{{ etcd_backup_sync_directory }}/" + flat: yes + fail_on_missing: yes + validate_checksum: yes diff --git a/roles/etcd/tasks/backup/unarchive.yml b/roles/etcd/tasks/backup/unarchive.yml new file mode 100644 index 000000000..a85f533c2 --- /dev/null +++ b/roles/etcd/tasks/backup/unarchive.yml @@ -0,0 +1,14 @@ +--- +- shell: ls /var/lib/etcd + register: output + +- debug: + msg: "output: {{ output }}" + +- name: Unarchive backup + # can't use unarchive https://github.com/ansible/ansible/issues/30821 + # unarchive: + # src: "{{ l_etcd_backup_dir }}.tgz" + # dest: "{{ l_etcd_backup_dir }}" + command: > + tar -xf "{{ l_etcd_backup_dir }}.tgz" -C "{{ etcd_data_dir }}" diff --git a/roles/etcd/tasks/backup/vars.yml b/roles/etcd/tasks/backup/vars.yml new file mode 100644 index 000000000..3ffa641b3 --- /dev/null +++ b/roles/etcd/tasks/backup/vars.yml @@ -0,0 +1,15 @@ +--- +# set the etcd backup directory name here in case the tag or sufix consists of dynamic value that changes over time +# e.g. openshift-backup-{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }} value will change every second so if the date changes +# right after setting l_etcd_incontainer_backup_dir and before l_etcd_backup_dir facts, the backup directory name is different +- set_fact: + l_backup_dir_name: "openshift-backup-{{ r_etcd_common_backup_tag }}{{ r_etcd_common_backup_sufix_name }}" + +- set_fact: + l_etcd_incontainer_data_dir: "{{ etcd_data_dir }}" + +- set_fact: + l_etcd_incontainer_backup_dir: "{{ l_etcd_incontainer_data_dir }}/{{ l_backup_dir_name }}" + +- set_fact: + l_etcd_backup_dir: "{{ etcd_data_dir }}/{{ l_backup_dir_name }}" diff --git a/roles/etcd/tasks/backup_master_etcd_certificates.yml b/roles/etcd/tasks/backup_master_etcd_certificates.yml new file mode 100644 index 000000000..129e1831c --- /dev/null +++ b/roles/etcd/tasks/backup_master_etcd_certificates.yml @@ -0,0 +1,2 @@ +--- +- include: certificates/backup_master_etcd_certificates.yml diff --git a/roles/etcd/tasks/certificates/backup_master_etcd_certificates.yml b/roles/etcd/tasks/certificates/backup_master_etcd_certificates.yml new file mode 100644 index 000000000..e65b3e5a2 --- /dev/null +++ b/roles/etcd/tasks/certificates/backup_master_etcd_certificates.yml @@ -0,0 +1,7 @@ +--- +- name: Backup master etcd certificates + shell: > + tar -czvf /etc/origin/master/master-etcd-certificate-backup-{{ ansible_date_time.epoch }}.tgz + /etc/origin/master/master.etcd-* + args: + warn: no diff --git a/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml b/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml index 26492fb3c..00b8f4a0b 100644 --- a/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml +++ b/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml @@ -12,9 +12,6 @@ - "{{ etcd_cert_config_dir }}/{{ etcd_cert_prefix }}server.crt" - "{{ etcd_cert_config_dir }}/{{ etcd_cert_prefix }}peer.crt" - "{{ etcd_cert_config_dir }}/{{ etcd_cert_prefix }}ca.crt" - - "{{ etcd_system_container_cert_config_dir }}/{{ etcd_cert_prefix }}server.crt" - - "{{ etcd_system_container_cert_config_dir }}/{{ etcd_cert_prefix }}peer.crt" - - "{{ etcd_system_container_cert_config_dir }}/{{ etcd_cert_prefix }}ca.crt" register: g_etcd_server_cert_stat_result when: not etcd_certificates_redeploy | default(false) | bool @@ -141,7 +138,6 @@ state: directory with_items: - "{{ etcd_cert_config_dir }}" - - "{{ etcd_system_container_cert_config_dir }}" when: etcd_server_certs_missing | bool - name: Unarchive cert tarball @@ -176,25 +172,8 @@ state: directory with_items: - "{{ etcd_ca_dir }}" - - "{{ etcd_system_container_cert_config_dir }}/ca" when: etcd_server_certs_missing | bool -- name: Unarchive cert tarball for the system container - unarchive: - src: "{{ g_etcd_server_mktemp.stdout }}/{{ etcd_cert_subdir }}.tgz" - dest: "{{ etcd_system_container_cert_config_dir }}" - when: - - etcd_server_certs_missing | bool - - r_etcd_common_etcd_runtime == 'runc' - -- name: Unarchive etcd ca cert tarballs for the system container - unarchive: - src: "{{ g_etcd_server_mktemp.stdout }}/{{ etcd_ca_name }}.tgz" - dest: "{{ etcd_system_container_cert_config_dir }}/ca" - when: - - etcd_server_certs_missing | bool - - r_etcd_common_etcd_runtime == 'runc' - - name: Delete temporary directory local_action: file path="{{ g_etcd_server_mktemp.stdout }}" state=absent become: no diff --git a/roles/etcd/tasks/check_cluster_health.yml b/roles/etcd/tasks/check_cluster_health.yml new file mode 100644 index 000000000..75c110972 --- /dev/null +++ b/roles/etcd/tasks/check_cluster_health.yml @@ -0,0 +1,2 @@ +--- +- include: migration/check_cluster_health.yml diff --git a/roles/etcd/tasks/disable_etcd.yml b/roles/etcd/tasks/disable_etcd.yml new file mode 100644 index 000000000..9202e6e48 --- /dev/null +++ b/roles/etcd/tasks/disable_etcd.yml @@ -0,0 +1,2 @@ +--- +- include: auxiliary/disable_etcd.yml diff --git a/roles/etcd/tasks/fetch_backup.yml b/roles/etcd/tasks/fetch_backup.yml new file mode 100644 index 000000000..513eed17a --- /dev/null +++ b/roles/etcd/tasks/fetch_backup.yml @@ -0,0 +1,8 @@ +--- +- include: backup/vars.yml + +- include: backup/archive.yml + +- include: backup/sync_backup.yml + +- include: backup/ diff --git a/roles/etcd/tasks/migration/check.yml b/roles/etcd/tasks/migration/check.yml index 0804d9e1c..5c45e5ae1 100644 --- a/roles/etcd/tasks/migration/check.yml +++ b/roles/etcd/tasks/migration/check.yml @@ -3,6 +3,17 @@ # Check the cluster is healthy - include: check_cluster_health.yml +# Check if there is at least one v2 snapshot +- name: Check if there is at least one v2 snapshot + find: + paths: "{{ etcd_data_dir }}/member/snap" + patterns: '*.snap' + register: snapshots_result + +- fail: + msg: "Before the migration can proceed the etcd member must write down at least one snapshot under {{ etcd_data_dir }}/member/snap directory." + when: snapshots_result.matched | int == 0 + # Check if the member has v3 data already # Run the migration only if the data are v2 - name: Check if there are any v3 data diff --git a/roles/etcd/tasks/system_container.yml b/roles/etcd/tasks/system_container.yml index e735bf50a..f71d9b551 100644 --- a/roles/etcd/tasks/system_container.yml +++ b/roles/etcd/tasks/system_container.yml @@ -1,6 +1,8 @@ --- -- set_fact: - l_etcd_src_data_dir: "{{ '/var/lib/origin/openshift.local.etcd' if r_etcd_common_embedded_etcd | bool else '/var/lib/etcd/' }}" +- name: Ensure proxies are in the atomic.conf + include_role: + name: openshift_atomic + tasks_from: proxy - name: Pull etcd system container command: atomic pull --storage=ostree {{ openshift.etcd.etcd_image }} @@ -17,6 +19,7 @@ {{ hostvars[host].etcd_hostname }}={{ etcd_peer_url_scheme }}://{{ hostvars[host].etcd_ip }}:{{ etcd_peer_port }}, {%- endif -%} {% endfor -%} + when: etcd_initial_cluster is undefined - name: Check etcd system container package command: > @@ -51,36 +54,13 @@ - name: Systemd reload configuration systemd: name=etcd_container daemon_reload=yes -- name: Check for previous etcd data store - stat: - path: "{{ l_etcd_src_data_dir }}/member/" - register: src_datastore - -- name: Check for etcd system container data store - stat: - path: "{{ r_etcd_common_system_container_host_dir }}/etcd.etcd/member" - register: dest_datastore - -- name: Ensure that etcd system container data dirs exist - file: path="{{ item }}" state=directory - with_items: - - "{{ r_etcd_common_system_container_host_dir }}/etc" - - "{{ r_etcd_common_system_container_host_dir }}/etcd.etcd" - -- name: Copy etcd data store - command: > - cp -a {{ l_etcd_src_data_dir }}/member - {{ r_etcd_common_system_container_host_dir }}/etcd.etcd/member - when: - - src_datastore.stat.exists - - not dest_datastore.stat.exists - - name: Install or Update Etcd system container package oc_atomic_container: name: etcd image: "{{ openshift.etcd.etcd_image }}" state: latest values: + - ETCD_DATA_DIR=/var/lib/etcd - ETCD_LISTEN_PEER_URLS={{ etcd_listen_peer_urls }} - ETCD_NAME={{ etcd_hostname }} - ETCD_INITIAL_CLUSTER={{ etcd_initial_cluster }} @@ -89,11 +69,21 @@ - ETCD_INITIAL_CLUSTER_STATE={{ etcd_initial_cluster_state }} - ETCD_INITIAL_CLUSTER_TOKEN={{ etcd_initial_cluster_token }} - ETCD_ADVERTISE_CLIENT_URLS={{ etcd_advertise_client_urls }} - - ETCD_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt - - ETCD_CERT_FILE={{ etcd_system_container_conf_dir }}/server.crt - - ETCD_KEY_FILE={{ etcd_system_container_conf_dir }}/server.key - - ETCD_PEER_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt - - ETCD_PEER_CERT_FILE={{ etcd_system_container_conf_dir }}/peer.crt - - ETCD_PEER_KEY_FILE={{ etcd_system_container_conf_dir }}/peer.key - - ETCD_TRUSTED_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt - - ETCD_PEER_TRUSTED_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt + - ETCD_CA_FILE={{ etcd_ca_file }} + - ETCD_CERT_FILE={{ etcd_cert_file }} + - ETCD_KEY_FILE={{ etcd_key_file }} + - ETCD_PEER_CA_FILE={{ etcd_peer_ca_file }} + - ETCD_PEER_CERT_FILE={{ etcd_peer_cert_file }} + - ETCD_PEER_KEY_FILE={{ etcd_peer_key_file }} + - ETCD_TRUSTED_CA_FILE={{ etcd_ca_file }} + - ETCD_PEER_TRUSTED_CA_FILE={{ etcd_peer_ca_file }} + - 'ADDTL_MOUNTS=,{"type":"bind","source":"/etc/","destination":"/etc/","options":["rbind","rw","rslave"]},{"type":"bind","source":"/var/lib/etcd","destination":"/var/lib/etcd/","options":["rbind","rw","rslave"]}' + +- name: Ensure etcd datadir ownership for the system container + file: + path: "{{ etcd_data_dir }}" + state: directory + mode: 0700 + owner: root + group: root + recurse: True |