summaryrefslogtreecommitdiffstats
path: root/roles/etcd
diff options
context:
space:
mode:
Diffstat (limited to 'roles/etcd')
-rw-r--r--roles/etcd/meta/main.yml2
-rw-r--r--roles/etcd/tasks/auxiliary/clean_data.yml5
-rw-r--r--roles/etcd/tasks/ca.yml2
-rw-r--r--roles/etcd/tasks/ca/deploy.yml78
-rw-r--r--roles/etcd/tasks/clean_data.yml2
-rw-r--r--roles/etcd/tasks/client_certificates.yml2
-rw-r--r--roles/etcd/tasks/client_certificates/fetch_from_ca.yml138
-rw-r--r--roles/etcd/tasks/main.yml2
-rw-r--r--roles/etcd/tasks/migrate.add_ttls.yml2
-rw-r--r--roles/etcd/tasks/migrate.configure_master.yml2
-rw-r--r--roles/etcd/tasks/migrate.pre_check.yml2
-rw-r--r--roles/etcd/tasks/migrate.yml2
-rw-r--r--roles/etcd/tasks/migration/add_ttls.yml34
-rw-r--r--roles/etcd/tasks/migration/check.yml56
-rw-r--r--roles/etcd/tasks/migration/check_cluster_health.yml23
-rw-r--r--roles/etcd/tasks/migration/check_cluster_status.yml32
-rw-r--r--roles/etcd/tasks/migration/configure_master.yml13
-rw-r--r--roles/etcd/tasks/migration/migrate.yml56
-rw-r--r--roles/etcd/tasks/server_certificates.yml2
-rw-r--r--roles/etcd/tasks/server_certificates/fetch_from_ca.yml238
-rw-r--r--roles/etcd/tasks/upgrade/upgrade_image.yml55
-rw-r--r--roles/etcd/tasks/upgrade/upgrade_rpm.yml32
-rw-r--r--roles/etcd/tasks/upgrade_image.yml2
-rw-r--r--roles/etcd/tasks/upgrade_rpm.yml2
-rw-r--r--roles/etcd/templates/openssl_append.j251
25 files changed, 834 insertions, 1 deletions
diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml
index 9a955c822..537c01c0e 100644
--- a/roles/etcd/meta/main.yml
+++ b/roles/etcd/meta/main.yml
@@ -18,5 +18,5 @@ galaxy_info:
dependencies:
- role: lib_openshift
- role: lib_os_firewall
-- role: etcd_server_certificates
+- role: lib_utils
- role: etcd_common
diff --git a/roles/etcd/tasks/auxiliary/clean_data.yml b/roles/etcd/tasks/auxiliary/clean_data.yml
new file mode 100644
index 000000000..95a0e7c0a
--- /dev/null
+++ b/roles/etcd/tasks/auxiliary/clean_data.yml
@@ -0,0 +1,5 @@
+---
+- name: Remove member data
+ file:
+ path: /var/lib/etcd/member
+ state: absent
diff --git a/roles/etcd/tasks/ca.yml b/roles/etcd/tasks/ca.yml
new file mode 100644
index 000000000..7cda49069
--- /dev/null
+++ b/roles/etcd/tasks/ca.yml
@@ -0,0 +1,2 @@
+---
+- include: ca/deploy.yml
diff --git a/roles/etcd/tasks/ca/deploy.yml b/roles/etcd/tasks/ca/deploy.yml
new file mode 100644
index 000000000..3d32290a2
--- /dev/null
+++ b/roles/etcd/tasks/ca/deploy.yml
@@ -0,0 +1,78 @@
+---
+- name: Install openssl
+ package:
+ name: openssl
+ state: present
+ when: not etcd_is_atomic | bool
+ delegate_to: "{{ etcd_ca_host }}"
+ run_once: true
+
+- file:
+ path: "{{ item }}"
+ state: directory
+ mode: 0700
+ owner: root
+ group: root
+ with_items:
+ - "{{ etcd_ca_new_certs_dir }}"
+ - "{{ etcd_ca_crl_dir }}"
+ - "{{ etcd_ca_dir }}/fragments"
+ delegate_to: "{{ etcd_ca_host }}"
+ run_once: true
+
+- command: cp /etc/pki/tls/openssl.cnf ./
+ args:
+ chdir: "{{ etcd_ca_dir }}/fragments"
+ creates: "{{ etcd_ca_dir }}/fragments/openssl.cnf"
+ delegate_to: "{{ etcd_ca_host }}"
+ run_once: true
+
+- template:
+ dest: "{{ etcd_ca_dir }}/fragments/openssl_append.cnf"
+ src: openssl_append.j2
+ backup: true
+ delegate_to: "{{ etcd_ca_host }}"
+ run_once: true
+
+- assemble:
+ src: "{{ etcd_ca_dir }}/fragments"
+ dest: "{{ etcd_openssl_conf }}"
+ delegate_to: "{{ etcd_ca_host }}"
+ run_once: true
+
+- name: Check etcd_ca_db exist
+ stat: path="{{ etcd_ca_db }}"
+ register: etcd_ca_db_check
+ changed_when: false
+ delegate_to: "{{ etcd_ca_host }}"
+ run_once: true
+
+- name: Touch etcd_ca_db file
+ file:
+ path: "{{ etcd_ca_db }}"
+ state: touch
+ when: etcd_ca_db_check.stat.isreg is not defined
+ delegate_to: "{{ etcd_ca_host }}"
+ run_once: true
+
+- copy:
+ dest: "{{ etcd_ca_serial }}"
+ content: "01"
+ force: no
+ delegate_to: "{{ etcd_ca_host }}"
+ run_once: true
+
+- name: Create etcd CA certificate
+ command: >
+ openssl req -config {{ etcd_openssl_conf }} -newkey rsa:4096
+ -keyout {{ etcd_ca_key }} -new -out {{ etcd_ca_cert }}
+ -x509 -extensions {{ etcd_ca_exts_self }} -batch -nodes
+ -days {{ etcd_ca_default_days }}
+ -subj /CN=etcd-signer@{{ ansible_date_time.epoch }}
+ args:
+ chdir: "{{ etcd_ca_dir }}"
+ creates: "{{ etcd_ca_cert }}"
+ environment:
+ SAN: 'etcd-signer'
+ delegate_to: "{{ etcd_ca_host }}"
+ run_once: true
diff --git a/roles/etcd/tasks/clean_data.yml b/roles/etcd/tasks/clean_data.yml
new file mode 100644
index 000000000..d131ffd21
--- /dev/null
+++ b/roles/etcd/tasks/clean_data.yml
@@ -0,0 +1,2 @@
+---
+- include: auxiliary/clean_data.yml
diff --git a/roles/etcd/tasks/client_certificates.yml b/roles/etcd/tasks/client_certificates.yml
new file mode 100644
index 000000000..2e9c078b9
--- /dev/null
+++ b/roles/etcd/tasks/client_certificates.yml
@@ -0,0 +1,2 @@
+---
+- include: client_certificates/fetch_from_ca.yml
diff --git a/roles/etcd/tasks/client_certificates/fetch_from_ca.yml b/roles/etcd/tasks/client_certificates/fetch_from_ca.yml
new file mode 100644
index 000000000..119071a72
--- /dev/null
+++ b/roles/etcd/tasks/client_certificates/fetch_from_ca.yml
@@ -0,0 +1,138 @@
+---
+- name: Ensure CA certificate exists on etcd_ca_host
+ stat:
+ path: "{{ etcd_ca_cert }}"
+ register: g_ca_cert_stat_result
+ delegate_to: "{{ etcd_ca_host }}"
+ run_once: true
+
+- fail:
+ msg: >
+ CA certificate {{ etcd_ca_cert }} doesn't exist on CA host
+ {{ etcd_ca_host }}. Apply 'etcd_ca' action from `etcd` role to
+ {{ etcd_ca_host }}.
+ when: not g_ca_cert_stat_result.stat.exists | bool
+ run_once: true
+
+- name: Check status of external etcd certificatees
+ stat:
+ path: "{{ etcd_cert_config_dir }}/{{ item }}"
+ with_items:
+ - "{{ etcd_cert_prefix }}client.crt"
+ - "{{ etcd_cert_prefix }}client.key"
+ - "{{ etcd_cert_prefix }}ca.crt"
+ register: g_external_etcd_cert_stat_result
+ when: not etcd_certificates_redeploy | default(false) | bool
+
+- set_fact:
+ etcd_client_certs_missing: "{{ true if etcd_certificates_redeploy | default(false) | bool
+ else (False in (g_external_etcd_cert_stat_result.results
+ | default({})
+ | oo_collect(attribute='stat.exists')
+ | list)) }}"
+
+- name: Ensure generated_certs directory present
+ file:
+ path: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}"
+ state: directory
+ mode: 0700
+ when: etcd_client_certs_missing | bool
+ delegate_to: "{{ etcd_ca_host }}"
+
+- name: Create the client csr
+ command: >
+ openssl req -new -keyout {{ etcd_cert_prefix }}client.key
+ -config {{ etcd_openssl_conf }}
+ -out {{ etcd_cert_prefix }}client.csr
+ -reqexts {{ etcd_req_ext }} -batch -nodes
+ -subj /CN={{ etcd_hostname }}
+ args:
+ chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}"
+ creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/'
+ ~ etcd_cert_prefix ~ 'client.csr' }}"
+ environment:
+ SAN: "IP:{{ etcd_ip }},DNS:{{ etcd_hostname }}"
+ when: etcd_client_certs_missing | bool
+ delegate_to: "{{ etcd_ca_host }}"
+
+# Certificates must be signed serially in order to avoid competing
+# for the serial file.
+- name: Sign and create the client crt
+ delegated_serial_command:
+ command: >
+ openssl ca -name {{ etcd_ca_name }} -config {{ etcd_openssl_conf }}
+ -out {{ etcd_cert_prefix }}client.crt
+ -in {{ etcd_cert_prefix }}client.csr
+ -batch
+ chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}"
+ creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/'
+ ~ etcd_cert_prefix ~ 'client.crt' }}"
+ environment:
+ SAN: "IP:{{ etcd_ip }}"
+ when: etcd_client_certs_missing | bool
+ delegate_to: "{{ etcd_ca_host }}"
+
+- file:
+ src: "{{ etcd_ca_cert }}"
+ dest: "{{ etcd_generated_certs_dir}}/{{ etcd_cert_subdir }}/{{ etcd_cert_prefix }}ca.crt"
+ state: hard
+ when: etcd_client_certs_missing | bool
+ delegate_to: "{{ etcd_ca_host }}"
+
+- name: Create local temp directory for syncing certs
+ local_action: command mktemp -d /tmp/etcd_certificates-XXXXXXX
+ register: g_etcd_client_mktemp
+ changed_when: False
+ when: etcd_client_certs_missing | bool
+ become: no
+
+- name: Create a tarball of the etcd certs
+ command: >
+ tar -czvf {{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz
+ -C {{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }} .
+ args:
+ creates: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz"
+ # Disables the following warning:
+ # Consider using unarchive module rather than running tar
+ warn: no
+ when: etcd_client_certs_missing | bool
+ delegate_to: "{{ etcd_ca_host }}"
+
+- name: Retrieve the etcd cert tarballs
+ fetch:
+ src: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz"
+ dest: "{{ g_etcd_client_mktemp.stdout }}/"
+ flat: yes
+ fail_on_missing: yes
+ validate_checksum: yes
+ when: etcd_client_certs_missing | bool
+ delegate_to: "{{ etcd_ca_host }}"
+
+- name: Ensure certificate directory exists
+ file:
+ path: "{{ etcd_cert_config_dir }}"
+ state: directory
+ when: etcd_client_certs_missing | bool
+
+- name: Unarchive etcd cert tarballs
+ unarchive:
+ src: "{{ g_etcd_client_mktemp.stdout }}/{{ etcd_cert_subdir }}.tgz"
+ dest: "{{ etcd_cert_config_dir }}"
+ when: etcd_client_certs_missing | bool
+
+- file:
+ path: "{{ etcd_cert_config_dir }}/{{ item }}"
+ owner: root
+ group: root
+ mode: 0600
+ with_items:
+ - "{{ etcd_cert_prefix }}client.crt"
+ - "{{ etcd_cert_prefix }}client.key"
+ - "{{ etcd_cert_prefix }}ca.crt"
+ when: etcd_client_certs_missing | bool
+
+- name: Delete temporary directory
+ local_action: file path="{{ g_etcd_client_mktemp.stdout }}" state=absent
+ changed_when: False
+ when: etcd_client_certs_missing | bool
+ become: no
diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml
index 78e543ef1..870c11ad4 100644
--- a/roles/etcd/tasks/main.yml
+++ b/roles/etcd/tasks/main.yml
@@ -1,4 +1,6 @@
---
+- include: server_certificates.yml
+
- name: Set hostname and ip facts
set_fact:
# Store etcd_hostname and etcd_ip such that they will be available
diff --git a/roles/etcd/tasks/migrate.add_ttls.yml b/roles/etcd/tasks/migrate.add_ttls.yml
new file mode 100644
index 000000000..bc27e4ea1
--- /dev/null
+++ b/roles/etcd/tasks/migrate.add_ttls.yml
@@ -0,0 +1,2 @@
+---
+- include: migration/add_ttls.yml
diff --git a/roles/etcd/tasks/migrate.configure_master.yml b/roles/etcd/tasks/migrate.configure_master.yml
new file mode 100644
index 000000000..3ada6e362
--- /dev/null
+++ b/roles/etcd/tasks/migrate.configure_master.yml
@@ -0,0 +1,2 @@
+---
+- include: migration/configure_master.yml
diff --git a/roles/etcd/tasks/migrate.pre_check.yml b/roles/etcd/tasks/migrate.pre_check.yml
new file mode 100644
index 000000000..124d21561
--- /dev/null
+++ b/roles/etcd/tasks/migrate.pre_check.yml
@@ -0,0 +1,2 @@
+---
+- include: migration/check.yml
diff --git a/roles/etcd/tasks/migrate.yml b/roles/etcd/tasks/migrate.yml
new file mode 100644
index 000000000..5d5385873
--- /dev/null
+++ b/roles/etcd/tasks/migrate.yml
@@ -0,0 +1,2 @@
+---
+- include: migration/migrate.yml
diff --git a/roles/etcd/tasks/migration/add_ttls.yml b/roles/etcd/tasks/migration/add_ttls.yml
new file mode 100644
index 000000000..14625e49e
--- /dev/null
+++ b/roles/etcd/tasks/migration/add_ttls.yml
@@ -0,0 +1,34 @@
+---
+# To be executed on first master
+- slurp:
+ src: "{{ openshift.common.config_base }}/master/master-config.yaml"
+ register: g_master_config_output
+
+- set_fact:
+ accessTokenMaxAgeSeconds: "{{ (g_master_config_output.content|b64decode|from_yaml).oauthConfig.tokenConfig.accessTokenMaxAgeSeconds | default(86400) }}"
+ authroizeTokenMaxAgeSeconds: "{{ (g_master_config_output.content|b64decode|from_yaml).oauthConfig.tokenConfig.authroizeTokenMaxAgeSeconds | default(500) }}"
+ controllerLeaseTTL: "{{ (g_master_config_output.content|b64decode|from_yaml).controllerLeaseTTL | default(30) }}"
+
+- name: Re-introduce leases (as a replacement for key TTLs)
+ command: >
+ oadm migrate etcd-ttl \
+ --cert {{ r_etcd_common_master_peer_cert_file }} \
+ --key {{ r_etcd_common_master_peer_key_file }} \
+ --cacert {{ r_etcd_common_master_peer_ca_file }} \
+ --etcd-address 'https://{{ etcd_peer }}:{{ etcd_client_port }}' \
+ --ttl-keys-prefix {{ item.keys }} \
+ --lease-duration {{ item.ttl }}
+ environment:
+ ETCDCTL_API: 3
+ PATH: "/usr/local/bin:/var/usrlocal/bin:{{ ansible_env.PATH }}"
+ with_items:
+ - keys: "/kubernetes.io/events"
+ ttl: "1h"
+ - keys: "/kubernetes.io/masterleases"
+ ttl: "10s"
+ - keys: "/openshift.io/oauth/accesstokens"
+ ttl: "{{ accessTokenMaxAgeSeconds }}s"
+ - keys: "/openshift.io/oauth/authorizetokens"
+ ttl: "{{ authroizeTokenMaxAgeSeconds }}s"
+ - keys: "/openshift.io/leases/controllers"
+ ttl: "{{ controllerLeaseTTL }}s"
diff --git a/roles/etcd/tasks/migration/check.yml b/roles/etcd/tasks/migration/check.yml
new file mode 100644
index 000000000..0804d9e1c
--- /dev/null
+++ b/roles/etcd/tasks/migration/check.yml
@@ -0,0 +1,56 @@
+---
+
+# Check the cluster is healthy
+- include: check_cluster_health.yml
+
+# Check if the member has v3 data already
+# Run the migration only if the data are v2
+- name: Check if there are any v3 data
+ command: >
+ etcdctl --cert {{ etcd_peer_cert_file }} --key {{ etcd_peer_key_file }} --cacert {{ etcd_peer_ca_file }} --endpoints 'https://{{ etcd_peer }}:{{ etcd_client_port }}' get "" --from-key --keys-only -w json --limit 1
+ environment:
+ ETCDCTL_API: 3
+ register: l_etcdctl_output
+
+- fail:
+ msg: "Unable to get a number of v3 keys"
+ when: l_etcdctl_output.rc != 0
+
+- fail:
+ msg: "The etcd has at least one v3 key"
+ when: "'count' in (l_etcdctl_output.stdout | from_json) and (l_etcdctl_output.stdout | from_json).count != 0"
+
+
+# TODO(jchaloup): once the until loop can be used over include/block,
+# remove the repetive code
+# - until loop not supported over include statement (nor block)
+# https://github.com/ansible/ansible/issues/17098
+# - with_items not supported over block
+
+# Check the cluster status for the first time
+- include: check_cluster_status.yml
+
+# Check the cluster status for the second time
+- block:
+ - debug:
+ msg: "l_etcd_cluster_status_ok: {{ l_etcd_cluster_status_ok }}"
+ - name: Wait a while before another check
+ pause:
+ seconds: 5
+ when: not l_etcd_cluster_status_ok | bool
+
+ - include: check_cluster_status.yml
+ when: not l_etcd_cluster_status_ok | bool
+
+
+# Check the cluster status for the third time
+- block:
+ - debug:
+ msg: "l_etcd_cluster_status_ok: {{ l_etcd_cluster_status_ok }}"
+ - name: Wait a while before another check
+ pause:
+ seconds: 5
+ when: not l_etcd_cluster_status_ok | bool
+
+ - include: check_cluster_status.yml
+ when: not l_etcd_cluster_status_ok | bool
diff --git a/roles/etcd/tasks/migration/check_cluster_health.yml b/roles/etcd/tasks/migration/check_cluster_health.yml
new file mode 100644
index 000000000..201d83f99
--- /dev/null
+++ b/roles/etcd/tasks/migration/check_cluster_health.yml
@@ -0,0 +1,23 @@
+---
+- name: Check cluster health
+ command: >
+ etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoint https://{{ etcd_peer }}:{{ etcd_client_port }} cluster-health
+ register: etcd_cluster_health
+ changed_when: false
+ failed_when: false
+
+- name: Assume a member is not healthy
+ set_fact:
+ etcd_member_healthy: false
+
+- name: Get member item health status
+ set_fact:
+ etcd_member_healthy: true
+ with_items: "{{ etcd_cluster_health.stdout_lines }}"
+ when: "(etcd_peer in item) and ('is healthy' in item)"
+
+- name: Check the etcd cluster health
+ # TODO(jchaloup): should we fail or ask user if he wants to continue? Or just wait until the cluster is healthy?
+ fail:
+ msg: "Etcd member {{ etcd_peer }} is not healthy"
+ when: not etcd_member_healthy
diff --git a/roles/etcd/tasks/migration/check_cluster_status.yml b/roles/etcd/tasks/migration/check_cluster_status.yml
new file mode 100644
index 000000000..b69fb5a52
--- /dev/null
+++ b/roles/etcd/tasks/migration/check_cluster_status.yml
@@ -0,0 +1,32 @@
+---
+# etcd_ip originates from etcd_common role
+- name: Check cluster status
+ command: >
+ etcdctl --cert {{ etcd_peer_cert_file }} --key {{ etcd_peer_key_file }} --cacert {{ etcd_peer_ca_file }} --endpoints 'https://{{ etcd_peer }}:{{ etcd_client_port }}' -w json endpoint status
+ environment:
+ ETCDCTL_API: 3
+ register: l_etcd_cluster_status
+
+- name: Retrieve raftIndex
+ set_fact:
+ etcd_member_raft_index: "{{ (l_etcd_cluster_status.stdout | from_json)[0]['Status']['raftIndex'] }}"
+
+- block:
+ # http://docs.ansible.com/ansible/playbooks_filters.html#extracting-values-from-containers
+ - name: Group all raftIndices into a list
+ set_fact:
+ etcd_members_raft_indices: "{{ groups['oo_etcd_to_migrate'] | map('extract', hostvars, 'etcd_member_raft_index') | list | unique }}"
+
+ - name: Check the minimum and the maximum of raftIndices is at most 1
+ set_fact:
+ etcd_members_raft_indices_diff: "{{ ((etcd_members_raft_indices | max | int) - (etcd_members_raft_indices | min | int)) | int }}"
+
+ - debug:
+ msg: "Raft indices difference: {{ etcd_members_raft_indices_diff }}"
+
+ when: inventory_hostname in groups.oo_etcd_to_migrate[0]
+
+# The cluster raft status is ok if the difference of the max and min raft index is at most 1
+- name: capture the status
+ set_fact:
+ l_etcd_cluster_status_ok: "{{ hostvars[groups.oo_etcd_to_migrate[0]]['etcd_members_raft_indices_diff'] | int < 2 }}"
diff --git a/roles/etcd/tasks/migration/configure_master.yml b/roles/etcd/tasks/migration/configure_master.yml
new file mode 100644
index 000000000..a305d5bf3
--- /dev/null
+++ b/roles/etcd/tasks/migration/configure_master.yml
@@ -0,0 +1,13 @@
+---
+- name: Configure master to use etcd3 storage backend
+ yedit:
+ src: /etc/origin/master/master-config.yaml
+ key: "{{ item.key }}"
+ value: "{{ item.value }}"
+ with_items:
+ - key: kubernetesMasterConfig.apiServerArguments.storage-backend
+ value:
+ - etcd3
+ - key: kubernetesMasterConfig.apiServerArguments.storage-media-type
+ value:
+ - application/vnd.kubernetes.protobuf
diff --git a/roles/etcd/tasks/migration/migrate.yml b/roles/etcd/tasks/migration/migrate.yml
new file mode 100644
index 000000000..54a9c74ff
--- /dev/null
+++ b/roles/etcd/tasks/migration/migrate.yml
@@ -0,0 +1,56 @@
+---
+# Should this be run in a serial manner?
+- set_fact:
+ l_etcd_service: "{{ 'etcd_container' if openshift.common.is_containerized else 'etcd' }}"
+
+- name: Migrate etcd data
+ command: >
+ etcdctl migrate --data-dir={{ etcd_data_dir }}
+ environment:
+ ETCDCTL_API: 3
+ register: l_etcdctl_migrate
+# TODO(jchaloup): If any of the members fails, we need to restore all members to v2 from the pre-migrate backup
+- name: Check the etcd v2 data are correctly migrated
+ fail:
+ msg: "Failed to migrate a member"
+ when: "'finished transforming keys' not in l_etcdctl_migrate.stdout and 'no v2 keys to migrate' not in l_etcdctl_migrate.stdout"
+- name: Migration message
+ debug:
+ msg: "Etcd migration finished with: {{ l_etcdctl_migrate.stdout }}"
+- name: Set ETCD_FORCE_NEW_CLUSTER=true on first etcd host
+ lineinfile:
+ line: "ETCD_FORCE_NEW_CLUSTER=true"
+ dest: /etc/etcd/etcd.conf
+ backup: true
+- name: Start etcd
+ systemd:
+ name: "{{ l_etcd_service }}"
+ state: started
+- name: Wait for cluster to become healthy after bringing up first member
+ command: >
+ etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoint https://{{ etcd_peer }}:{{ etcd_client_port }} cluster-health
+ register: l_etcd_migrate_health
+ until: l_etcd_migrate_health.rc == 0
+ retries: 3
+ delay: 30
+- name: Unset ETCD_FORCE_NEW_CLUSTER=true on first etcd host
+ lineinfile:
+ line: "ETCD_FORCE_NEW_CLUSTER=true"
+ dest: /etc/etcd/etcd.conf
+ state: absent
+ backup: true
+- name: Restart first etcd host
+ systemd:
+ name: "{{ l_etcd_service }}"
+ state: restarted
+
+- name: Wait for cluster to become healthy after bringing up first member
+ command: >
+ etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoint https://{{ etcd_peer }}:{{ etcd_client_port }} cluster-health
+ register: l_etcd_migrate_health
+ until: l_etcd_migrate_health.rc == 0
+ retries: 3
+ delay: 30
+
+- set_fact:
+ r_etcd_migrate_success: true
diff --git a/roles/etcd/tasks/server_certificates.yml b/roles/etcd/tasks/server_certificates.yml
new file mode 100644
index 000000000..f0ba58b6e
--- /dev/null
+++ b/roles/etcd/tasks/server_certificates.yml
@@ -0,0 +1,2 @@
+---
+- include: server_certificates/fetch_from_ca.yml
diff --git a/roles/etcd/tasks/server_certificates/fetch_from_ca.yml b/roles/etcd/tasks/server_certificates/fetch_from_ca.yml
new file mode 100644
index 000000000..064fe1952
--- /dev/null
+++ b/roles/etcd/tasks/server_certificates/fetch_from_ca.yml
@@ -0,0 +1,238 @@
+---
+- include: ../ca/deploy.yml
+ when:
+ - etcd_ca_setup | default(True) | bool
+
+- name: Install etcd
+ package:
+ name: "etcd{{ '-' + etcd_version if etcd_version is defined else '' }}"
+ state: present
+ when: not etcd_is_containerized | bool
+
+- name: Check status of etcd certificates
+ stat:
+ path: "{{ item }}"
+ with_items:
+ - "{{ etcd_cert_config_dir }}/{{ etcd_cert_prefix }}server.crt"
+ - "{{ etcd_cert_config_dir }}/{{ etcd_cert_prefix }}peer.crt"
+ - "{{ etcd_cert_config_dir }}/{{ etcd_cert_prefix }}ca.crt"
+ - "{{ etcd_system_container_cert_config_dir }}/{{ etcd_cert_prefix }}server.crt"
+ - "{{ etcd_system_container_cert_config_dir }}/{{ etcd_cert_prefix }}peer.crt"
+ - "{{ etcd_system_container_cert_config_dir }}/{{ etcd_cert_prefix }}ca.crt"
+ register: g_etcd_server_cert_stat_result
+ when: not etcd_certificates_redeploy | default(false) | bool
+
+- set_fact:
+ etcd_server_certs_missing: "{{ true if etcd_certificates_redeploy | default(false) | bool
+ else (False in (g_etcd_server_cert_stat_result.results
+ | default({})
+ | oo_collect(attribute='stat.exists')
+ | list)) }}"
+
+- name: Ensure generated_certs directory present
+ file:
+ path: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}"
+ state: directory
+ mode: 0700
+ when: etcd_server_certs_missing | bool
+ delegate_to: "{{ etcd_ca_host }}"
+
+- name: Create the server csr
+ command: >
+ openssl req -new -keyout {{ etcd_cert_prefix }}server.key
+ -config {{ etcd_openssl_conf }}
+ -out {{ etcd_cert_prefix }}server.csr
+ -reqexts {{ etcd_req_ext }} -batch -nodes
+ -subj /CN={{ etcd_hostname }}
+ args:
+ chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}"
+ creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/'
+ ~ etcd_cert_prefix ~ 'server.csr' }}"
+ environment:
+ SAN: "IP:{{ etcd_ip }},DNS:{{ etcd_hostname }}"
+ when: etcd_server_certs_missing | bool
+ delegate_to: "{{ etcd_ca_host }}"
+
+# Certificates must be signed serially in order to avoid competing
+# for the serial file.
+- name: Sign and create the server crt
+ delegated_serial_command:
+ command: >
+ openssl ca -name {{ etcd_ca_name }} -config {{ etcd_openssl_conf }}
+ -out {{ etcd_cert_prefix }}server.crt
+ -in {{ etcd_cert_prefix }}server.csr
+ -extensions {{ etcd_ca_exts_server }} -batch
+ chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}"
+ creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/'
+ ~ etcd_cert_prefix ~ 'server.crt' }}"
+ environment:
+ SAN: "IP:{{ etcd_ip }}"
+ when: etcd_server_certs_missing | bool
+ delegate_to: "{{ etcd_ca_host }}"
+
+- name: Create the peer csr
+ command: >
+ openssl req -new -keyout {{ etcd_cert_prefix }}peer.key
+ -config {{ etcd_openssl_conf }}
+ -out {{ etcd_cert_prefix }}peer.csr
+ -reqexts {{ etcd_req_ext }} -batch -nodes
+ -subj /CN={{ etcd_hostname }}
+ args:
+ chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}"
+ creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/'
+ ~ etcd_cert_prefix ~ 'peer.csr' }}"
+ environment:
+ SAN: "IP:{{ etcd_ip }},DNS:{{ etcd_hostname }}"
+ when: etcd_server_certs_missing | bool
+ delegate_to: "{{ etcd_ca_host }}"
+
+# Certificates must be signed serially in order to avoid competing
+# for the serial file.
+- name: Sign and create the peer crt
+ delegated_serial_command:
+ command: >
+ openssl ca -name {{ etcd_ca_name }} -config {{ etcd_openssl_conf }}
+ -out {{ etcd_cert_prefix }}peer.crt
+ -in {{ etcd_cert_prefix }}peer.csr
+ -extensions {{ etcd_ca_exts_peer }} -batch
+ chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}"
+ creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/'
+ ~ etcd_cert_prefix ~ 'peer.crt' }}"
+ environment:
+ SAN: "IP:{{ etcd_ip }}"
+ when: etcd_server_certs_missing | bool
+ delegate_to: "{{ etcd_ca_host }}"
+
+- file:
+ src: "{{ etcd_ca_cert }}"
+ dest: "{{ etcd_generated_certs_dir}}/{{ etcd_cert_subdir }}/{{ etcd_cert_prefix }}ca.crt"
+ state: hard
+ when: etcd_server_certs_missing | bool
+ delegate_to: "{{ etcd_ca_host }}"
+
+- name: Create local temp directory for syncing certs
+ local_action: command mktemp -d /tmp/etcd_certificates-XXXXXXX
+ become: no
+ register: g_etcd_server_mktemp
+ changed_when: False
+ when: etcd_server_certs_missing | bool
+
+- name: Create a tarball of the etcd certs
+ command: >
+ tar -czvf {{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz
+ -C {{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }} .
+ args:
+ creates: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz"
+ # Disables the following warning:
+ # Consider using unarchive module rather than running tar
+ warn: no
+ when: etcd_server_certs_missing | bool
+ delegate_to: "{{ etcd_ca_host }}"
+
+- name: Retrieve etcd cert tarball
+ fetch:
+ src: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz"
+ dest: "{{ g_etcd_server_mktemp.stdout }}/"
+ flat: yes
+ fail_on_missing: yes
+ validate_checksum: yes
+ when: etcd_server_certs_missing | bool
+ delegate_to: "{{ etcd_ca_host }}"
+
+- name: Ensure certificate directory exists
+ file:
+ path: "{{ item }}"
+ state: directory
+ with_items:
+ - "{{ etcd_cert_config_dir }}"
+ - "{{ etcd_system_container_cert_config_dir }}"
+ when: etcd_server_certs_missing | bool
+
+- name: Unarchive cert tarball
+ unarchive:
+ src: "{{ g_etcd_server_mktemp.stdout }}/{{ etcd_cert_subdir }}.tgz"
+ dest: "{{ etcd_cert_config_dir }}"
+ when: etcd_server_certs_missing | bool
+
+- name: Create a tarball of the etcd ca certs
+ command: >
+ tar -czvf {{ etcd_generated_certs_dir }}/{{ etcd_ca_name }}.tgz
+ -C {{ etcd_ca_dir }} .
+ args:
+ creates: "{{ etcd_generated_certs_dir }}/{{ etcd_ca_name }}.tgz"
+ warn: no
+ when: etcd_server_certs_missing | bool
+ delegate_to: "{{ etcd_ca_host }}"
+
+- name: Retrieve etcd ca cert tarball
+ fetch:
+ src: "{{ etcd_generated_certs_dir }}/{{ etcd_ca_name }}.tgz"
+ dest: "{{ g_etcd_server_mktemp.stdout }}/"
+ flat: yes
+ fail_on_missing: yes
+ validate_checksum: yes
+ when: etcd_server_certs_missing | bool
+ delegate_to: "{{ etcd_ca_host }}"
+
+- name: Ensure ca directory exists
+ file:
+ path: "{{ item }}"
+ state: directory
+ with_items:
+ - "{{ etcd_ca_dir }}"
+ - "{{ etcd_system_container_cert_config_dir }}/ca"
+ when: etcd_server_certs_missing | bool
+
+- name: Unarchive cert tarball for the system container
+ unarchive:
+ src: "{{ g_etcd_server_mktemp.stdout }}/{{ etcd_cert_subdir }}.tgz"
+ dest: "{{ etcd_system_container_cert_config_dir }}"
+ when:
+ - etcd_server_certs_missing | bool
+ - r_etcd_common_etcd_runtime == 'runc'
+
+- name: Unarchive etcd ca cert tarballs for the system container
+ unarchive:
+ src: "{{ g_etcd_server_mktemp.stdout }}/{{ etcd_ca_name }}.tgz"
+ dest: "{{ etcd_system_container_cert_config_dir }}/ca"
+ when:
+ - etcd_server_certs_missing | bool
+ - r_etcd_common_etcd_runtime == 'runc'
+
+- name: Delete temporary directory
+ local_action: file path="{{ g_etcd_server_mktemp.stdout }}" state=absent
+ become: no
+ changed_when: False
+ when: etcd_server_certs_missing | bool
+
+- name: Validate permissions on certificate files
+ file:
+ path: "{{ item }}"
+ mode: 0600
+ owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
+ group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
+ when: etcd_url_scheme == 'https'
+ with_items:
+ - "{{ etcd_ca_file }}"
+ - "{{ etcd_cert_file }}"
+ - "{{ etcd_key_file }}"
+
+- name: Validate permissions on peer certificate files
+ file:
+ path: "{{ item }}"
+ mode: 0600
+ owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
+ group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
+ when: etcd_peer_url_scheme == 'https'
+ with_items:
+ - "{{ etcd_peer_ca_file }}"
+ - "{{ etcd_peer_cert_file }}"
+ - "{{ etcd_peer_key_file }}"
+
+- name: Validate permissions on the config dir
+ file:
+ path: "{{ etcd_conf_dir }}"
+ state: directory
+ owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
+ group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
+ mode: 0700
diff --git a/roles/etcd/tasks/upgrade/upgrade_image.yml b/roles/etcd/tasks/upgrade/upgrade_image.yml
new file mode 100644
index 000000000..cea95a1b3
--- /dev/null
+++ b/roles/etcd/tasks/upgrade/upgrade_image.yml
@@ -0,0 +1,55 @@
+---
+# INPUT r_etcd_upgrade_version
+- name: Verify cluster is healthy pre-upgrade
+ command: "{{ etcdctlv2 }} cluster-health"
+
+- name: Get current image
+ shell: "grep 'ExecStart=' {{ etcd_service_file }} | awk '{print $NF}'"
+ register: current_image
+
+- name: Set new_etcd_image
+ set_fact:
+ new_etcd_image: "{{ current_image.stdout | regex_replace('/etcd.*$','/etcd:' ~ r_etcd_upgrade_version ) }}"
+
+- name: Pull new etcd image
+ command: "docker pull {{ new_etcd_image }}"
+
+- name: Update to latest etcd image
+ replace:
+ dest: "{{ etcd_service_file }}"
+ regexp: "{{ current_image.stdout }}$"
+ replace: "{{ new_etcd_image }}"
+
+- name: Restart etcd_container
+ systemd:
+ name: "{{ etcd_service }}"
+ daemon_reload: yes
+ state: restarted
+
+## TODO: probably should just move this into the backup playbooks, also this
+## will fail on atomic host. We need to revisit how to do etcd backups there as
+## the container may be newer than etcdctl on the host. Assumes etcd3 obsoletes etcd (7.3.1)
+- name: Detecting Atomic Host Operating System
+ stat:
+ path: /run/ostree-booted
+ register: l_ostree_booted
+
+- name: Upgrade etcd for etcdctl when not atomic
+ package:
+ name: etcd
+ state: latest
+ when: not l_ostree_booted.stat.exists | bool
+
+- name: Verify cluster is healthy
+ command: "{{ etcdctlv2 }} cluster-health"
+ register: etcdctl
+ until: etcdctl.rc == 0
+ retries: 3
+ delay: 10
+
+- name: Store new etcd_image
+ # DEPENDENCY openshift_facts
+ openshift_facts:
+ role: etcd
+ local_facts:
+ etcd_image: "{{ new_etcd_image }}"
diff --git a/roles/etcd/tasks/upgrade/upgrade_rpm.yml b/roles/etcd/tasks/upgrade/upgrade_rpm.yml
new file mode 100644
index 000000000..324b69605
--- /dev/null
+++ b/roles/etcd/tasks/upgrade/upgrade_rpm.yml
@@ -0,0 +1,32 @@
+---
+# INPUT r_etcd_upgrade_version?
+
+# F23 GA'd with etcd 2.0, currently has 2.2 in updates
+# F24 GA'd with etcd-2.2, currently has 2.2 in updates
+# F25 Beta currently has etcd 3.0
+# RHEL 7.3.4 with etcd-3.1.3-1.el7
+# RHEL 7.3.3 with etcd-3.1.0-2.el7
+# RHEL 7.3.2 with etcd-3.0.15-1.el7
+
+- name: Verify cluster is healthy pre-upgrade
+ command: "{{ etcdctlv2 }} cluster-health"
+
+- set_fact:
+ l_etcd_target_package: "{{ 'etcd' if r_etcd_upgrade_version is not defined else 'etcd-'+r_etcd_upgrade_version+'*' }}"
+
+- name: Update etcd RPM to {{ l_etcd_target_package }}
+ package:
+ name: "{{ l_etcd_target_package }}"
+ state: latest
+
+- name: Restart etcd
+ service:
+ name: "{{ etcd_service }}"
+ state: restarted
+
+- name: Verify cluster is healthy
+ command: "{{ etcdctlv2 }} cluster-health"
+ register: etcdctl
+ until: etcdctl.rc == 0
+ retries: 3
+ delay: 10
diff --git a/roles/etcd/tasks/upgrade_image.yml b/roles/etcd/tasks/upgrade_image.yml
new file mode 100644
index 000000000..9e69027eb
--- /dev/null
+++ b/roles/etcd/tasks/upgrade_image.yml
@@ -0,0 +1,2 @@
+---
+- include: upgrade/upgrade_image.yml
diff --git a/roles/etcd/tasks/upgrade_rpm.yml b/roles/etcd/tasks/upgrade_rpm.yml
new file mode 100644
index 000000000..29603d2b6
--- /dev/null
+++ b/roles/etcd/tasks/upgrade_rpm.yml
@@ -0,0 +1,2 @@
+---
+- include: upgrade/upgrade_rpm.yml
diff --git a/roles/etcd/templates/openssl_append.j2 b/roles/etcd/templates/openssl_append.j2
new file mode 100644
index 000000000..f28316fc2
--- /dev/null
+++ b/roles/etcd/templates/openssl_append.j2
@@ -0,0 +1,51 @@
+
+[ {{ etcd_req_ext }} ]
+basicConstraints = critical,CA:FALSE
+keyUsage = digitalSignature,keyEncipherment
+subjectAltName = ${ENV::SAN}
+
+[ {{ etcd_ca_name }} ]
+dir = {{ etcd_ca_dir }}
+crl_dir = {{ etcd_ca_crl_dir }}
+database = {{ etcd_ca_db }}
+new_certs_dir = {{ etcd_ca_new_certs_dir }}
+certificate = {{ etcd_ca_cert }}
+serial = {{ etcd_ca_serial }}
+private_key = {{ etcd_ca_key }}
+crl_number = {{ etcd_ca_crl_number }}
+x509_extensions = {{ etcd_ca_exts_client }}
+default_days = {{ etcd_ca_default_days }}
+default_md = sha256
+preserve = no
+name_opt = ca_default
+cert_opt = ca_default
+policy = policy_anything
+unique_subject = no
+copy_extensions = copy
+
+[ {{ etcd_ca_exts_self }} ]
+authorityKeyIdentifier = keyid,issuer
+basicConstraints = critical,CA:TRUE,pathlen:0
+keyUsage = critical,digitalSignature,keyEncipherment,keyCertSign
+subjectKeyIdentifier = hash
+
+[ {{ etcd_ca_exts_peer }} ]
+authorityKeyIdentifier = keyid,issuer:always
+basicConstraints = critical,CA:FALSE
+extendedKeyUsage = clientAuth,serverAuth
+keyUsage = digitalSignature,keyEncipherment
+subjectKeyIdentifier = hash
+
+[ {{ etcd_ca_exts_server }} ]
+authorityKeyIdentifier = keyid,issuer:always
+basicConstraints = critical,CA:FALSE
+extendedKeyUsage = serverAuth
+keyUsage = digitalSignature,keyEncipherment
+subjectKeyIdentifier = hash
+
+[ {{ etcd_ca_exts_client }} ]
+authorityKeyIdentifier = keyid,issuer:always
+basicConstraints = critical,CA:FALSE
+extendedKeyUsage = clientAuth
+keyUsage = digitalSignature,keyEncipherment
+subjectKeyIdentifier = hash