diff options
Diffstat (limited to 'roles/etcd')
71 files changed, 2022 insertions, 0 deletions
diff --git a/roles/etcd/README.md b/roles/etcd/README.md new file mode 100644 index 000000000..c936dbabc --- /dev/null +++ b/roles/etcd/README.md @@ -0,0 +1,40 @@ +Role Name +========= + +Configures an etcd cluster for an arbitrary number of hosts + +Requirements +------------ + +* Ansible 2.2 +* This role assumes it's being deployed on a RHEL/Fedora based host with package +named 'etcd' available via yum or dnf (conditionally). + +Role Variables +-------------- + +TODO + +Dependencies +------------ + +etcd-common + +Example Playbook +---------------- + + - hosts: etcd + roles: + - { etcd } + +License +------- + +MIT + +Author Information +------------------ + +Scott Dodson <sdodson@redhat.com> +Adapted from https://github.com/retr0h/ansible-etcd for use on RHEL/Fedora. We +should at some point submit a PR to merge this with that module. diff --git a/roles/etcd/defaults/main.yaml b/roles/etcd/defaults/main.yaml new file mode 100644 index 000000000..78f231416 --- /dev/null +++ b/roles/etcd/defaults/main.yaml @@ -0,0 +1,93 @@ +--- +r_etcd_common_backup_tag: '' +r_etcd_common_backup_sufix_name: '' + +# runc, docker, host +r_etcd_common_etcd_runtime: "docker" +r_etcd_common_embedded_etcd: false + +# etcd run on a host => use etcdctl command directly +# etcd run as a docker container => use docker exec +# etcd run as a runc container => use runc exec +r_etcd_common_etcdctl_command: "{{ 'etcdctl' if r_etcd_common_etcd_runtime == 'host' or r_etcd_common_embedded_etcd | bool else 'docker exec etcd_container etcdctl' if r_etcd_common_etcd_runtime == 'docker' else 'runc exec etcd etcdctl' }}" + +# etcd server vars +etcd_conf_dir: '/etc/etcd' +r_etcd_common_system_container_host_dir: /var/lib/etcd/etcd.etcd +etcd_system_container_conf_dir: /var/lib/etcd/etc +etcd_conf_file: "{{ etcd_conf_dir }}/etcd.conf" +etcd_ca_file: "{{ etcd_conf_dir }}/ca.crt" +etcd_cert_file: "{{ etcd_conf_dir }}/server.crt" +etcd_key_file: "{{ etcd_conf_dir }}/server.key" +etcd_peer_ca_file: "{{ etcd_conf_dir }}/ca.crt" +etcd_peer_cert_file: "{{ etcd_conf_dir }}/peer.crt" +etcd_peer_key_file: "{{ etcd_conf_dir }}/peer.key" + +# etcd ca vars +etcd_ca_dir: "{{ etcd_conf_dir}}/ca" +etcd_generated_certs_dir: "{{ etcd_conf_dir }}/generated_certs" +etcd_ca_cert: "{{ etcd_ca_dir }}/ca.crt" +etcd_ca_key: "{{ etcd_ca_dir }}/ca.key" +etcd_openssl_conf: "{{ etcd_ca_dir }}/openssl.cnf" +etcd_ca_name: etcd_ca +etcd_req_ext: etcd_v3_req +etcd_ca_exts_peer: etcd_v3_ca_peer +etcd_ca_exts_server: etcd_v3_ca_server +etcd_ca_exts_self: etcd_v3_ca_self +etcd_ca_exts_client: etcd_v3_ca_client +etcd_ca_crl_dir: "{{ etcd_ca_dir }}/crl" +etcd_ca_new_certs_dir: "{{ etcd_ca_dir }}/certs" +etcd_ca_db: "{{ etcd_ca_dir }}/index.txt" +etcd_ca_serial: "{{ etcd_ca_dir }}/serial" +etcd_ca_crl_number: "{{ etcd_ca_dir }}/crlnumber" +etcd_ca_default_days: 1825 + +r_etcd_common_master_peer_cert_file: /etc/origin/master/master.etcd-client.crt +r_etcd_common_master_peer_key_file: /etc/origin/master/master.etcd-client.key +r_etcd_common_master_peer_ca_file: /etc/origin/master/master.etcd-ca.crt + +# etcd server & certificate vars +etcd_hostname: "{{ inventory_hostname }}" +etcd_ip: "{{ ansible_default_ipv4.address }}" +etcd_is_atomic: False +etcd_is_containerized: False +etcd_is_thirdparty: False + +# etcd dir vars +etcd_data_dir: "{{ '/var/lib/origin/openshift.local.etcd' if r_etcd_common_embedded_etcd | bool else '/var/lib/etcd/' if r_etcd_common_etcd_runtime != 'runc' else '/var/lib/etcd/etcd.etcd/' }}" + +# etcd ports and protocols +etcd_client_port: 2379 +etcd_peer_port: 2380 +etcd_url_scheme: http +etcd_peer_url_scheme: http + +etcd_initial_cluster_state: new +etcd_initial_cluster_token: etcd-cluster-1 + +etcd_initial_advertise_peer_urls: "{{ etcd_peer_url_scheme }}://{{ etcd_ip }}:{{ etcd_peer_port }}" +etcd_listen_peer_urls: "{{ etcd_peer_url_scheme }}://{{ etcd_ip }}:{{ etcd_peer_port }}" +etcd_advertise_client_urls: "{{ etcd_url_scheme }}://{{ etcd_ip }}:{{ etcd_client_port }}" +etcd_listen_client_urls: "{{ etcd_url_scheme }}://{{ etcd_ip }}:{{ etcd_client_port }}" + +# required role variable +#etcd_peer: 127.0.0.1 +etcdctlv2: "etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} -C https://{{ etcd_peer }}:{{ etcd_client_port }}" + +etcd_service: "{{ 'etcd_container' if r_etcd_common_etcd_runtime == 'docker' else 'etcd' }}" +# Location of the service file is fixed and not meant to be changed +etcd_service_file: "/etc/systemd/system/{{ etcd_service }}.service" + +r_etcd_firewall_enabled: "{{ os_firewall_enabled | default(True) }}" +r_etcd_use_firewalld: "{{ os_firewall_use_firewalld | default(False) }}" + +etcd_systemd_dir: "/etc/systemd/system/{{ etcd_service }}.service.d" +r_etcd_os_firewall_deny: [] +r_etcd_os_firewall_allow: +- service: etcd + port: "{{etcd_client_port}}/tcp" +- service: etcd peering + port: "{{ etcd_peer_port }}/tcp" + +# set the backend quota to 4GB by default +etcd_quota_backend_bytes: 4294967296 diff --git a/roles/etcd/etcdctl.sh b/roles/etcd/etcdctl.sh new file mode 100644 index 000000000..0e324a8a9 --- /dev/null +++ b/roles/etcd/etcdctl.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Sets up handy aliases for etcd, need etcdctl2 and etcdctl3 because +# command flags are different between the two. Should work on stand +# alone etcd hosts and master + etcd hosts too because we use the peer keys. +etcdctl2() { + /usr/bin/etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://`hostname`:2379 ${@} +} + +etcdctl3() { + ETCDCTL_API=3 /usr/bin/etcdctl --cert /etc/etcd/peer.crt --key /etc/etcd/peer.key --cacert /etc/etcd/ca.crt --endpoints https://`hostname`:2379 ${@} +} diff --git a/roles/etcd/handlers/main.yml b/roles/etcd/handlers/main.yml new file mode 100644 index 000000000..95076b19e --- /dev/null +++ b/roles/etcd/handlers/main.yml @@ -0,0 +1,5 @@ +--- + +- name: restart etcd + systemd: name={{ etcd_service }} state=restarted + when: not (etcd_service_status_changed | default(false) | bool) diff --git a/roles/etcd/library/delegated_serial_command.py b/roles/etcd/library/delegated_serial_command.py new file mode 100755 index 000000000..0cab1ca88 --- /dev/null +++ b/roles/etcd/library/delegated_serial_command.py @@ -0,0 +1,274 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>, and others +# (c) 2016, Andrew Butcher <abutcher@redhat.com> +# +# This module is derrived from the Ansible command module. +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + + +# pylint: disable=unused-wildcard-import,wildcard-import,unused-import,redefined-builtin + +''' delegated_serial_command ''' + +import datetime +import errno +import glob +import shlex +import os +import fcntl +import time + +DOCUMENTATION = ''' +--- +module: delegated_serial_command +short_description: Executes a command on a remote node +version_added: historical +description: + - The M(command) module takes the command name followed by a list + of space-delimited arguments. + - The given command will be executed on all selected nodes. It + will not be processed through the shell, so variables like + C($HOME) and operations like C("<"), C(">"), C("|"), and C("&") + will not work (use the M(shell) module if you need these + features). + - Creates and maintains a lockfile such that this module will + wait for other invocations to proceed. +options: + command: + description: + - the command to run + required: true + default: null + creates: + description: + - a filename or (since 2.0) glob pattern, when it already + exists, this step will B(not) be run. + required: no + default: null + removes: + description: + - a filename or (since 2.0) glob pattern, when it does not + exist, this step will B(not) be run. + version_added: "0.8" + required: no + default: null + chdir: + description: + - cd into this directory before running the command + version_added: "0.6" + required: false + default: null + executable: + description: + - change the shell used to execute the command. Should be an + absolute path to the executable. + required: false + default: null + version_added: "0.9" + warn: + version_added: "1.8" + default: yes + description: + - if command warnings are on in ansible.cfg, do not warn about + this particular line if set to no/false. + required: false + lockfile: + default: yes + description: + - the lockfile that will be created + timeout: + default: yes + description: + - time in milliseconds to wait to obtain the lock +notes: + - If you want to run a command through the shell (say you are using C(<), + C(>), C(|), etc), you actually want the M(shell) module instead. The + M(command) module is much more secure as it's not affected by the user's + environment. + - " C(creates), C(removes), and C(chdir) can be specified after + the command. For instance, if you only want to run a command if + a certain file does not exist, use this." +author: + - Ansible Core Team + - Michael DeHaan + - Andrew Butcher +''' + +EXAMPLES = ''' +# Example from Ansible Playbooks. +- delegated_serial_command: + command: /sbin/shutdown -t now + +# Run the command if the specified file does not exist. +- delegated_serial_command: + command: /usr/bin/make_database.sh arg1 arg2 + creates: /path/to/database +''' + +# Dict of options and their defaults +OPTIONS = {'chdir': None, + 'creates': None, + 'command': None, + 'executable': None, + 'NO_LOG': None, + 'removes': None, + 'warn': True, + 'lockfile': None, + 'timeout': None} + + +def check_command(commandline): + ''' Check provided command ''' + arguments = {'chown': 'owner', 'chmod': 'mode', 'chgrp': 'group', + 'ln': 'state=link', 'mkdir': 'state=directory', + 'rmdir': 'state=absent', 'rm': 'state=absent', 'touch': 'state=touch'} + commands = {'git': 'git', 'hg': 'hg', 'curl': 'get_url or uri', 'wget': 'get_url or uri', + 'svn': 'subversion', 'service': 'service', + 'mount': 'mount', 'rpm': 'yum, dnf or zypper', 'yum': 'yum', 'apt-get': 'apt', + 'tar': 'unarchive', 'unzip': 'unarchive', 'sed': 'template or lineinfile', + 'rsync': 'synchronize', 'dnf': 'dnf', 'zypper': 'zypper'} + become = ['sudo', 'su', 'pbrun', 'pfexec', 'runas'] + warnings = list() + command = os.path.basename(commandline.split()[0]) + # pylint: disable=line-too-long + if command in arguments: + warnings.append("Consider using file module with {0} rather than running {1}".format(arguments[command], command)) + if command in commands: + warnings.append("Consider using {0} module rather than running {1}".format(commands[command], command)) + if command in become: + warnings.append( + "Consider using 'become', 'become_method', and 'become_user' rather than running {0}".format(command,)) + return warnings + + +# pylint: disable=too-many-statements,too-many-branches,too-many-locals +def main(): + ''' Main module function ''' + module = AnsibleModule( # noqa: F405 + argument_spec=dict( + _uses_shell=dict(type='bool', default=False), + command=dict(required=True), + chdir=dict(), + executable=dict(), + creates=dict(), + removes=dict(), + warn=dict(type='bool', default=True), + lockfile=dict(default='/tmp/delegated_serial_command.lock'), + timeout=dict(type='int', default=30) + ) + ) + + shell = module.params['_uses_shell'] + chdir = module.params['chdir'] + executable = module.params['executable'] + command = module.params['command'] + creates = module.params['creates'] + removes = module.params['removes'] + warn = module.params['warn'] + lockfile = module.params['lockfile'] + timeout = module.params['timeout'] + + if command.strip() == '': + module.fail_json(rc=256, msg="no command given") + + iterated = 0 + lockfd = open(lockfile, 'w+') + while iterated < timeout: + try: + fcntl.flock(lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + # pylint: disable=invalid-name + except IOError as e: + if e.errno != errno.EAGAIN: + module.fail_json(msg="I/O Error {0}: {1}".format(e.errno, e.strerror)) + else: + iterated += 1 + time.sleep(0.1) + + if chdir: + chdir = os.path.abspath(os.path.expanduser(chdir)) + os.chdir(chdir) + + if creates: + # do not run the command if the line contains creates=filename + # and the filename already exists. This allows idempotence + # of command executions. + path = os.path.expanduser(creates) + if glob.glob(path): + module.exit_json( + cmd=command, + stdout="skipped, since %s exists" % path, + changed=False, + stderr=False, + rc=0 + ) + + if removes: + # do not run the command if the line contains removes=filename + # and the filename does not exist. This allows idempotence + # of command executions. + path = os.path.expanduser(removes) + if not glob.glob(path): + module.exit_json( + cmd=command, + stdout="skipped, since %s does not exist" % path, + changed=False, + stderr=False, + rc=0 + ) + + warnings = list() + if warn: + warnings = check_command(command) + + if not shell: + command = shlex.split(command) + startd = datetime.datetime.now() + + # pylint: disable=invalid-name + rc, out, err = module.run_command(command, executable=executable, use_unsafe_shell=shell) + + fcntl.flock(lockfd, fcntl.LOCK_UN) + lockfd.close() + + endd = datetime.datetime.now() + delta = endd - startd + + if out is None: + out = '' + if err is None: + err = '' + + module.exit_json( + cmd=command, + stdout=out.rstrip("\r\n"), + stderr=err.rstrip("\r\n"), + rc=rc, + start=str(startd), + end=str(endd), + delta=str(delta), + changed=True, + warnings=warnings, + iterated=iterated + ) + + +# import module snippets +# pylint: disable=wrong-import-position +from ansible.module_utils.basic import * # noqa: F402,F403 + +main() diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml new file mode 100644 index 000000000..879ca4f4e --- /dev/null +++ b/roles/etcd/meta/main.yml @@ -0,0 +1,21 @@ +--- +# This module is based on https://github.com/retr0h/ansible-etcd with most +# changes centered around installing from a pre-existing rpm +# TODO: Extend https://github.com/retr0h/ansible-etcd rather than forking +galaxy_info: + author: Scott Dodson + description: etcd management + company: Red Hat, Inc. + license: Apache License, Version 2.0 + min_ansible_version: 2.2 + platforms: + - name: EL + versions: + - 7 + categories: + - cloud + - system +dependencies: +- role: lib_openshift +- role: lib_os_firewall +- role: lib_utils diff --git a/roles/etcd/tasks/auxiliary/clean_data.yml b/roles/etcd/tasks/auxiliary/clean_data.yml new file mode 100644 index 000000000..1ed2db5bc --- /dev/null +++ b/roles/etcd/tasks/auxiliary/clean_data.yml @@ -0,0 +1,5 @@ +--- +- name: Remove member data + file: + path: "{{ etcd_data_dir }}/member" + state: absent diff --git a/roles/etcd/tasks/auxiliary/disable_etcd.yml b/roles/etcd/tasks/auxiliary/disable_etcd.yml new file mode 100644 index 000000000..7c6d0409d --- /dev/null +++ b/roles/etcd/tasks/auxiliary/disable_etcd.yml @@ -0,0 +1,5 @@ +--- +- name: Disable etcd members + service: + name: "{{ etcd_service }}" + state: stopped diff --git a/roles/etcd/tasks/auxiliary/drop_etcdctl.yml b/roles/etcd/tasks/auxiliary/drop_etcdctl.yml new file mode 100644 index 000000000..11bd2310e --- /dev/null +++ b/roles/etcd/tasks/auxiliary/drop_etcdctl.yml @@ -0,0 +1,12 @@ +--- +- name: Install etcd for etcdctl + package: name=etcd{{ '-' + etcd_version if etcd_version is defined else '' }} state=present + when: not openshift.common.is_atomic | bool + +- name: Configure etcd profile.d aliases + template: + dest: "/etc/profile.d/etcdctl.sh" + src: etcdctl.sh.j2 + mode: 0755 + owner: root + group: root diff --git a/roles/etcd/tasks/auxiliary/force_new_cluster.yml b/roles/etcd/tasks/auxiliary/force_new_cluster.yml new file mode 100644 index 000000000..ae8a36130 --- /dev/null +++ b/roles/etcd/tasks/auxiliary/force_new_cluster.yml @@ -0,0 +1,31 @@ +--- +- name: Set ETCD_FORCE_NEW_CLUSTER=true on first etcd host + lineinfile: + line: "ETCD_FORCE_NEW_CLUSTER=true" + dest: /etc/etcd/etcd.conf + backup: true + +- name: Start etcd + systemd: + name: "{{ etcd_service }}" + state: started + +- name: Wait for cluster to become healthy after bringing up first member + command: > + etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoint https://{{ etcd_peer }}:{{ etcd_client_port }} cluster-health + register: l_etcd_migrate_health + until: l_etcd_migrate_health.rc == 0 + retries: 3 + delay: 30 + +- name: Unset ETCD_FORCE_NEW_CLUSTER=true on first etcd host + lineinfile: + line: "ETCD_FORCE_NEW_CLUSTER=true" + dest: /etc/etcd/etcd.conf + state: absent + backup: true + +- name: Restart first etcd host + systemd: + name: "{{ etcd_service }}" + state: restarted diff --git a/roles/etcd/tasks/backup.archive.yml b/roles/etcd/tasks/backup.archive.yml new file mode 100644 index 000000000..6daa6dc51 --- /dev/null +++ b/roles/etcd/tasks/backup.archive.yml @@ -0,0 +1,3 @@ +--- +- include: backup/vars.yml +- include: backup/archive.yml diff --git a/roles/etcd/tasks/backup.copy.yml b/roles/etcd/tasks/backup.copy.yml new file mode 100644 index 000000000..cc540cbca --- /dev/null +++ b/roles/etcd/tasks/backup.copy.yml @@ -0,0 +1,3 @@ +--- +- include: backup/vars.yml +- include: backup/copy.yml diff --git a/roles/etcd/tasks/backup.fetch.yml b/roles/etcd/tasks/backup.fetch.yml new file mode 100644 index 000000000..26ec15043 --- /dev/null +++ b/roles/etcd/tasks/backup.fetch.yml @@ -0,0 +1,3 @@ +--- +- include: backup/vars.yml +- include: backup/fetch.yml diff --git a/roles/etcd/tasks/backup.force_new_cluster.yml b/roles/etcd/tasks/backup.force_new_cluster.yml new file mode 100644 index 000000000..24bd0540d --- /dev/null +++ b/roles/etcd/tasks/backup.force_new_cluster.yml @@ -0,0 +1,12 @@ +--- +- include: backup/vars.yml + +- name: Move content of etcd backup under the etcd data directory + command: > + mv "{{ l_etcd_backup_dir }}/member" "{{ l_etcd_data_dir }}" + +- name: Set etcd group for the etcd data directory + command: > + chown -R etcd:etcd "{{ l_etcd_data_dir }}" + +- include: auxiliary/force_new_cluster.yml diff --git a/roles/etcd/tasks/backup.unarchive.yml b/roles/etcd/tasks/backup.unarchive.yml new file mode 100644 index 000000000..77a637360 --- /dev/null +++ b/roles/etcd/tasks/backup.unarchive.yml @@ -0,0 +1,3 @@ +--- +- include: backup/vars.yml +- include: backup/unarchive.yml diff --git a/roles/etcd/tasks/backup.yml b/roles/etcd/tasks/backup.yml new file mode 100644 index 000000000..c0538e596 --- /dev/null +++ b/roles/etcd/tasks/backup.yml @@ -0,0 +1,2 @@ +--- +- include: backup/backup.yml diff --git a/roles/etcd/tasks/backup/archive.yml b/roles/etcd/tasks/backup/archive.yml new file mode 100644 index 000000000..f6aa68a6e --- /dev/null +++ b/roles/etcd/tasks/backup/archive.yml @@ -0,0 +1,5 @@ +--- +- name: Archive backup + archive: + path: "{{ l_etcd_backup_dir }}" + dest: "{{ l_etcd_backup_dir }}.tgz" diff --git a/roles/etcd/tasks/backup/backup.yml b/roles/etcd/tasks/backup/backup.yml new file mode 100644 index 000000000..ec1a1989c --- /dev/null +++ b/roles/etcd/tasks/backup/backup.yml @@ -0,0 +1,85 @@ +--- +- include: vars.yml + +# TODO: replace shell module with command and update later checks +- name: Check available disk space for etcd backup + shell: df --output=avail -k {{ l_etcd_data_dir }} | tail -n 1 + register: l_avail_disk + # AUDIT:changed_when: `false` because we are only inspecting + # state, not manipulating anything + changed_when: false + +# TODO: replace shell module with command and update later checks +- name: Check current etcd disk usage + shell: du --exclude='*openshift-backup*' -k {{ l_etcd_data_dir }} | tail -n 1 | cut -f1 + register: l_etcd_disk_usage + # AUDIT:changed_when: `false` because we are only inspecting + # state, not manipulating anything + changed_when: false + +- name: Abort if insufficient disk space for etcd backup + fail: + msg: > + {{ l_etcd_disk_usage.stdout|int*2 }} Kb disk space required for etcd backup, + {{ l_avail_disk.stdout }} Kb available. + when: l_etcd_disk_usage.stdout|int*2 > l_avail_disk.stdout|int + +# For non containerized and non embedded we should have the correct version of +# etcd installed already. So don't do anything. +# +# For containerized installs we now exec into etcd_container +# +# For embedded non containerized we need to ensure we have the latest version +# etcd on the host. +- name: Detecting Atomic Host Operating System + stat: + path: /run/ostree-booted + register: l_ostree_booted + +- name: Install latest etcd for embedded + package: + name: etcd + state: latest + when: + - r_etcd_common_embedded_etcd | bool + - not l_ostree_booted.stat.exists | bool + +- name: Check selinux label of '{{ l_etcd_data_dir }}' + command: > + stat -c '%C' {{ l_etcd_data_dir }} + register: l_etcd_selinux_labels + +- debug: + msg: "{{ l_etcd_selinux_labels }}" + +- name: Make sure the '{{ l_etcd_data_dir }}' has the proper label + command: > + chcon -t svirt_sandbox_file_t "{{ l_etcd_data_dir }}" + when: + - l_etcd_selinux_labels.rc == 0 + - "'svirt_sandbox_file_t' not in l_etcd_selinux_labels.stdout" + +- name: Generate etcd backup + command: > + {{ r_etcd_common_etcdctl_command }} backup --data-dir={{ l_etcd_incontainer_data_dir }} + --backup-dir={{ l_etcd_incontainer_backup_dir }} + +# According to the docs change you can simply copy snap/db +# https://github.com/openshift/openshift-docs/commit/b38042de02d9780842dce95cfa0ef45d53b58bc6 +- name: Check for v3 data store + stat: + path: "{{ l_etcd_data_dir }}/member/snap/db" + register: l_v3_db + +- name: Copy etcd v3 data store + command: > + cp -a {{ l_etcd_data_dir }}/member/snap/db + {{ l_etcd_backup_dir }}/member/snap/ + when: l_v3_db.stat.exists + +- set_fact: + r_etcd_common_backup_complete: True + +- name: Display location of etcd backup + debug: + msg: "Etcd backup created in {{ l_etcd_backup_dir }}" diff --git a/roles/etcd/tasks/backup/copy.yml b/roles/etcd/tasks/backup/copy.yml new file mode 100644 index 000000000..16604bae8 --- /dev/null +++ b/roles/etcd/tasks/backup/copy.yml @@ -0,0 +1,5 @@ +--- +- name: Copy etcd backup + copy: + src: "{{ etcd_backup_sync_directory }}/{{ l_backup_dir_name }}.tgz" + dest: "{{ l_etcd_data_dir }}" diff --git a/roles/etcd/tasks/backup/fetch.yml b/roles/etcd/tasks/backup/fetch.yml new file mode 100644 index 000000000..610ce1960 --- /dev/null +++ b/roles/etcd/tasks/backup/fetch.yml @@ -0,0 +1,8 @@ +--- +- name: Fetch etcd backup + fetch: + src: "{{ l_etcd_backup_dir }}.tgz" + dest: "{{ etcd_backup_sync_directory }}/" + flat: yes + fail_on_missing: yes + validate_checksum: yes diff --git a/roles/etcd/tasks/backup/unarchive.yml b/roles/etcd/tasks/backup/unarchive.yml new file mode 100644 index 000000000..6c75d00a7 --- /dev/null +++ b/roles/etcd/tasks/backup/unarchive.yml @@ -0,0 +1,14 @@ +--- +- shell: ls /var/lib/etcd + register: output + +- debug: + msg: "output: {{ output }}" + +- name: Unarchive backup + # can't use unarchive https://github.com/ansible/ansible/issues/30821 + # unarchive: + # src: "{{ l_etcd_backup_dir }}.tgz" + # dest: "{{ l_etcd_backup_dir }}" + command: > + tar -xf "{{ l_etcd_backup_dir }}.tgz" -C "{{ l_etcd_data_dir }}" diff --git a/roles/etcd/tasks/backup/vars.yml b/roles/etcd/tasks/backup/vars.yml new file mode 100644 index 000000000..3c009f557 --- /dev/null +++ b/roles/etcd/tasks/backup/vars.yml @@ -0,0 +1,18 @@ +--- +# set the etcd backup directory name here in case the tag or sufix consists of dynamic value that changes over time +# e.g. openshift-backup-{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }} value will change every second so if the date changes +# right after setting l_etcd_incontainer_backup_dir and before l_etcd_backup_dir facts, the backup directory name is different +- set_fact: + l_backup_dir_name: "openshift-backup-{{ r_etcd_common_backup_tag }}{{ r_etcd_common_backup_sufix_name }}" + +- set_fact: + l_etcd_data_dir: "{{ etcd_data_dir }}{{ '/etcd.etcd' if r_etcd_common_etcd_runtime == 'runc' else '' }}" + +- set_fact: + l_etcd_incontainer_data_dir: "{{ etcd_data_dir }}" + +- set_fact: + l_etcd_incontainer_backup_dir: "{{ l_etcd_incontainer_data_dir }}/{{ l_backup_dir_name }}" + +- set_fact: + l_etcd_backup_dir: "{{ l_etcd_data_dir }}/{{ l_backup_dir_name }}" diff --git a/roles/etcd/tasks/backup_ca_certificates.yml b/roles/etcd/tasks/backup_ca_certificates.yml new file mode 100644 index 000000000..a41b032f3 --- /dev/null +++ b/roles/etcd/tasks/backup_ca_certificates.yml @@ -0,0 +1,2 @@ +--- +- include: certificates/backup_ca_certificates.yml diff --git a/roles/etcd/tasks/backup_generated_certificates.yml b/roles/etcd/tasks/backup_generated_certificates.yml new file mode 100644 index 000000000..8cf2a10cc --- /dev/null +++ b/roles/etcd/tasks/backup_generated_certificates.yml @@ -0,0 +1,2 @@ +--- +- include: certificates/backup_generated_certificates.yml diff --git a/roles/etcd/tasks/backup_master_etcd_certificates.yml b/roles/etcd/tasks/backup_master_etcd_certificates.yml new file mode 100644 index 000000000..129e1831c --- /dev/null +++ b/roles/etcd/tasks/backup_master_etcd_certificates.yml @@ -0,0 +1,2 @@ +--- +- include: certificates/backup_master_etcd_certificates.yml diff --git a/roles/etcd/tasks/backup_server_certificates.yml b/roles/etcd/tasks/backup_server_certificates.yml new file mode 100644 index 000000000..267ffeb4d --- /dev/null +++ b/roles/etcd/tasks/backup_server_certificates.yml @@ -0,0 +1,2 @@ +--- +- include: certificates/backup_server_certificates.yml diff --git a/roles/etcd/tasks/ca.yml b/roles/etcd/tasks/ca.yml new file mode 100644 index 000000000..cca1e9ad7 --- /dev/null +++ b/roles/etcd/tasks/ca.yml @@ -0,0 +1,2 @@ +--- +- include: certificates/deploy_ca.yml diff --git a/roles/etcd/tasks/certificates/backup_ca_certificates.yml b/roles/etcd/tasks/certificates/backup_ca_certificates.yml new file mode 100644 index 000000000..f60eb82ef --- /dev/null +++ b/roles/etcd/tasks/certificates/backup_ca_certificates.yml @@ -0,0 +1,12 @@ +--- +- name: Determine if CA certificate directory exists + stat: + path: "{{ etcd_ca_dir }}" + register: etcd_ca_certs_dir_stat +- name: Backup generated etcd certificates + command: > + tar -czf {{ etcd_conf_dir }}/etcd-ca-certificate-backup-{{ ansible_date_time.epoch }}.tgz + {{ etcd_ca_dir }} + args: + warn: no + when: etcd_ca_certs_dir_stat.stat.exists | bool diff --git a/roles/etcd/tasks/certificates/backup_generated_certificates.yml b/roles/etcd/tasks/certificates/backup_generated_certificates.yml new file mode 100644 index 000000000..6a24cfcb3 --- /dev/null +++ b/roles/etcd/tasks/certificates/backup_generated_certificates.yml @@ -0,0 +1,13 @@ +--- +- name: Determine if generated etcd certificates exist + stat: + path: "{{ etcd_conf_dir }}/generated_certs" + register: etcd_generated_certs_dir_stat + +- name: Backup generated etcd certificates + command: > + tar -czf {{ etcd_conf_dir }}/etcd-generated-certificate-backup-{{ ansible_date_time.epoch }}.tgz + {{ etcd_conf_dir }}/generated_certs + args: + warn: no + when: etcd_generated_certs_dir_stat.stat.exists | bool diff --git a/roles/etcd/tasks/certificates/backup_master_etcd_certificates.yml b/roles/etcd/tasks/certificates/backup_master_etcd_certificates.yml new file mode 100644 index 000000000..e65b3e5a2 --- /dev/null +++ b/roles/etcd/tasks/certificates/backup_master_etcd_certificates.yml @@ -0,0 +1,7 @@ +--- +- name: Backup master etcd certificates + shell: > + tar -czvf /etc/origin/master/master-etcd-certificate-backup-{{ ansible_date_time.epoch }}.tgz + /etc/origin/master/master.etcd-* + args: + warn: no diff --git a/roles/etcd/tasks/certificates/backup_server_certificates.yml b/roles/etcd/tasks/certificates/backup_server_certificates.yml new file mode 100644 index 000000000..8e6cc6965 --- /dev/null +++ b/roles/etcd/tasks/certificates/backup_server_certificates.yml @@ -0,0 +1,11 @@ +--- +- name: Backup etcd certificates + command: > + tar -czvf /etc/etcd/etcd-server-certificate-backup-{{ ansible_date_time.epoch }}.tgz + {{ etcd_conf_dir }}/ca.crt + {{ etcd_conf_dir }}/server.crt + {{ etcd_conf_dir }}/server.key + {{ etcd_conf_dir }}/peer.crt + {{ etcd_conf_dir }}/peer.key + args: + warn: no diff --git a/roles/etcd/tasks/certificates/deploy_ca.yml b/roles/etcd/tasks/certificates/deploy_ca.yml new file mode 100644 index 000000000..3d32290a2 --- /dev/null +++ b/roles/etcd/tasks/certificates/deploy_ca.yml @@ -0,0 +1,78 @@ +--- +- name: Install openssl + package: + name: openssl + state: present + when: not etcd_is_atomic | bool + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- file: + path: "{{ item }}" + state: directory + mode: 0700 + owner: root + group: root + with_items: + - "{{ etcd_ca_new_certs_dir }}" + - "{{ etcd_ca_crl_dir }}" + - "{{ etcd_ca_dir }}/fragments" + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- command: cp /etc/pki/tls/openssl.cnf ./ + args: + chdir: "{{ etcd_ca_dir }}/fragments" + creates: "{{ etcd_ca_dir }}/fragments/openssl.cnf" + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- template: + dest: "{{ etcd_ca_dir }}/fragments/openssl_append.cnf" + src: openssl_append.j2 + backup: true + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- assemble: + src: "{{ etcd_ca_dir }}/fragments" + dest: "{{ etcd_openssl_conf }}" + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- name: Check etcd_ca_db exist + stat: path="{{ etcd_ca_db }}" + register: etcd_ca_db_check + changed_when: false + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- name: Touch etcd_ca_db file + file: + path: "{{ etcd_ca_db }}" + state: touch + when: etcd_ca_db_check.stat.isreg is not defined + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- copy: + dest: "{{ etcd_ca_serial }}" + content: "01" + force: no + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- name: Create etcd CA certificate + command: > + openssl req -config {{ etcd_openssl_conf }} -newkey rsa:4096 + -keyout {{ etcd_ca_key }} -new -out {{ etcd_ca_cert }} + -x509 -extensions {{ etcd_ca_exts_self }} -batch -nodes + -days {{ etcd_ca_default_days }} + -subj /CN=etcd-signer@{{ ansible_date_time.epoch }} + args: + chdir: "{{ etcd_ca_dir }}" + creates: "{{ etcd_ca_cert }}" + environment: + SAN: 'etcd-signer' + delegate_to: "{{ etcd_ca_host }}" + run_once: true diff --git a/roles/etcd/tasks/certificates/distribute_ca.yml b/roles/etcd/tasks/certificates/distribute_ca.yml new file mode 100644 index 000000000..632ac15dd --- /dev/null +++ b/roles/etcd/tasks/certificates/distribute_ca.yml @@ -0,0 +1,47 @@ +--- +- name: Create a tarball of the etcd ca certs + command: > + tar -czvf {{ etcd_conf_dir }}/{{ etcd_ca_name }}.tgz + -C {{ etcd_ca_dir }} . + args: + creates: "{{ etcd_conf_dir }}/{{ etcd_ca_name }}.tgz" + warn: no + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- name: Retrieve etcd ca cert tarball + fetch: + src: "{{ etcd_conf_dir }}/{{ etcd_ca_name }}.tgz" + dest: "{{ etcd_sync_cert_dir }}/" + flat: yes + fail_on_missing: yes + validate_checksum: yes + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- name: Ensure ca directory exists + file: + path: "{{ etcd_ca_dir }}" + state: directory + +- name: Unarchive etcd ca cert tarballs + unarchive: + src: "{{ etcd_sync_cert_dir }}/{{ etcd_ca_name }}.tgz" + dest: "{{ etcd_ca_dir }}" + +- name: Read current etcd CA + slurp: + src: "{{ etcd_conf_dir }}/ca.crt" + register: g_current_etcd_ca_output + +- name: Read new etcd CA + slurp: + src: "{{ etcd_ca_dir }}/ca.crt" + register: g_new_etcd_ca_output + +- copy: + content: "{{ (g_new_etcd_ca_output.content|b64decode) + (g_current_etcd_ca_output.content|b64decode) }}" + dest: "{{ item }}/ca.crt" + with_items: + - "{{ etcd_conf_dir }}" + - "{{ etcd_ca_dir }}" diff --git a/roles/etcd/tasks/certificates/fetch_client_certificates_from_ca.yml b/roles/etcd/tasks/certificates/fetch_client_certificates_from_ca.yml new file mode 100644 index 000000000..119071a72 --- /dev/null +++ b/roles/etcd/tasks/certificates/fetch_client_certificates_from_ca.yml @@ -0,0 +1,138 @@ +--- +- name: Ensure CA certificate exists on etcd_ca_host + stat: + path: "{{ etcd_ca_cert }}" + register: g_ca_cert_stat_result + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- fail: + msg: > + CA certificate {{ etcd_ca_cert }} doesn't exist on CA host + {{ etcd_ca_host }}. Apply 'etcd_ca' action from `etcd` role to + {{ etcd_ca_host }}. + when: not g_ca_cert_stat_result.stat.exists | bool + run_once: true + +- name: Check status of external etcd certificatees + stat: + path: "{{ etcd_cert_config_dir }}/{{ item }}" + with_items: + - "{{ etcd_cert_prefix }}client.crt" + - "{{ etcd_cert_prefix }}client.key" + - "{{ etcd_cert_prefix }}ca.crt" + register: g_external_etcd_cert_stat_result + when: not etcd_certificates_redeploy | default(false) | bool + +- set_fact: + etcd_client_certs_missing: "{{ true if etcd_certificates_redeploy | default(false) | bool + else (False in (g_external_etcd_cert_stat_result.results + | default({}) + | oo_collect(attribute='stat.exists') + | list)) }}" + +- name: Ensure generated_certs directory present + file: + path: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}" + state: directory + mode: 0700 + when: etcd_client_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Create the client csr + command: > + openssl req -new -keyout {{ etcd_cert_prefix }}client.key + -config {{ etcd_openssl_conf }} + -out {{ etcd_cert_prefix }}client.csr + -reqexts {{ etcd_req_ext }} -batch -nodes + -subj /CN={{ etcd_hostname }} + args: + chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}" + creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/' + ~ etcd_cert_prefix ~ 'client.csr' }}" + environment: + SAN: "IP:{{ etcd_ip }},DNS:{{ etcd_hostname }}" + when: etcd_client_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +# Certificates must be signed serially in order to avoid competing +# for the serial file. +- name: Sign and create the client crt + delegated_serial_command: + command: > + openssl ca -name {{ etcd_ca_name }} -config {{ etcd_openssl_conf }} + -out {{ etcd_cert_prefix }}client.crt + -in {{ etcd_cert_prefix }}client.csr + -batch + chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}" + creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/' + ~ etcd_cert_prefix ~ 'client.crt' }}" + environment: + SAN: "IP:{{ etcd_ip }}" + when: etcd_client_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- file: + src: "{{ etcd_ca_cert }}" + dest: "{{ etcd_generated_certs_dir}}/{{ etcd_cert_subdir }}/{{ etcd_cert_prefix }}ca.crt" + state: hard + when: etcd_client_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Create local temp directory for syncing certs + local_action: command mktemp -d /tmp/etcd_certificates-XXXXXXX + register: g_etcd_client_mktemp + changed_when: False + when: etcd_client_certs_missing | bool + become: no + +- name: Create a tarball of the etcd certs + command: > + tar -czvf {{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz + -C {{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }} . + args: + creates: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz" + # Disables the following warning: + # Consider using unarchive module rather than running tar + warn: no + when: etcd_client_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Retrieve the etcd cert tarballs + fetch: + src: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz" + dest: "{{ g_etcd_client_mktemp.stdout }}/" + flat: yes + fail_on_missing: yes + validate_checksum: yes + when: etcd_client_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Ensure certificate directory exists + file: + path: "{{ etcd_cert_config_dir }}" + state: directory + when: etcd_client_certs_missing | bool + +- name: Unarchive etcd cert tarballs + unarchive: + src: "{{ g_etcd_client_mktemp.stdout }}/{{ etcd_cert_subdir }}.tgz" + dest: "{{ etcd_cert_config_dir }}" + when: etcd_client_certs_missing | bool + +- file: + path: "{{ etcd_cert_config_dir }}/{{ item }}" + owner: root + group: root + mode: 0600 + with_items: + - "{{ etcd_cert_prefix }}client.crt" + - "{{ etcd_cert_prefix }}client.key" + - "{{ etcd_cert_prefix }}ca.crt" + when: etcd_client_certs_missing | bool + +- name: Delete temporary directory + local_action: file path="{{ g_etcd_client_mktemp.stdout }}" state=absent + changed_when: False + when: etcd_client_certs_missing | bool + become: no diff --git a/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml b/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml new file mode 100644 index 000000000..26492fb3c --- /dev/null +++ b/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml @@ -0,0 +1,234 @@ +--- +- name: Install etcd + package: + name: "etcd{{ '-' + etcd_version if etcd_version is defined else '' }}" + state: present + when: not etcd_is_containerized | bool + +- name: Check status of etcd certificates + stat: + path: "{{ item }}" + with_items: + - "{{ etcd_cert_config_dir }}/{{ etcd_cert_prefix }}server.crt" + - "{{ etcd_cert_config_dir }}/{{ etcd_cert_prefix }}peer.crt" + - "{{ etcd_cert_config_dir }}/{{ etcd_cert_prefix }}ca.crt" + - "{{ etcd_system_container_cert_config_dir }}/{{ etcd_cert_prefix }}server.crt" + - "{{ etcd_system_container_cert_config_dir }}/{{ etcd_cert_prefix }}peer.crt" + - "{{ etcd_system_container_cert_config_dir }}/{{ etcd_cert_prefix }}ca.crt" + register: g_etcd_server_cert_stat_result + when: not etcd_certificates_redeploy | default(false) | bool + +- set_fact: + etcd_server_certs_missing: "{{ true if etcd_certificates_redeploy | default(false) | bool + else (False in (g_etcd_server_cert_stat_result.results + | default({}) + | oo_collect(attribute='stat.exists') + | list)) }}" + +- name: Ensure generated_certs directory present + file: + path: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}" + state: directory + mode: 0700 + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Create the server csr + command: > + openssl req -new -keyout {{ etcd_cert_prefix }}server.key + -config {{ etcd_openssl_conf }} + -out {{ etcd_cert_prefix }}server.csr + -reqexts {{ etcd_req_ext }} -batch -nodes + -subj /CN={{ etcd_hostname }} + args: + chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}" + creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/' + ~ etcd_cert_prefix ~ 'server.csr' }}" + environment: + SAN: "IP:{{ etcd_ip }},DNS:{{ etcd_hostname }}" + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +# Certificates must be signed serially in order to avoid competing +# for the serial file. +- name: Sign and create the server crt + delegated_serial_command: + command: > + openssl ca -name {{ etcd_ca_name }} -config {{ etcd_openssl_conf }} + -out {{ etcd_cert_prefix }}server.crt + -in {{ etcd_cert_prefix }}server.csr + -extensions {{ etcd_ca_exts_server }} -batch + chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}" + creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/' + ~ etcd_cert_prefix ~ 'server.crt' }}" + environment: + SAN: "IP:{{ etcd_ip }}" + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Create the peer csr + command: > + openssl req -new -keyout {{ etcd_cert_prefix }}peer.key + -config {{ etcd_openssl_conf }} + -out {{ etcd_cert_prefix }}peer.csr + -reqexts {{ etcd_req_ext }} -batch -nodes + -subj /CN={{ etcd_hostname }} + args: + chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}" + creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/' + ~ etcd_cert_prefix ~ 'peer.csr' }}" + environment: + SAN: "IP:{{ etcd_ip }},DNS:{{ etcd_hostname }}" + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +# Certificates must be signed serially in order to avoid competing +# for the serial file. +- name: Sign and create the peer crt + delegated_serial_command: + command: > + openssl ca -name {{ etcd_ca_name }} -config {{ etcd_openssl_conf }} + -out {{ etcd_cert_prefix }}peer.crt + -in {{ etcd_cert_prefix }}peer.csr + -extensions {{ etcd_ca_exts_peer }} -batch + chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}" + creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/' + ~ etcd_cert_prefix ~ 'peer.crt' }}" + environment: + SAN: "IP:{{ etcd_ip }}" + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- file: + src: "{{ etcd_ca_cert }}" + dest: "{{ etcd_generated_certs_dir}}/{{ etcd_cert_subdir }}/{{ etcd_cert_prefix }}ca.crt" + state: hard + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Create local temp directory for syncing certs + local_action: command mktemp -d /tmp/etcd_certificates-XXXXXXX + become: no + register: g_etcd_server_mktemp + changed_when: False + when: etcd_server_certs_missing | bool + +- name: Create a tarball of the etcd certs + command: > + tar -czvf {{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz + -C {{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }} . + args: + creates: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz" + # Disables the following warning: + # Consider using unarchive module rather than running tar + warn: no + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Retrieve etcd cert tarball + fetch: + src: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz" + dest: "{{ g_etcd_server_mktemp.stdout }}/" + flat: yes + fail_on_missing: yes + validate_checksum: yes + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Ensure certificate directory exists + file: + path: "{{ item }}" + state: directory + with_items: + - "{{ etcd_cert_config_dir }}" + - "{{ etcd_system_container_cert_config_dir }}" + when: etcd_server_certs_missing | bool + +- name: Unarchive cert tarball + unarchive: + src: "{{ g_etcd_server_mktemp.stdout }}/{{ etcd_cert_subdir }}.tgz" + dest: "{{ etcd_cert_config_dir }}" + when: etcd_server_certs_missing | bool + +- name: Create a tarball of the etcd ca certs + command: > + tar -czvf {{ etcd_generated_certs_dir }}/{{ etcd_ca_name }}.tgz + -C {{ etcd_ca_dir }} . + args: + creates: "{{ etcd_generated_certs_dir }}/{{ etcd_ca_name }}.tgz" + warn: no + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Retrieve etcd ca cert tarball + fetch: + src: "{{ etcd_generated_certs_dir }}/{{ etcd_ca_name }}.tgz" + dest: "{{ g_etcd_server_mktemp.stdout }}/" + flat: yes + fail_on_missing: yes + validate_checksum: yes + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Ensure ca directory exists + file: + path: "{{ item }}" + state: directory + with_items: + - "{{ etcd_ca_dir }}" + - "{{ etcd_system_container_cert_config_dir }}/ca" + when: etcd_server_certs_missing | bool + +- name: Unarchive cert tarball for the system container + unarchive: + src: "{{ g_etcd_server_mktemp.stdout }}/{{ etcd_cert_subdir }}.tgz" + dest: "{{ etcd_system_container_cert_config_dir }}" + when: + - etcd_server_certs_missing | bool + - r_etcd_common_etcd_runtime == 'runc' + +- name: Unarchive etcd ca cert tarballs for the system container + unarchive: + src: "{{ g_etcd_server_mktemp.stdout }}/{{ etcd_ca_name }}.tgz" + dest: "{{ etcd_system_container_cert_config_dir }}/ca" + when: + - etcd_server_certs_missing | bool + - r_etcd_common_etcd_runtime == 'runc' + +- name: Delete temporary directory + local_action: file path="{{ g_etcd_server_mktemp.stdout }}" state=absent + become: no + changed_when: False + when: etcd_server_certs_missing | bool + +- name: Validate permissions on certificate files + file: + path: "{{ item }}" + mode: 0600 + owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}" + group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}" + when: etcd_url_scheme == 'https' + with_items: + - "{{ etcd_ca_file }}" + - "{{ etcd_cert_file }}" + - "{{ etcd_key_file }}" + +- name: Validate permissions on peer certificate files + file: + path: "{{ item }}" + mode: 0600 + owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}" + group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}" + when: etcd_peer_url_scheme == 'https' + with_items: + - "{{ etcd_peer_ca_file }}" + - "{{ etcd_peer_cert_file }}" + - "{{ etcd_peer_key_file }}" + +- name: Validate permissions on the config dir + file: + path: "{{ etcd_conf_dir }}" + state: directory + owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}" + group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}" + mode: 0700 diff --git a/roles/etcd/tasks/certificates/remove_ca_certificates.yml b/roles/etcd/tasks/certificates/remove_ca_certificates.yml new file mode 100644 index 000000000..4a86eb60d --- /dev/null +++ b/roles/etcd/tasks/certificates/remove_ca_certificates.yml @@ -0,0 +1,5 @@ +--- +- name: Remove CA certificate directory + file: + path: "{{ etcd_ca_dir }}" + state: absent diff --git a/roles/etcd/tasks/certificates/remove_generated_certificates.yml b/roles/etcd/tasks/certificates/remove_generated_certificates.yml new file mode 100644 index 000000000..993b18de2 --- /dev/null +++ b/roles/etcd/tasks/certificates/remove_generated_certificates.yml @@ -0,0 +1,5 @@ +--- +- name: Remove generated etcd certificates + file: + path: "{{ etcd_conf_dir }}/generated_certs" + state: absent diff --git a/roles/etcd/tasks/certificates/retrieve_ca_certificates.yml b/roles/etcd/tasks/certificates/retrieve_ca_certificates.yml new file mode 100644 index 000000000..70b5c6523 --- /dev/null +++ b/roles/etcd/tasks/certificates/retrieve_ca_certificates.yml @@ -0,0 +1,8 @@ +--- +- name: Retrieve etcd CA certificate + fetch: + src: "{{ etcd_conf_dir }}/ca.crt" + dest: "{{ etcd_sync_cert_dir }}/" + flat: yes + fail_on_missing: yes + validate_checksum: yes diff --git a/roles/etcd/tasks/check_cluster_health.yml b/roles/etcd/tasks/check_cluster_health.yml new file mode 100644 index 000000000..75c110972 --- /dev/null +++ b/roles/etcd/tasks/check_cluster_health.yml @@ -0,0 +1,2 @@ +--- +- include: migration/check_cluster_health.yml diff --git a/roles/etcd/tasks/clean_data.yml b/roles/etcd/tasks/clean_data.yml new file mode 100644 index 000000000..d131ffd21 --- /dev/null +++ b/roles/etcd/tasks/clean_data.yml @@ -0,0 +1,2 @@ +--- +- include: auxiliary/clean_data.yml diff --git a/roles/etcd/tasks/client_certificates.yml b/roles/etcd/tasks/client_certificates.yml new file mode 100644 index 000000000..2f4108a0d --- /dev/null +++ b/roles/etcd/tasks/client_certificates.yml @@ -0,0 +1,2 @@ +--- +- include: certificates/fetch_client_certificates_from_ca.yml diff --git a/roles/etcd/tasks/disable_etcd.yml b/roles/etcd/tasks/disable_etcd.yml new file mode 100644 index 000000000..9202e6e48 --- /dev/null +++ b/roles/etcd/tasks/disable_etcd.yml @@ -0,0 +1,2 @@ +--- +- include: auxiliary/disable_etcd.yml diff --git a/roles/etcd/tasks/distribute_ca b/roles/etcd/tasks/distribute_ca new file mode 100644 index 000000000..040c5f7af --- /dev/null +++ b/roles/etcd/tasks/distribute_ca @@ -0,0 +1,2 @@ +--- +- include: certificates/distribute_ca.yml diff --git a/roles/etcd/tasks/drop_etcdctl.yml b/roles/etcd/tasks/drop_etcdctl.yml new file mode 100644 index 000000000..4c1f609f7 --- /dev/null +++ b/roles/etcd/tasks/drop_etcdctl.yml @@ -0,0 +1,2 @@ +--- +- include: auxiliary/drop_etcdctl.yml diff --git a/roles/etcd/tasks/fetch_backup.yml b/roles/etcd/tasks/fetch_backup.yml new file mode 100644 index 000000000..513eed17a --- /dev/null +++ b/roles/etcd/tasks/fetch_backup.yml @@ -0,0 +1,8 @@ +--- +- include: backup/vars.yml + +- include: backup/archive.yml + +- include: backup/sync_backup.yml + +- include: backup/ diff --git a/roles/etcd/tasks/firewall.yml b/roles/etcd/tasks/firewall.yml new file mode 100644 index 000000000..4d0f6290a --- /dev/null +++ b/roles/etcd/tasks/firewall.yml @@ -0,0 +1,40 @@ +--- +- when: r_etcd_firewall_enabled | bool and not r_etcd_use_firewalld | bool + block: + - name: Add iptables allow rules + os_firewall_manage_iptables: + name: "{{ item.service }}" + action: add + protocol: "{{ item.port.split('/')[1] }}" + port: "{{ item.port.split('/')[0] }}" + when: item.cond | default(True) + with_items: "{{ r_etcd_os_firewall_allow }}" + + - name: Remove iptables rules + os_firewall_manage_iptables: + name: "{{ item.service }}" + action: remove + protocol: "{{ item.port.split('/')[1] }}" + port: "{{ item.port.split('/')[0] }}" + when: item.cond | default(True) + with_items: "{{ r_etcd_os_firewall_deny }}" + +- when: r_etcd_firewall_enabled | bool and r_etcd_use_firewalld | bool + block: + - name: Add firewalld allow rules + firewalld: + port: "{{ item.port }}" + permanent: true + immediate: true + state: enabled + when: item.cond | default(True) + with_items: "{{ r_etcd_os_firewall_allow }}" + + - name: Remove firewalld allow rules + firewalld: + port: "{{ item.port }}" + permanent: true + immediate: true + state: disabled + when: item.cond | default(True) + with_items: "{{ r_etcd_os_firewall_deny }}" diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml new file mode 100644 index 000000000..3e69af314 --- /dev/null +++ b/roles/etcd/tasks/main.yml @@ -0,0 +1,133 @@ +--- +- name: Set hostname and ip facts + set_fact: + # Store etcd_hostname and etcd_ip such that they will be available + # in hostvars. Defaults for these variables are set in etcd_common. + etcd_hostname: "{{ etcd_hostname }}" + etcd_ip: "{{ etcd_ip }}" + +- name: setup firewall + include: firewall.yml + static: yes + +- name: Install etcd + package: name=etcd{{ '-' + etcd_version if etcd_version is defined else '' }} state=present + when: not etcd_is_containerized | bool + +- include: drop_etcdctl.yml + when: + - openshift_etcd_etcdctl_profile | default(true) | bool + +- block: + - name: Pull etcd container + command: docker pull {{ openshift.etcd.etcd_image }} + register: pull_result + changed_when: "'Downloaded newer image' in pull_result.stdout" + + - name: Install etcd container service file + template: + dest: "/etc/systemd/system/etcd_container.service" + src: etcd.docker.service + when: + - etcd_is_containerized | bool + - not openshift.common.is_etcd_system_container | bool + +# Start secondary etcd instance for third party integrations +# TODO: Determine an alternative to using thirdparty variable +- block: + - name: Create configuration directory + file: + path: "{{ etcd_conf_dir }}" + state: directory + mode: 0700 + + # TODO: retest with symlink to confirm it does or does not function + - name: Copy service file for etcd instance + copy: + src: /usr/lib/systemd/system/etcd.service + dest: "/etc/systemd/system/{{ etcd_service }}.service" + remote_src: True + + - name: Create third party etcd service.d directory exists + file: + path: "{{ etcd_systemd_dir }}" + state: directory + + - name: Configure third part etcd service unit file + template: + dest: "{{ etcd_systemd_dir }}/custom.conf" + src: custom.conf.j2 + when: etcd_is_thirdparty + + # TODO: this task may not be needed with Validate permissions +- name: Ensure etcd datadir exists + file: + path: "{{ etcd_data_dir }}" + state: directory + mode: 0700 + when: etcd_is_containerized | bool + +- name: Ensure etcd datadir ownership for thirdparty datadir + file: + path: "{{ etcd_data_dir }}" + state: directory + mode: 0700 + owner: etcd + group: etcd + recurse: True + when: etcd_is_thirdparty | bool + + # TODO: Determine if the below reload would work here, for now just reload +- name: + command: systemctl daemon-reload + when: etcd_is_thirdparty | bool + +- block: + - name: Disable system etcd when containerized + systemd: + name: etcd + state: stopped + enabled: no + masked: yes + daemon_reload: yes + when: not openshift.common.is_etcd_system_container | bool + register: task_result + failed_when: task_result|failed and 'could not' not in task_result.msg|lower + + - name: Install etcd container service file + template: + dest: "/etc/systemd/system/etcd_container.service" + src: etcd.docker.service + when: not openshift.common.is_etcd_system_container | bool + + - name: Install Etcd system container + include: system_container.yml + when: openshift.common.is_etcd_system_container | bool + when: etcd_is_containerized | bool + +- name: Validate permissions on the config dir + file: + path: "{{ etcd_conf_dir }}" + state: directory + owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}" + group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}" + mode: 0700 + +- name: Write etcd global config file + template: + src: etcd.conf.j2 + dest: "{{ etcd_conf_file }}" + backup: true + notify: + - restart etcd + +- name: Enable etcd + systemd: + name: "{{ etcd_service }}" + state: started + enabled: yes + register: start_result + +- name: Set fact etcd_service_status_changed + set_fact: + etcd_service_status_changed: "{{ start_result | changed }}" diff --git a/roles/etcd/tasks/migrate.add_ttls.yml b/roles/etcd/tasks/migrate.add_ttls.yml new file mode 100644 index 000000000..bc27e4ea1 --- /dev/null +++ b/roles/etcd/tasks/migrate.add_ttls.yml @@ -0,0 +1,2 @@ +--- +- include: migration/add_ttls.yml diff --git a/roles/etcd/tasks/migrate.configure_master.yml b/roles/etcd/tasks/migrate.configure_master.yml new file mode 100644 index 000000000..3ada6e362 --- /dev/null +++ b/roles/etcd/tasks/migrate.configure_master.yml @@ -0,0 +1,2 @@ +--- +- include: migration/configure_master.yml diff --git a/roles/etcd/tasks/migrate.pre_check.yml b/roles/etcd/tasks/migrate.pre_check.yml new file mode 100644 index 000000000..124d21561 --- /dev/null +++ b/roles/etcd/tasks/migrate.pre_check.yml @@ -0,0 +1,2 @@ +--- +- include: migration/check.yml diff --git a/roles/etcd/tasks/migrate.yml b/roles/etcd/tasks/migrate.yml new file mode 100644 index 000000000..5d5385873 --- /dev/null +++ b/roles/etcd/tasks/migrate.yml @@ -0,0 +1,2 @@ +--- +- include: migration/migrate.yml diff --git a/roles/etcd/tasks/migration/add_ttls.yml b/roles/etcd/tasks/migration/add_ttls.yml new file mode 100644 index 000000000..14625e49e --- /dev/null +++ b/roles/etcd/tasks/migration/add_ttls.yml @@ -0,0 +1,34 @@ +--- +# To be executed on first master +- slurp: + src: "{{ openshift.common.config_base }}/master/master-config.yaml" + register: g_master_config_output + +- set_fact: + accessTokenMaxAgeSeconds: "{{ (g_master_config_output.content|b64decode|from_yaml).oauthConfig.tokenConfig.accessTokenMaxAgeSeconds | default(86400) }}" + authroizeTokenMaxAgeSeconds: "{{ (g_master_config_output.content|b64decode|from_yaml).oauthConfig.tokenConfig.authroizeTokenMaxAgeSeconds | default(500) }}" + controllerLeaseTTL: "{{ (g_master_config_output.content|b64decode|from_yaml).controllerLeaseTTL | default(30) }}" + +- name: Re-introduce leases (as a replacement for key TTLs) + command: > + oadm migrate etcd-ttl \ + --cert {{ r_etcd_common_master_peer_cert_file }} \ + --key {{ r_etcd_common_master_peer_key_file }} \ + --cacert {{ r_etcd_common_master_peer_ca_file }} \ + --etcd-address 'https://{{ etcd_peer }}:{{ etcd_client_port }}' \ + --ttl-keys-prefix {{ item.keys }} \ + --lease-duration {{ item.ttl }} + environment: + ETCDCTL_API: 3 + PATH: "/usr/local/bin:/var/usrlocal/bin:{{ ansible_env.PATH }}" + with_items: + - keys: "/kubernetes.io/events" + ttl: "1h" + - keys: "/kubernetes.io/masterleases" + ttl: "10s" + - keys: "/openshift.io/oauth/accesstokens" + ttl: "{{ accessTokenMaxAgeSeconds }}s" + - keys: "/openshift.io/oauth/authorizetokens" + ttl: "{{ authroizeTokenMaxAgeSeconds }}s" + - keys: "/openshift.io/leases/controllers" + ttl: "{{ controllerLeaseTTL }}s" diff --git a/roles/etcd/tasks/migration/check.yml b/roles/etcd/tasks/migration/check.yml new file mode 100644 index 000000000..0804d9e1c --- /dev/null +++ b/roles/etcd/tasks/migration/check.yml @@ -0,0 +1,56 @@ +--- + +# Check the cluster is healthy +- include: check_cluster_health.yml + +# Check if the member has v3 data already +# Run the migration only if the data are v2 +- name: Check if there are any v3 data + command: > + etcdctl --cert {{ etcd_peer_cert_file }} --key {{ etcd_peer_key_file }} --cacert {{ etcd_peer_ca_file }} --endpoints 'https://{{ etcd_peer }}:{{ etcd_client_port }}' get "" --from-key --keys-only -w json --limit 1 + environment: + ETCDCTL_API: 3 + register: l_etcdctl_output + +- fail: + msg: "Unable to get a number of v3 keys" + when: l_etcdctl_output.rc != 0 + +- fail: + msg: "The etcd has at least one v3 key" + when: "'count' in (l_etcdctl_output.stdout | from_json) and (l_etcdctl_output.stdout | from_json).count != 0" + + +# TODO(jchaloup): once the until loop can be used over include/block, +# remove the repetive code +# - until loop not supported over include statement (nor block) +# https://github.com/ansible/ansible/issues/17098 +# - with_items not supported over block + +# Check the cluster status for the first time +- include: check_cluster_status.yml + +# Check the cluster status for the second time +- block: + - debug: + msg: "l_etcd_cluster_status_ok: {{ l_etcd_cluster_status_ok }}" + - name: Wait a while before another check + pause: + seconds: 5 + when: not l_etcd_cluster_status_ok | bool + + - include: check_cluster_status.yml + when: not l_etcd_cluster_status_ok | bool + + +# Check the cluster status for the third time +- block: + - debug: + msg: "l_etcd_cluster_status_ok: {{ l_etcd_cluster_status_ok }}" + - name: Wait a while before another check + pause: + seconds: 5 + when: not l_etcd_cluster_status_ok | bool + + - include: check_cluster_status.yml + when: not l_etcd_cluster_status_ok | bool diff --git a/roles/etcd/tasks/migration/check_cluster_health.yml b/roles/etcd/tasks/migration/check_cluster_health.yml new file mode 100644 index 000000000..201d83f99 --- /dev/null +++ b/roles/etcd/tasks/migration/check_cluster_health.yml @@ -0,0 +1,23 @@ +--- +- name: Check cluster health + command: > + etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoint https://{{ etcd_peer }}:{{ etcd_client_port }} cluster-health + register: etcd_cluster_health + changed_when: false + failed_when: false + +- name: Assume a member is not healthy + set_fact: + etcd_member_healthy: false + +- name: Get member item health status + set_fact: + etcd_member_healthy: true + with_items: "{{ etcd_cluster_health.stdout_lines }}" + when: "(etcd_peer in item) and ('is healthy' in item)" + +- name: Check the etcd cluster health + # TODO(jchaloup): should we fail or ask user if he wants to continue? Or just wait until the cluster is healthy? + fail: + msg: "Etcd member {{ etcd_peer }} is not healthy" + when: not etcd_member_healthy diff --git a/roles/etcd/tasks/migration/check_cluster_status.yml b/roles/etcd/tasks/migration/check_cluster_status.yml new file mode 100644 index 000000000..b69fb5a52 --- /dev/null +++ b/roles/etcd/tasks/migration/check_cluster_status.yml @@ -0,0 +1,32 @@ +--- +# etcd_ip originates from etcd_common role +- name: Check cluster status + command: > + etcdctl --cert {{ etcd_peer_cert_file }} --key {{ etcd_peer_key_file }} --cacert {{ etcd_peer_ca_file }} --endpoints 'https://{{ etcd_peer }}:{{ etcd_client_port }}' -w json endpoint status + environment: + ETCDCTL_API: 3 + register: l_etcd_cluster_status + +- name: Retrieve raftIndex + set_fact: + etcd_member_raft_index: "{{ (l_etcd_cluster_status.stdout | from_json)[0]['Status']['raftIndex'] }}" + +- block: + # http://docs.ansible.com/ansible/playbooks_filters.html#extracting-values-from-containers + - name: Group all raftIndices into a list + set_fact: + etcd_members_raft_indices: "{{ groups['oo_etcd_to_migrate'] | map('extract', hostvars, 'etcd_member_raft_index') | list | unique }}" + + - name: Check the minimum and the maximum of raftIndices is at most 1 + set_fact: + etcd_members_raft_indices_diff: "{{ ((etcd_members_raft_indices | max | int) - (etcd_members_raft_indices | min | int)) | int }}" + + - debug: + msg: "Raft indices difference: {{ etcd_members_raft_indices_diff }}" + + when: inventory_hostname in groups.oo_etcd_to_migrate[0] + +# The cluster raft status is ok if the difference of the max and min raft index is at most 1 +- name: capture the status + set_fact: + l_etcd_cluster_status_ok: "{{ hostvars[groups.oo_etcd_to_migrate[0]]['etcd_members_raft_indices_diff'] | int < 2 }}" diff --git a/roles/etcd/tasks/migration/configure_master.yml b/roles/etcd/tasks/migration/configure_master.yml new file mode 100644 index 000000000..a305d5bf3 --- /dev/null +++ b/roles/etcd/tasks/migration/configure_master.yml @@ -0,0 +1,13 @@ +--- +- name: Configure master to use etcd3 storage backend + yedit: + src: /etc/origin/master/master-config.yaml + key: "{{ item.key }}" + value: "{{ item.value }}" + with_items: + - key: kubernetesMasterConfig.apiServerArguments.storage-backend + value: + - etcd3 + - key: kubernetesMasterConfig.apiServerArguments.storage-media-type + value: + - application/vnd.kubernetes.protobuf diff --git a/roles/etcd/tasks/migration/migrate.yml b/roles/etcd/tasks/migration/migrate.yml new file mode 100644 index 000000000..54a9c74ff --- /dev/null +++ b/roles/etcd/tasks/migration/migrate.yml @@ -0,0 +1,56 @@ +--- +# Should this be run in a serial manner? +- set_fact: + l_etcd_service: "{{ 'etcd_container' if openshift.common.is_containerized else 'etcd' }}" + +- name: Migrate etcd data + command: > + etcdctl migrate --data-dir={{ etcd_data_dir }} + environment: + ETCDCTL_API: 3 + register: l_etcdctl_migrate +# TODO(jchaloup): If any of the members fails, we need to restore all members to v2 from the pre-migrate backup +- name: Check the etcd v2 data are correctly migrated + fail: + msg: "Failed to migrate a member" + when: "'finished transforming keys' not in l_etcdctl_migrate.stdout and 'no v2 keys to migrate' not in l_etcdctl_migrate.stdout" +- name: Migration message + debug: + msg: "Etcd migration finished with: {{ l_etcdctl_migrate.stdout }}" +- name: Set ETCD_FORCE_NEW_CLUSTER=true on first etcd host + lineinfile: + line: "ETCD_FORCE_NEW_CLUSTER=true" + dest: /etc/etcd/etcd.conf + backup: true +- name: Start etcd + systemd: + name: "{{ l_etcd_service }}" + state: started +- name: Wait for cluster to become healthy after bringing up first member + command: > + etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoint https://{{ etcd_peer }}:{{ etcd_client_port }} cluster-health + register: l_etcd_migrate_health + until: l_etcd_migrate_health.rc == 0 + retries: 3 + delay: 30 +- name: Unset ETCD_FORCE_NEW_CLUSTER=true on first etcd host + lineinfile: + line: "ETCD_FORCE_NEW_CLUSTER=true" + dest: /etc/etcd/etcd.conf + state: absent + backup: true +- name: Restart first etcd host + systemd: + name: "{{ l_etcd_service }}" + state: restarted + +- name: Wait for cluster to become healthy after bringing up first member + command: > + etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoint https://{{ etcd_peer }}:{{ etcd_client_port }} cluster-health + register: l_etcd_migrate_health + until: l_etcd_migrate_health.rc == 0 + retries: 3 + delay: 30 + +- set_fact: + r_etcd_migrate_success: true diff --git a/roles/etcd/tasks/remove_ca_certificates.yml b/roles/etcd/tasks/remove_ca_certificates.yml new file mode 100644 index 000000000..36df1a1cc --- /dev/null +++ b/roles/etcd/tasks/remove_ca_certificates.yml @@ -0,0 +1,2 @@ +--- +- include: certificates/remove_ca_certificates.yml diff --git a/roles/etcd/tasks/remove_generated_certificates.yml b/roles/etcd/tasks/remove_generated_certificates.yml new file mode 100644 index 000000000..b10a4b32d --- /dev/null +++ b/roles/etcd/tasks/remove_generated_certificates.yml @@ -0,0 +1,2 @@ +--- +- include: certificates/remove_generated_certificates.yml diff --git a/roles/etcd/tasks/retrieve_ca_certificates.yml b/roles/etcd/tasks/retrieve_ca_certificates.yml new file mode 100644 index 000000000..bd6c4ec85 --- /dev/null +++ b/roles/etcd/tasks/retrieve_ca_certificates.yml @@ -0,0 +1,2 @@ +--- +- include: certificates/retrieve_ca_certificates.yml diff --git a/roles/etcd/tasks/server_certificates.yml b/roles/etcd/tasks/server_certificates.yml new file mode 100644 index 000000000..ae26079f9 --- /dev/null +++ b/roles/etcd/tasks/server_certificates.yml @@ -0,0 +1,6 @@ +--- +- include: ca.yml + when: + - etcd_ca_setup | default(True) | bool + +- include: certificates/fetch_server_certificates_from_ca.yml diff --git a/roles/etcd/tasks/system_container.yml b/roles/etcd/tasks/system_container.yml new file mode 100644 index 000000000..024479fb4 --- /dev/null +++ b/roles/etcd/tasks/system_container.yml @@ -0,0 +1,100 @@ +--- +- set_fact: + l_etcd_src_data_dir: "{{ '/var/lib/origin/openshift.local.etcd' if r_etcd_common_embedded_etcd | bool else '/var/lib/etcd/' }}" + +- name: Pull etcd system container + command: atomic pull --storage=ostree {{ openshift.etcd.etcd_image }} + register: pull_result + changed_when: "'Pulling layer' in pull_result.stdout" + +- name: Set initial Etcd cluster + set_fact: + etcd_initial_cluster: >- + {% for host in etcd_peers | default([]) -%} + {% if loop.last -%} + {{ hostvars[host].etcd_hostname }}={{ etcd_peer_url_scheme }}://{{ hostvars[host].etcd_ip }}:{{ etcd_peer_port }} + {%- else -%} + {{ hostvars[host].etcd_hostname }}={{ etcd_peer_url_scheme }}://{{ hostvars[host].etcd_ip }}:{{ etcd_peer_port }}, + {%- endif -%} + {% endfor -%} + when: etcd_initial_cluster is undefined + +- name: Check etcd system container package + command: > + atomic containers list --no-trunc -a -f container=etcd -f backend=ostree + register: etcd_result + +- name: Unmask etcd service + systemd: + name: etcd + state: stopped + enabled: no + masked: no + daemon_reload: yes + register: task_result + failed_when: task_result|failed and 'could not' not in task_result.msg|lower + when: "'etcd' not in etcd_result.stdout" + +- name: Disable etcd_container + systemd: + name: etcd_container + state: stopped + enabled: no + daemon_reload: yes + register: task_result + failed_when: task_result|failed and 'could not' not in task_result.msg|lower + +- name: Remove etcd_container.service + file: + path: /etc/systemd/system/etcd_container.service + state: absent + +- name: Systemd reload configuration + systemd: name=etcd_container daemon_reload=yes + +- name: Check for previous etcd data store + stat: + path: "{{ l_etcd_src_data_dir }}/member/" + register: src_datastore + +- name: Check for etcd system container data store + stat: + path: "{{ r_etcd_common_system_container_host_dir }}/etcd.etcd/member" + register: dest_datastore + +- name: Ensure that etcd system container data dirs exist + file: path="{{ item }}" state=directory + with_items: + - "{{ r_etcd_common_system_container_host_dir }}/etc" + - "{{ r_etcd_common_system_container_host_dir }}/etcd.etcd" + +- name: Copy etcd data store + command: > + cp -a {{ l_etcd_src_data_dir }}/member + {{ r_etcd_common_system_container_host_dir }}/etcd.etcd/member + when: + - src_datastore.stat.exists + - not dest_datastore.stat.exists + +- name: Install or Update Etcd system container package + oc_atomic_container: + name: etcd + image: "{{ openshift.etcd.etcd_image }}" + state: latest + values: + - ETCD_LISTEN_PEER_URLS={{ etcd_listen_peer_urls }} + - ETCD_NAME={{ etcd_hostname }} + - ETCD_INITIAL_CLUSTER={{ etcd_initial_cluster }} + - ETCD_LISTEN_CLIENT_URLS={{ etcd_listen_client_urls }} + - ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_initial_advertise_peer_urls }} + - ETCD_INITIAL_CLUSTER_STATE={{ etcd_initial_cluster_state }} + - ETCD_INITIAL_CLUSTER_TOKEN={{ etcd_initial_cluster_token }} + - ETCD_ADVERTISE_CLIENT_URLS={{ etcd_advertise_client_urls }} + - ETCD_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt + - ETCD_CERT_FILE={{ etcd_system_container_conf_dir }}/server.crt + - ETCD_KEY_FILE={{ etcd_system_container_conf_dir }}/server.key + - ETCD_PEER_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt + - ETCD_PEER_CERT_FILE={{ etcd_system_container_conf_dir }}/peer.crt + - ETCD_PEER_KEY_FILE={{ etcd_system_container_conf_dir }}/peer.key + - ETCD_TRUSTED_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt + - ETCD_PEER_TRUSTED_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt diff --git a/roles/etcd/tasks/upgrade/upgrade_image.yml b/roles/etcd/tasks/upgrade/upgrade_image.yml new file mode 100644 index 000000000..24071f9ad --- /dev/null +++ b/roles/etcd/tasks/upgrade/upgrade_image.yml @@ -0,0 +1,60 @@ +--- +# INPUT r_etcd_upgrade_version +- name: Verify cluster is healthy pre-upgrade + command: "{{ etcdctlv2 }} cluster-health" + +- name: Get current image + shell: "grep 'ExecStart=' {{ etcd_service_file }} | awk '{print $NF}'" + register: current_image + +- name: Set new_etcd_image + set_fact: + new_etcd_image: "{{ current_image.stdout | regex_replace('/etcd.*$','/etcd:' ~ r_etcd_upgrade_version ) }}" + +- name: Pull new etcd image + command: "docker pull {{ new_etcd_image }}" + +- name: Update to latest etcd image + replace: + dest: "{{ etcd_service_file }}" + regexp: "{{ current_image.stdout }}$" + replace: "{{ new_etcd_image }}" + +- lineinfile: + destfile: "{{ etcd_conf_file }}" + regexp: '^ETCD_QUOTA_BACKEND_BYTES=' + line: "ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }}" + +- name: Restart etcd_container + systemd: + name: "{{ etcd_service }}" + daemon_reload: yes + state: restarted + +## TODO: probably should just move this into the backup playbooks, also this +## will fail on atomic host. We need to revisit how to do etcd backups there as +## the container may be newer than etcdctl on the host. Assumes etcd3 obsoletes etcd (7.3.1) +- name: Detecting Atomic Host Operating System + stat: + path: /run/ostree-booted + register: l_ostree_booted + +- name: Upgrade etcd for etcdctl when not atomic + package: + name: etcd + state: latest + when: not l_ostree_booted.stat.exists | bool + +- name: Verify cluster is healthy + command: "{{ etcdctlv2 }} cluster-health" + register: etcdctl + until: etcdctl.rc == 0 + retries: 3 + delay: 10 + +- name: Store new etcd_image + # DEPENDENCY openshift_facts + openshift_facts: + role: etcd + local_facts: + etcd_image: "{{ new_etcd_image }}" diff --git a/roles/etcd/tasks/upgrade/upgrade_rpm.yml b/roles/etcd/tasks/upgrade/upgrade_rpm.yml new file mode 100644 index 000000000..505e28afb --- /dev/null +++ b/roles/etcd/tasks/upgrade/upgrade_rpm.yml @@ -0,0 +1,37 @@ +--- +# INPUT r_etcd_upgrade_version? + +# F23 GA'd with etcd 2.0, currently has 2.2 in updates +# F24 GA'd with etcd-2.2, currently has 2.2 in updates +# F25 Beta currently has etcd 3.0 +# RHEL 7.3.4 with etcd-3.1.3-1.el7 +# RHEL 7.3.3 with etcd-3.1.0-2.el7 +# RHEL 7.3.2 with etcd-3.0.15-1.el7 + +- name: Verify cluster is healthy pre-upgrade + command: "{{ etcdctlv2 }} cluster-health" + +- set_fact: + l_etcd_target_package: "{{ 'etcd' if r_etcd_upgrade_version is not defined else 'etcd-'+r_etcd_upgrade_version+'*' }}" + +- name: Update etcd RPM to {{ l_etcd_target_package }} + package: + name: "{{ l_etcd_target_package }}" + state: latest + +- lineinfile: + destfile: "{{ etcd_conf_file }}" + regexp: '^ETCD_QUOTA_BACKEND_BYTES=' + line: "ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }}" + +- name: Restart etcd + service: + name: "{{ etcd_service }}" + state: restarted + +- name: Verify cluster is healthy + command: "{{ etcdctlv2 }} cluster-health" + register: etcdctl + until: etcdctl.rc == 0 + retries: 3 + delay: 10 diff --git a/roles/etcd/tasks/upgrade_image.yml b/roles/etcd/tasks/upgrade_image.yml new file mode 100644 index 000000000..9e69027eb --- /dev/null +++ b/roles/etcd/tasks/upgrade_image.yml @@ -0,0 +1,2 @@ +--- +- include: upgrade/upgrade_image.yml diff --git a/roles/etcd/tasks/upgrade_rpm.yml b/roles/etcd/tasks/upgrade_rpm.yml new file mode 100644 index 000000000..29603d2b6 --- /dev/null +++ b/roles/etcd/tasks/upgrade_rpm.yml @@ -0,0 +1,2 @@ +--- +- include: upgrade/upgrade_rpm.yml diff --git a/roles/etcd/templates/custom.conf.j2 b/roles/etcd/templates/custom.conf.j2 new file mode 100644 index 000000000..d3433c658 --- /dev/null +++ b/roles/etcd/templates/custom.conf.j2 @@ -0,0 +1,3 @@ +[Service] +WorkingDirectory={{ etcd_data_dir }} +EnvironmentFile=-{{ etcd_conf_file }} diff --git a/roles/etcd/templates/etcd.conf.j2 b/roles/etcd/templates/etcd.conf.j2 new file mode 100644 index 000000000..3027a9447 --- /dev/null +++ b/roles/etcd/templates/etcd.conf.j2 @@ -0,0 +1,85 @@ +{% macro initial_cluster() -%} +{% for host in etcd_peers | default([]) -%} +{% if loop.last -%} +{{ hostvars[host].etcd_hostname }}={{ etcd_peer_url_scheme }}://{{ hostvars[host].etcd_ip }}:{{ etcd_peer_port }} +{%- else -%} +{{ hostvars[host].etcd_hostname }}={{ etcd_peer_url_scheme }}://{{ hostvars[host].etcd_ip }}:{{ etcd_peer_port }}, +{%- endif -%} +{% endfor -%} +{% endmacro -%} + +ETCD_NAME={{ etcd_hostname }} +ETCD_LISTEN_PEER_URLS={{ etcd_listen_peer_urls }} +ETCD_DATA_DIR={{ etcd_data_dir }} +#ETCD_WAL_DIR="" +#ETCD_SNAPSHOT_COUNT=10000 +ETCD_HEARTBEAT_INTERVAL=500 +ETCD_ELECTION_TIMEOUT=2500 +ETCD_LISTEN_CLIENT_URLS={{ etcd_listen_client_urls }} +#ETCD_MAX_SNAPSHOTS=5 +#ETCD_MAX_WALS=5 +#ETCD_CORS= + + +#[cluster] +ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_initial_advertise_peer_urls }} +{% if etcd_is_thirdparty %} +# TODO: This needs to be altered to support the correct etcd instances +ETCD_INITIAL_CLUSTER={{ etcd_hostname}}={{ etcd_initial_advertise_peer_urls }} +ETCD_INITIAL_CLUSTER_STATE={{ etcd_initial_cluster_state }} +ETCD_INITIAL_CLUSTER_TOKEN=thirdparty-etcd-cluster-1 +{% else %} +{% if etcd_initial_cluster is defined and etcd_initial_cluster %} +ETCD_INITIAL_CLUSTER={{ etcd_initial_cluster }} +{% else %} +ETCD_INITIAL_CLUSTER={{ initial_cluster() }} +{% endif %} +ETCD_INITIAL_CLUSTER_STATE={{ etcd_initial_cluster_state }} +ETCD_INITIAL_CLUSTER_TOKEN={{ etcd_initial_cluster_token }} +#ETCD_DISCOVERY= +#ETCD_DISCOVERY_SRV= +#ETCD_DISCOVERY_FALLBACK=proxy +#ETCD_DISCOVERY_PROXY= +{% endif %} +ETCD_ADVERTISE_CLIENT_URLS={{ etcd_advertise_client_urls }} +#ETCD_STRICT_RECONFIG_CHECK="false" +#ETCD_AUTO_COMPACTION_RETENTION="0" +#ETCD_ENABLE_V2="true" +ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }} + +#[proxy] +#ETCD_PROXY=off +#ETCD_PROXY_FAILURE_WAIT="5000" +#ETCD_PROXY_REFRESH_INTERVAL="30000" +#ETCD_PROXY_DIAL_TIMEOUT="1000" +#ETCD_PROXY_WRITE_TIMEOUT="5000" +#ETCD_PROXY_READ_TIMEOUT="0" + +#[security] +{% if etcd_url_scheme == 'https' -%} +ETCD_TRUSTED_CA_FILE={{ etcd_ca_file }} +ETCD_CLIENT_CERT_AUTH="true" +ETCD_CERT_FILE={{ etcd_cert_file }} +ETCD_KEY_FILE={{ etcd_key_file }} +{% endif -%} +#ETCD_AUTO_TLS="false" +{% if etcd_peer_url_scheme == 'https' -%} +ETCD_PEER_TRUSTED_CA_FILE={{ etcd_peer_ca_file }} +ETCD_PEER_CLIENT_CERT_AUTH="true" +ETCD_PEER_CERT_FILE={{ etcd_peer_cert_file }} +ETCD_PEER_KEY_FILE={{ etcd_peer_key_file }} +{% endif -%} +#ETCD_PEER_AUTO_TLS="false" + +#[logging] +ETCD_DEBUG="{{ etcd_debug | default(false) | bool | string }}" +{% if etcd_log_package_levels is defined %} +ETCD_LOG_PACKAGE_LEVELS="{{ etcd_log_package_levels }}" +{% endif %} + +#[profiling] +#ETCD_ENABLE_PPROF="false" +#ETCD_METRICS="basic" +# +#[auth] +#ETCD_AUTH_TOKEN="simple" diff --git a/roles/etcd/templates/etcd.docker.service b/roles/etcd/templates/etcd.docker.service new file mode 100644 index 000000000..adeca7a91 --- /dev/null +++ b/roles/etcd/templates/etcd.docker.service @@ -0,0 +1,17 @@ +[Unit] +Description=The Etcd Server container +After={{ openshift.docker.service_name }}.service +Requires={{ openshift.docker.service_name }}.service +PartOf={{ openshift.docker.service_name }}.service + +[Service] +EnvironmentFile={{ etcd_conf_file }} +ExecStartPre=-/usr/bin/docker rm -f {{ etcd_service }} +ExecStart=/usr/bin/docker run --name {{ etcd_service }} --rm -v {{ etcd_data_dir }}:{{ etcd_data_dir }}:z -v {{ etcd_conf_dir }}:{{ etcd_conf_dir }}:ro --env-file={{ etcd_conf_file }} --net=host --entrypoint=/usr/bin/etcd {{ openshift.etcd.etcd_image }} +ExecStop=/usr/bin/docker stop {{ etcd_service }} +SyslogIdentifier=etcd_container +Restart=always +RestartSec=5s + +[Install] +WantedBy={{ openshift.docker.service_name }}.service diff --git a/roles/etcd/templates/etcdctl.sh.j2 b/roles/etcd/templates/etcdctl.sh.j2 new file mode 100644 index 000000000..ac7d9c72f --- /dev/null +++ b/roles/etcd/templates/etcdctl.sh.j2 @@ -0,0 +1,12 @@ +#!/bin/bash +# Sets up handy aliases for etcd, need etcdctl2 and etcdctl3 because +# command flags are different between the two. Should work on stand +# alone etcd hosts and master + etcd hosts too because we use the peer keys. +etcdctl2() { + /usr/bin/etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} -C https://`hostname`:2379 ${@} + +} + +etcdctl3() { + ETCDCTL_API=3 /usr/bin/etcdctl --cert {{ etcd_peer_cert_file }} --key {{ etcd_peer_key_file }} --cacert {{ etcd_peer_ca_file }} --endpoints https://`hostname`:2379 ${@} +} diff --git a/roles/etcd/templates/openssl_append.j2 b/roles/etcd/templates/openssl_append.j2 new file mode 100644 index 000000000..f28316fc2 --- /dev/null +++ b/roles/etcd/templates/openssl_append.j2 @@ -0,0 +1,51 @@ + +[ {{ etcd_req_ext }} ] +basicConstraints = critical,CA:FALSE +keyUsage = digitalSignature,keyEncipherment +subjectAltName = ${ENV::SAN} + +[ {{ etcd_ca_name }} ] +dir = {{ etcd_ca_dir }} +crl_dir = {{ etcd_ca_crl_dir }} +database = {{ etcd_ca_db }} +new_certs_dir = {{ etcd_ca_new_certs_dir }} +certificate = {{ etcd_ca_cert }} +serial = {{ etcd_ca_serial }} +private_key = {{ etcd_ca_key }} +crl_number = {{ etcd_ca_crl_number }} +x509_extensions = {{ etcd_ca_exts_client }} +default_days = {{ etcd_ca_default_days }} +default_md = sha256 +preserve = no +name_opt = ca_default +cert_opt = ca_default +policy = policy_anything +unique_subject = no +copy_extensions = copy + +[ {{ etcd_ca_exts_self }} ] +authorityKeyIdentifier = keyid,issuer +basicConstraints = critical,CA:TRUE,pathlen:0 +keyUsage = critical,digitalSignature,keyEncipherment,keyCertSign +subjectKeyIdentifier = hash + +[ {{ etcd_ca_exts_peer }} ] +authorityKeyIdentifier = keyid,issuer:always +basicConstraints = critical,CA:FALSE +extendedKeyUsage = clientAuth,serverAuth +keyUsage = digitalSignature,keyEncipherment +subjectKeyIdentifier = hash + +[ {{ etcd_ca_exts_server }} ] +authorityKeyIdentifier = keyid,issuer:always +basicConstraints = critical,CA:FALSE +extendedKeyUsage = serverAuth +keyUsage = digitalSignature,keyEncipherment +subjectKeyIdentifier = hash + +[ {{ etcd_ca_exts_client }} ] +authorityKeyIdentifier = keyid,issuer:always +basicConstraints = critical,CA:FALSE +extendedKeyUsage = clientAuth +keyUsage = digitalSignature,keyEncipherment +subjectKeyIdentifier = hash |