Merge pull request #1142 from abutcher/new-certs-who-dis

Support for redeploying certificates

15 files changed, 496 insertions, 86 deletions

diff --git a/filter_plugins/openshift_master.py b/filter_plugins/openshift_master.py
index b3f284a8e..048cd04c7 100644
--- a/filter_plugins/openshift_master.py
+++ b/filter_plugins/openshift_master.py
@@ -521,7 +521,7 @@ class FilterModule(object):
         return valid
 
     @staticmethod
-    def certificates_to_synchronize(hostvars):
+    def certificates_to_synchronize(hostvars, include_keys=True):
         ''' Return certificates to synchronize based on facts. '''
         if not issubclass(type(hostvars), dict):
             raise errors.AnsibleFilterError("|failed expects hostvars is a dict")
@@ -535,9 +535,10 @@ class FilterModule(object):
                  'openshift-registry.kubeconfig',
                  'openshift-router.crt',
                  'openshift-router.key',
-                 'openshift-router.kubeconfig',
-                 'serviceaccounts.private.key',
-                 'serviceaccounts.public.key']
+                 'openshift-router.kubeconfig']
+        if bool(include_keys):
+            certs += ['serviceaccounts.private.key',
+                      'serviceaccounts.public.key']
         if bool(hostvars['openshift']['common']['version_gte_3_1_or_1_1']):
             certs += ['master.proxy-client.crt',
                       'master.proxy-client.key']
diff --git a/inventory/byo/hosts.origin.example b/inventory/byo/hosts.origin.example
index fd6699ac1..8e7883f3b 100644
--- a/inventory/byo/hosts.origin.example
+++ b/inventory/byo/hosts.origin.example
@@ -343,7 +343,6 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 #osm_cluster_network_cidr=10.1.0.0/16
 #openshift_portal_net=172.30.0.0/16
 
-
 # Configure number of bits to allocate to each host’s subnet e.g. 8
 # would mean a /24 network on the host.
 #osm_host_subnet_length=8
@@ -355,7 +354,21 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 # set RPM version for debugging purposes
 #openshift_pkg_version=-1.1
 
-# Configure custom named certificates
+# Configure custom ca certificate
+#openshift_master_ca_certificate={'certfile': '/path/to/ca.crt', 'keyfile': '/path/to/ca.key'}
+#
+# NOTE: CA certificate will not be replaced with existing clusters.
+# This option may only be specified when creating a new cluster or
+# when redeploying cluster certificates with the redeploy-certificates
+# playbook. If replacing the CA certificate in an existing cluster
+# with a custom ca certificate, the following variable must also be
+# set.
+#openshift_certificates_redeploy_ca=true
+
+# Configure custom named certificates (SNI certificates)
+#
+# https://docs.openshift.org/latest/install_config/certificate_customization.html
+#
 # NOTE: openshift_master_named_certificates is cached on masters and is an
 # additive fact, meaning that each run with a different set of certificates
 # will add the newly provided certificates to the cached set of certificates.
diff --git a/inventory/byo/hosts.ose.example b/inventory/byo/hosts.ose.example
index 323e385c0..7e3d68e92 100644
--- a/inventory/byo/hosts.ose.example
+++ b/inventory/byo/hosts.ose.example
@@ -349,7 +349,21 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 # set RPM version for debugging purposes
 #openshift_pkg_version=-3.1.0.0
 
-# Configure custom named certificates
+# Configure custom ca certificate
+#openshift_master_ca_certificate={'certfile': '/path/to/ca.crt', 'keyfile': '/path/to/ca.key'}
+#
+# NOTE: CA certificate will not be replaced with existing clusters.
+# This option may only be specified when creating a new cluster or
+# when redeploying cluster certificates with the redeploy-certificates
+# playbook. If replacing the CA certificate in an existing cluster
+# with a custom ca certificate, the following variable must also be
+# set.
+#openshift_certificates_redeploy_ca=true
+
+# Configure custom named certificates (SNI certificates)
+#
+# https://docs.openshift.com/enterprise/latest/install_config/certificate_customization.html
+#
 # NOTE: openshift_master_named_certificates is cached on masters and is an
 # additive fact, meaning that each run with a different set of certificates
 # will add the newly provided certificates to the cached set of certificates.
diff --git a/playbooks/byo/openshift-cluster/enable_dnsmasq.yml b/playbooks/byo/openshift-cluster/enable_dnsmasq.yml
index 1c8d99341..0ba11a21b 100644
--- a/playbooks/byo/openshift-cluster/enable_dnsmasq.yml
+++ b/playbooks/byo/openshift-cluster/enable_dnsmasq.yml
@@ -1,4 +1,6 @@
 ---
+- include: ../../common/openshift-cluster/verify_ansible_version.yml
+
 - hosts: localhost
   connection: local
   become: no
@@ -8,7 +10,7 @@
   - add_host:
       name: "{{ item }}"
       groups: l_oo_all_hosts
-    with_items: g_all_hosts
+    with_items: "{{ g_all_hosts | default([]) }}"
 
 - hosts: l_oo_all_hosts
   gather_facts: no
diff --git a/playbooks/byo/openshift-cluster/redeploy-certificates.yml b/playbooks/byo/openshift-cluster/redeploy-certificates.yml
new file mode 100644
index 000000000..6d1247e0f
--- /dev/null
+++ b/playbooks/byo/openshift-cluster/redeploy-certificates.yml
@@ -0,0 +1,22 @@
+---
+- include: ../../common/openshift-cluster/verify_ansible_version.yml
+
+- hosts: localhost
+  connection: local
+  become: no
+  gather_facts: no
+  tasks:
+  - include_vars: ../../byo/openshift-cluster/cluster_hosts.yml
+  - add_host:
+      name: "{{ item }}"
+      groups: l_oo_all_hosts
+    with_items: "{{ g_all_hosts | default([]) }}"
+
+- hosts: l_oo_all_hosts
+  gather_facts: no
+  tasks:
+  - include_vars: ../../byo/openshift-cluster/cluster_hosts.yml
+  
+- include: ../../common/openshift-cluster/redeploy-certificates.yml
+  vars:
+    openshift_deployment_type: "{{ deployment_type }}"
diff --git a/playbooks/common/openshift-cluster/redeploy-certificates.yml b/playbooks/common/openshift-cluster/redeploy-certificates.yml
new file mode 100644
index 000000000..6f559badd
--- /dev/null
+++ b/playbooks/common/openshift-cluster/redeploy-certificates.yml
@@ -0,0 +1,245 @@
+---
+- include: evaluate_groups.yml
+
+- include: initialize_facts.yml
+
+- include: initialize_openshift_version.yml
+
+- name: Load openshift_facts
+  hosts: oo_etcd_to_config:oo_masters_to_config:oo_nodes_to_config
+  roles:
+  - openshift_facts
+
+- name: Redeploy etcd certificates
+  hosts: oo_etcd_to_config
+  any_errors_fatal: true
+  vars:
+    etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+    etcd_conf_dir: /etc/etcd
+    etcd_generated_certs_dir: "{{ etcd_conf_dir }}/generated_certs"
+
+  pre_tasks:
+  - stat:
+      path: "{{ etcd_generated_certs_dir }}"
+    register: etcd_generated_certs_dir_stat
+  - name: Backup etcd certificates
+    command: >
+      tar -czvf /etc/etcd/etcd-certificate-backup-{{ ansible_date_time.epoch }}.tgz
+      {{ etcd_conf_dir }}/ca.crt
+      {{ etcd_conf_dir }}/ca
+      {{ etcd_generated_certs_dir }}
+    when: etcd_generated_certs_dir_stat.stat.exists
+    delegate_to: "{{ etcd_ca_host }}"
+    run_once: true
+  - name: Remove existing etcd certificates
+    file:
+      path: "{{ item }}"
+      state: absent
+    with_items:
+    - "{{ etcd_conf_dir }}/ca.crt"
+    - "{{ etcd_conf_dir }}/ca"
+    - "{{ etcd_generated_certs_dir }}"
+  roles:
+  - role: openshift_etcd_server_certificates
+    etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
+    etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
+    etcd_certificates_redeploy: true
+
+- name: Redeploy master certificates
+  hosts: oo_masters_to_config
+  any_errors_fatal: true
+  vars:
+    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+    openshift_master_count: "{{ openshift.master.master_count | default(groups.oo_masters | length) }}"
+  pre_tasks:
+  - stat:
+      path: "{{ openshift_generated_configs_dir }}"
+    register: openshift_generated_configs_dir_stat
+  - name: Backup generated certificate and config directories
+    command: >
+      tar -czvf /etc/origin/master-node-cert-config-backup-{{ ansible_date_time.epoch }}.tgz
+      {{ openshift_generated_configs_dir }}
+      {{ openshift.common.config_base }}/master
+    when: openshift_generated_configs_dir_stat.stat.exists
+    delegate_to: "{{ openshift_ca_host }}"
+    run_once: true
+  - name: Remove generated certificate directories
+    file:
+      path: "{{ item }}"
+      state: absent
+    with_items:
+    - "{{ openshift_generated_configs_dir }}"
+  - name: Remove generated certificates
+    file:
+      path: "{{ openshift.common.config_base }}/master/{{ item }}"
+      state: absent
+    with_items:
+    - "{{ hostvars[inventory_hostname] | certificates_to_synchronize(include_keys=false) }}"
+    - "etcd.server.crt"
+    - "etcd.server.key"
+    - "master.etcd-client.crt"
+    - "master.etcd-client.key"
+    - "master.server.crt"
+    - "master.server.key"
+    - "openshift-master.crt"
+    - "openshift-master.key"
+    - "openshift-master.kubeconfig"
+  - name: Remove CA certificate
+    file:
+      path: "{{ openshift.common.config_base }}/master/{{ item }}"
+      state: absent
+    when: openshift_certificates_redeploy_ca | default(false) | bool
+    with_items:
+    - "ca.crt"
+    - "ca.key"
+    - "ca.serial.txt"
+    - "ca-bundle.crt"
+  roles:
+  - role: openshift_master_certificates
+    openshift_master_etcd_hosts: "{{ hostvars
+                                     | oo_select_keys(groups['oo_etcd_to_config'] | default([]))
+                                     | oo_collect('openshift.common.hostname')
+                                     | default(none, true) }}"
+    openshift_master_hostnames: "{{ hostvars
+                                    | oo_select_keys(groups['oo_masters_to_config'] | default([]))
+                                    | oo_collect('openshift.common.all_hostnames')
+                                    | oo_flatten | unique }}"
+    openshift_certificates_redeploy: true
+  - role: openshift_etcd_client_certificates
+    etcd_certificates_redeploy: true
+    etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+    etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}"
+    etcd_cert_config_dir: "{{ openshift.common.config_base }}/master"
+    etcd_cert_prefix: "master.etcd-"
+    when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
+
+- name: Redeploy node certificates
+  hosts: oo_nodes_to_config
+  any_errors_fatal: true
+  pre_tasks:
+  - name: Remove CA certificate
+    file:
+      path: "{{ item }}"
+      state: absent
+    with_items:
+    - "{{ openshift.common.config_base }}/node/ca.crt"
+  roles:
+  - role: openshift_node_certificates
+    openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}"
+    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+    openshift_certificates_redeploy: true
+
+- name: Restart etcd
+  hosts: oo_etcd_to_config
+  tasks:
+  - name: restart etcd
+    service: name=etcd state=restarted
+
+- name: Stop master services
+  hosts: oo_masters_to_config
+  vars:
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
+  tasks:
+  - name: stop master
+    service: name={{ openshift.common.service_type }}-master state=stopped
+    when: not openshift_master_ha | bool
+  - name: stop master api
+    service: name={{ openshift.common.service_type }}-master-api state=stopped
+    when: openshift_master_ha | bool and openshift_master_cluster_method == 'native'
+  - name: stop master controllers
+    service: name={{ openshift.common.service_type }}-master-controllers state=stopped
+    when: openshift_master_ha | bool and openshift_master_cluster_method == 'native'
+
+- name: Start master services
+  hosts: oo_masters_to_config
+  serial: 1
+  vars:
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
+  tasks:
+  - name: start master
+    service: name={{ openshift.common.service_type }}-master state=started
+    when: not openshift_master_ha | bool
+  - name: start master api
+    service: name={{ openshift.common.service_type }}-master-api state=started
+    when: openshift_master_ha | bool and openshift_master_cluster_method == 'native'
+  - name: start master controllers
+    service: name={{ openshift.common.service_type }}-master-controllers state=started
+    when: openshift_master_ha | bool and openshift_master_cluster_method == 'native'
+
+- name: Restart masters (pacemaker)
+  hosts: oo_first_master
+  vars:
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
+  tasks:
+  - name: restart master
+    command: pcs resource restart master
+    when: openshift_master_ha | bool and openshift_master_cluster_method == 'pacemaker'
+
+- name: Restart nodes
+  hosts: oo_nodes_to_config
+  tasks:
+  - name: restart node
+    service: name={{ openshift.common.service_type }}-node state=restarted
+
+- name: Copy admin client config(s)
+  hosts: oo_first_master
+  tasks:
+  - name: Create temp directory for kubeconfig
+    command: mktemp -d /tmp/openshift-ansible-XXXXXX
+    register: mktemp
+    changed_when: False
+
+  - name: Copy admin client config(s)
+    command: >
+      cp {{ openshift.common.config_base }}/master//admin.kubeconfig {{ mktemp.stdout }}/admin.kubeconfig
+    changed_when: False
+
+- name: Serially evacuate all nodes to trigger redeployments
+  hosts: oo_nodes_to_config
+  serial: 1
+  any_errors_fatal: true
+  tasks:
+  - name: Determine if node is currently scheduleable
+    command: >
+      {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig
+      get node {{ openshift.common.hostname | lower }} -o json
+    register: node_output
+    when: openshift_certificates_redeploy_ca | default(false) | bool
+    delegate_to: "{{ groups.oo_first_master.0 }}"
+    changed_when: false
+
+  - set_fact:
+      was_schedulable: "{{ 'unschedulable' not in (node_output.stdout | from_json).spec }}"
+    when: openshift_certificates_redeploy_ca | default(false) | bool
+
+  - name: Prepare for node evacuation
+    command: >
+      {{ openshift.common.admin_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig
+      manage-node {{ openshift.common.hostname | lower }}
+      --schedulable=false
+    delegate_to: "{{ groups.oo_first_master.0 }}"
+    when: openshift_certificates_redeploy_ca | default(false) | bool and was_schedulable | bool
+
+  - name: Evacuate node
+    command: >
+      {{ openshift.common.admin_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig
+      manage-node {{ openshift.common.hostname | lower }}
+      --evacuate --force
+    delegate_to: "{{ groups.oo_first_master.0 }}"
+    when: openshift_certificates_redeploy_ca | default(false) | bool and was_schedulable | bool
+
+  - name: Set node schedulability
+    command: >
+      {{ openshift.common.admin_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig
+      manage-node {{ openshift.common.hostname | lower }} --schedulable=true
+    delegate_to: "{{ groups.oo_first_master.0 }}"
+    when: openshift_certificates_redeploy_ca | default(false) | bool and was_schedulable | bool
+
+- name: Delete temporary directory
+  hosts: oo_first_master
+  tasks:
+  - name: Delete temp directory
+    file:
+      name: "{{ mktemp.stdout }}"
+      state: absent
+    changed_when: False
diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml
index 75d40216d..ba4136327 100644
--- a/roles/etcd/tasks/main.yml
+++ b/roles/etcd/tasks/main.yml
@@ -58,30 +58,6 @@
     group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
     mode: 0700
 
-- name: Validate permissions on certificate files
-  file:
-    path: "{{ item }}"
-    mode: 0600
-    owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
-    group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
-  when: etcd_url_scheme == 'https'
-  with_items:
-  - "{{ etcd_ca_file }}"
-  - "{{ etcd_cert_file }}"
-  - "{{ etcd_key_file }}"
-
-- name: Validate permissions on peer certificate files
-  file:
-    path: "{{ item }}"
-    mode: 0600
-    owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
-    group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
-  when: etcd_peer_url_scheme == 'https'
-  with_items:
-  - "{{ etcd_peer_ca_file }}"
-  - "{{ etcd_peer_cert_file }}"
-  - "{{ etcd_peer_key_file }}"
-
 - name: Write etcd global config file
   template:
     src: etcd.conf.j2
diff --git a/roles/etcd_client_certificates/meta/main.yml b/roles/etcd_client_certificates/meta/main.yml
index 713c78c70..efebdb599 100644
--- a/roles/etcd_client_certificates/meta/main.yml
+++ b/roles/etcd_client_certificates/meta/main.yml
@@ -13,4 +13,4 @@ galaxy_info:
   - cloud
   - system
 dependencies:
-- role: etcd_ca
+- role: etcd_common
diff --git a/roles/etcd_client_certificates/tasks/main.yml b/roles/etcd_client_certificates/tasks/main.yml
index b86afb81c..275aa0a63 100644
--- a/roles/etcd_client_certificates/tasks/main.yml
+++ b/roles/etcd_client_certificates/tasks/main.yml
@@ -1,4 +1,19 @@
 ---
+- name: Ensure CA certificate exists on etcd_ca_host
+  stat:
+    path: "{{ etcd_ca_cert }}"
+  register: g_ca_cert_stat_result
+  delegate_to: "{{ etcd_ca_host }}"
+  run_once: true
+
+- fail:
+    msg: >
+      CA certificate {{ etcd_ca_cert }} doesn't exist on CA host
+      {{ etcd_ca_host }}. Apply 'etcd_ca' role to
+      {{ etcd_ca_host }}.
+  when: not g_ca_cert_stat_result.stat.exists | bool
+  run_once: true
+
 - name: Check status of external etcd certificatees
   stat:
     path: "{{ etcd_cert_config_dir }}/{{ item }}"
@@ -7,11 +22,14 @@
   - "{{ etcd_cert_prefix }}client.key"
   - "{{ etcd_cert_prefix }}ca.crt"
   register: g_external_etcd_cert_stat_result
+  when: not etcd_certificates_redeploy | default(false) | bool
 
 - set_fact:
-    etcd_client_certs_missing: "{{ False in (g_external_etcd_cert_stat_result.results
-                                   | oo_collect(attribute='stat.exists')
-                                   | list) }}"
+    etcd_client_certs_missing: "{{ true if etcd_certificates_redeploy | default(false) | bool
+                                   else (False in (g_external_etcd_cert_stat_result.results
+                                                   | default({})
+                                                   | oo_collect(attribute='stat.exists')
+                                                   | list)) }}"
 
 - name: Ensure generated_certs directory present
   file:
diff --git a/roles/etcd_server_certificates/tasks/main.yml b/roles/etcd_server_certificates/tasks/main.yml
index f11b51453..27bd2a88d 100644
--- a/roles/etcd_server_certificates/tasks/main.yml
+++ b/roles/etcd_server_certificates/tasks/main.yml
@@ -7,11 +7,14 @@
   - "{{ etcd_cert_prefix }}peer.crt"
   - "{{ etcd_cert_prefix }}ca.crt"
   register: g_etcd_server_cert_stat_result
+  when: not etcd_certificates_redeploy | default(false) | bool
 
 - set_fact:
-    etcd_server_certs_missing: "{{ False in (g_etcd_server_cert_stat_result.results
-                                   | oo_collect(attribute='stat.exists')
-                                   | list) }}"
+    etcd_server_certs_missing: "{{ true if etcd_certificates_redeploy | default(false) | bool
+                                   else (False in (g_etcd_server_cert_stat_result.results
+                                                   | default({})
+                                                   | oo_collect(attribute='stat.exists')
+                                                   | list)) }}"
 
 - name: Ensure generated_certs directory present
   file:
@@ -69,6 +72,8 @@
   when: etcd_server_certs_missing | bool
   delegate_to: "{{ etcd_ca_host }}"
 
+# Certificates must be signed serially in order to avoid competing
+# for the serial file.
 - name: Sign and create the peer crt
   delegated_serial_command:
     command: >
@@ -136,3 +141,35 @@
   changed_when: False
   when: etcd_server_certs_missing | bool
   delegate_to: localhost
+
+- name: Validate permissions on certificate files
+  file:
+    path: "{{ item }}"
+    mode: 0600
+    owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
+    group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
+  when: etcd_url_scheme == 'https'
+  with_items:
+  - "{{ etcd_ca_file }}"
+  - "{{ etcd_cert_file }}"
+  - "{{ etcd_key_file }}"
+
+- name: Validate permissions on peer certificate files
+  file:
+    path: "{{ item }}"
+    mode: 0600
+    owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
+    group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
+  when: etcd_peer_url_scheme == 'https'
+  with_items:
+  - "{{ etcd_peer_ca_file }}"
+  - "{{ etcd_peer_cert_file }}"
+  - "{{ etcd_peer_key_file }}"
+
+- name: Validate permissions on the config dir
+  file:
+    path: "{{ etcd_conf_dir }}"
+    state: directory
+    owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
+    group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
+    mode: 0700
diff --git a/roles/openshift_ca/tasks/main.yml b/roles/openshift_ca/tasks/main.yml
index e1bf7dcad..bb89b65a6 100644
--- a/roles/openshift_ca/tasks/main.yml
+++ b/roles/openshift_ca/tasks/main.yml
@@ -3,6 +3,10 @@
     msg: "openshift_ca_host variable must be defined for this role"
   when: openshift_ca_host is not defined
 
+- fail:
+    msg: "Both 'certfile' and 'keyfile' keys must be supplied when configuring openshift_master_ca_certificate"
+  when: openshift_master_ca_certificate is defined and ('certfile' not in openshift_master_ca_certificate or 'keyfile' not in openshift_master_ca_certificate)
+
 - name: Install the base package for admin tooling
   action: "{{ ansible_pkg_mgr }} name={{ openshift.common.service_type }}{{ openshift_pkg_version | default('') | oo_image_tag_to_rpm_version(include_dash=True) }} state=present"
   when: not openshift.common.is_containerized | bool
@@ -35,9 +39,43 @@
   run_once: true
 
 - set_fact:
-    master_ca_missing: "{{ False in (g_master_ca_stat_result.results
-                           | oo_collect(attribute='stat.exists')
-                           | list) }}"
+    master_ca_missing: "{{ true if openshift_certificates_redeploy | default(false) | bool
+                           else False in (g_master_ca_stat_result.results
+                                         | oo_collect(attribute='stat.exists')
+                                         | list) }}"
+  run_once: true
+
+- name: Retain original serviceaccount keys
+  copy:
+    src: "{{ item }}"
+    dest: "{{ item }}.keep"
+    remote_src: true
+  with_items:
+  - "{{ openshift_ca_config_dir }}/serviceaccounts.private.key"
+  - "{{ openshift_ca_config_dir }}/serviceaccounts.public.key"
+  when: openshift_certificates_redeploy | default(false) | bool
+
+- name: Deploy master ca certificate
+  copy:
+    src: "{{ item.src }}"
+    dest: "{{ openshift_ca_config_dir }}/{{ item.dest }}"
+    force: "{{ true if openshift_certificates_redeploy_ca | default(false) | bool else false }}"
+  with_items:
+  - src: "{{ (openshift_master_ca_certificate | default({'certfile':none})).certfile }}"
+    dest: ca.crt
+  - src: "{{ (openshift_master_ca_certificate | default({'keyfile':none})).keyfile }}"
+    dest: ca.key
+  when: openshift_master_ca_certificate is defined
+  delegate_to: "{{ openshift_ca_host }}"
+  run_once: true
+
+- name: Create ca serial
+  copy:
+    content: "1"
+    dest: "{{ openshift_ca_config_dir }}/ca.serial.txt"
+    force: "{{ true if openshift_certificates_redeploy | default(false) | bool else false }}"
+  when: openshift_master_ca_certificate is defined
+  delegate_to: "{{ openshift_ca_host }}"
   run_once: true
 
 - name: Create the master certificates if they do not already exist
@@ -54,3 +92,22 @@
   when: master_ca_missing | bool
   delegate_to: "{{ openshift_ca_host }}"
   run_once: true
+
+- name: Restore original serviceaccount keys
+  copy:
+    src: "{{ item }}.keep"
+    dest: "{{ item }}"
+    remote_src: true
+  with_items:
+  - "{{ openshift_ca_config_dir }}/serviceaccounts.private.key"
+  - "{{ openshift_ca_config_dir }}/serviceaccounts.public.key"
+  when: openshift_certificates_redeploy | default(false) | bool
+
+- name: Remove backup serviceaccount keys
+  file:
+    path: "{{ item }}.keep"
+    state: absent
+  with_items:
+  - "{{ openshift_ca_config_dir }}/serviceaccounts.private.key"
+  - "{{ openshift_ca_config_dir }}/serviceaccounts.public.key"
+  when: openshift_certificates_redeploy | default(false) | bool
diff --git a/roles/openshift_etcd_server_certificates/meta/main.yml b/roles/openshift_etcd_server_certificates/meta/main.yml
new file mode 100644
index 000000000..7750f14af
--- /dev/null
+++ b/roles/openshift_etcd_server_certificates/meta/main.yml
@@ -0,0 +1,16 @@
+---
+galaxy_info:
+  author: Jason DeTiberus
+  description: OpenShift Etcd Server Certificates
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 2.1
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
+dependencies:
+- role: openshift_etcd_facts
+- role: etcd_server_certificates
diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml
index e1efb4c2b..6259fd996 100644
--- a/roles/openshift_master/tasks/main.yml
+++ b/roles/openshift_master/tasks/main.yml
@@ -277,37 +277,3 @@
 - name: Set the cluster user password
   shell: echo {{ openshift_master_cluster_password | quote }} | passwd --stdin hacluster
   when: install_result | changed
-
-- name: Lookup default group for ansible_ssh_user
-  command: "/usr/bin/id -g {{ ansible_ssh_user }}"
-  changed_when: false
-  register: _ansible_ssh_user_gid
-
-- set_fact:
-    client_users: "{{ [ansible_ssh_user, 'root'] | unique }}"
-
-- name: Create the client config dir(s)
-  file:
-    path: "~{{ item }}/.kube"
-    state: directory
-    mode: 0700
-    owner: "{{ item }}"
-    group: "{{ 'root' if item == 'root' else _ansible_ssh_user_gid.stdout  }}"
-  with_items: "{{ client_users }}"
-
-# TODO: Update this file if the contents of the source file are not present in
-# the dest file, will need to make sure to ignore things that could be added
-- name: Copy the admin client config(s)
-  command: cp {{ openshift_master_config_dir }}/admin.kubeconfig ~{{ item }}/.kube/config
-  args:
-    creates: ~{{ item }}/.kube/config
-  with_items: "{{ client_users }}"
-
-- name: Update the permissions on the admin client config(s)
-  file:
-    path: "~{{ item }}/.kube/config"
-    state: file
-    mode: 0700
-    owner: "{{ item }}"
-    group: "{{ 'root' if item == 'root' else _ansible_ssh_user_gid.stdout  }}"
-  with_items: "{{ client_users }}"
diff --git a/roles/openshift_master_certificates/tasks/main.yml b/roles/openshift_master_certificates/tasks/main.yml
index 9ed082d9f..aafb06f93 100644
--- a/roles/openshift_master_certificates/tasks/main.yml
+++ b/roles/openshift_master_certificates/tasks/main.yml
@@ -21,18 +21,22 @@
   with_items:
   - "{{ openshift_master_certs }}"
   register: g_master_cert_stat_result
+  when: not openshift_certificates_redeploy | default(false) | bool
 
 - set_fact:
-    master_certs_missing: "{{ False in (g_master_cert_stat_result.results
-                              | oo_collect(attribute='stat.exists')
-                              | list) }}"
+    master_certs_missing: "{{ true if openshift_certificates_redeploy | default(false) | bool
+                              else (False in (g_master_cert_stat_result.results
+                                              | default({})
+                                              | oo_collect(attribute='stat.exists')
+                                              | list)) }}"
+
 
 - name: Ensure the generated_configs directory present
   file:
     path: "{{ openshift_master_generated_config_dir }}"
     state: directory
     mode: 0700
-  when: master_certs_missing | bool
+  when: master_certs_missing | bool and inventory_hostname != openshift_ca_host
   delegate_to: "{{ openshift_ca_host }}"
 
 - file:
@@ -43,7 +47,7 @@
   - ca.crt
   - ca.key
   - ca.serial.txt
-  when: master_certs_missing | bool
+  when: master_certs_missing | bool and inventory_hostname != openshift_ca_host
   delegate_to: "{{ openshift_ca_host }}"
 
 - name: Create the master certificates if they do not already exist
@@ -57,7 +61,7 @@
     --public-master={{ openshift.master.public_api_url }}
     --cert-dir={{ openshift_master_generated_config_dir }}
     --overwrite=false
-  when: master_certs_missing | bool
+  when: master_certs_missing | bool and inventory_hostname != openshift_ca_host
   delegate_to: "{{ openshift_ca_host }}"
 
 - file:
@@ -67,7 +71,7 @@
     force: true
   with_items:
   - "{{ hostvars[inventory_hostname] | certificates_to_synchronize }}"
-  when: master_certs_missing | bool
+  when: master_certs_missing | bool and inventory_hostname != openshift_ca_host
   delegate_to: "{{ openshift_ca_host }}"
 
 - name: Remove generated etcd client certs when using external etcd
@@ -124,3 +128,39 @@
   when: master_certs_missing | bool
   delegate_to: localhost
   become: no
+
+- name: Lookup default group for ansible_ssh_user
+  command: "/usr/bin/id -g {{ ansible_ssh_user }}"
+  changed_when: false
+  register: _ansible_ssh_user_gid
+
+- set_fact:
+    client_users: "{{ [ansible_ssh_user, 'root'] | unique }}"
+
+- name: Create the client config dir(s)
+  file:
+    path: "~{{ item }}/.kube"
+    state: directory
+    mode: 0700
+    owner: "{{ item }}"
+    group: "{{ 'root' if item == 'root' else _ansible_ssh_user_gid.stdout  }}"
+  with_items: "{{ client_users }}"
+
+# TODO: Update this file if the contents of the source file are not present in
+# the dest file, will need to make sure to ignore things that could be added
+- name: Copy the admin client config(s)
+  copy:
+    src: "{{ openshift_master_config_dir }}/admin.kubeconfig"
+    dest: "~{{ item }}/.kube/config"
+    remote_src: yes
+    force: "{{ openshift_certificates_redeploy | default(false) }}"
+  with_items: "{{ client_users }}"
+
+- name: Update the permissions on the admin client config(s)
+  file:
+    path: "~{{ item }}/.kube/config"
+    state: file
+    mode: 0700
+    owner: "{{ item }}"
+    group: "{{ 'root' if item == 'root' else _ansible_ssh_user_gid.stdout  }}"
+  with_items: "{{ client_users }}"
diff --git a/roles/openshift_node_certificates/tasks/main.yml b/roles/openshift_node_certificates/tasks/main.yml
index 8768fb0c2..fef7caab8 100644
--- a/roles/openshift_node_certificates/tasks/main.yml
+++ b/roles/openshift_node_certificates/tasks/main.yml
@@ -25,11 +25,14 @@
   - server.key
   - server.crt
   register: g_node_cert_stat_result
+  when: not openshift_certificates_redeploy | default(false) | bool
 
 - set_fact:
-    node_certs_missing: "{{ False in (g_node_cert_stat_result.results
-                            | oo_collect(attribute='stat.exists')
-                            | list) }}"
+    node_certs_missing: "{{ true if openshift_certificates_redeploy | default(false) | bool
+                            else (False in (g_node_cert_stat_result.results
+                                            | default({})
+                                            | oo_collect(attribute='stat.exists')
+                                            | list)) }}"
 
 - name: Create openshift_generated_configs_dir if it does not exist
   file: