Merge pull request #2671 from abutcher/cert-redeploy-restructure

Restructure certificate redeploy playbooks

16 files changed, 786 insertions, 266 deletions

diff --git a/playbooks/common/openshift-cluster/config.yml b/playbooks/common/openshift-cluster/config.yml
index a95cb68b7..a0ba735ab 100644
--- a/playbooks/common/openshift-cluster/config.yml
+++ b/playbooks/common/openshift-cluster/config.yml
@@ -1,20 +1,4 @@
 ---
-- include: evaluate_groups.yml
-  tags:
-  - always
-
-- include: initialize_facts.yml
-  tags:
-  - always
-
-- include: validate_hostnames.yml
-  tags:
-  - node
-
-- include: initialize_openshift_version.yml
-  tags:
-  - always
-
 - name: Set oo_option facts
   hosts: oo_all_hosts
   tags:
diff --git a/playbooks/common/openshift-cluster/redeploy-certificates.yml b/playbooks/common/openshift-cluster/redeploy-certificates.yml
deleted file mode 100644
index a0e3f1d8a..000000000
--- a/playbooks/common/openshift-cluster/redeploy-certificates.yml
+++ /dev/null
@@ -1,250 +0,0 @@
----
-- include: evaluate_groups.yml
-
-- include: initialize_facts.yml
-
-- include: initialize_openshift_version.yml
-
-- name: Load openshift_facts
-  hosts: oo_etcd_to_config:oo_masters_to_config:oo_nodes_to_config
-  roles:
-  - openshift_facts
-
-- name: Redeploy etcd certificates
-  hosts: oo_etcd_to_config
-  any_errors_fatal: true
-  vars:
-    etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
-    etcd_conf_dir: /etc/etcd
-    etcd_generated_certs_dir: "{{ etcd_conf_dir }}/generated_certs"
-
-  pre_tasks:
-  - stat:
-      path: "{{ etcd_generated_certs_dir }}"
-    register: etcd_generated_certs_dir_stat
-  - name: Backup etcd certificates
-    command: >
-      tar -czvf /etc/etcd/etcd-certificate-backup-{{ ansible_date_time.epoch }}.tgz
-      {{ etcd_conf_dir }}/ca.crt
-      {{ etcd_conf_dir }}/ca
-      {{ etcd_generated_certs_dir }}
-    when: etcd_generated_certs_dir_stat.stat.exists
-    delegate_to: "{{ etcd_ca_host }}"
-    run_once: true
-  - name: Remove existing etcd certificates
-    file:
-      path: "{{ item }}"
-      state: absent
-    with_items:
-    - "{{ etcd_conf_dir }}/ca.crt"
-    - "{{ etcd_conf_dir }}/ca"
-    - "{{ etcd_generated_certs_dir }}"
-  roles:
-  - role: openshift_etcd_server_certificates
-    etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
-    etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
-    etcd_certificates_redeploy: true
-
-- name: Redeploy master certificates
-  hosts: oo_masters_to_config
-  any_errors_fatal: true
-  vars:
-    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
-    openshift_master_count: "{{ openshift.master.master_count | default(groups.oo_masters | length) }}"
-  pre_tasks:
-  # set_fact task copied from playbooks/common/openshift-master/config.yml
-  # so that openshift_master_default_subdomain has a default value of ""
-  # (emptry string). openshift_master_default_subdomain must have a default
-  # value for openshift_master_facts to set metrics_public_url.
-  # TODO: clean this up.
-  - set_fact:
-      openshift_master_default_subdomain: "{{ lookup('oo_option', 'openshift_master_default_subdomain') | default(None, true) }}"
-    when: openshift_master_default_subdomain is not defined
-  - stat:
-      path: "{{ openshift_generated_configs_dir }}"
-    register: openshift_generated_configs_dir_stat
-  - name: Backup generated certificate and config directories
-    command: >
-      tar -czvf /etc/origin/master-node-cert-config-backup-{{ ansible_date_time.epoch }}.tgz
-      {{ openshift_generated_configs_dir }}
-      {{ openshift.common.config_base }}/master
-    when: openshift_generated_configs_dir_stat.stat.exists
-    delegate_to: "{{ openshift_ca_host }}"
-    run_once: true
-  - name: Remove generated certificate directories
-    file:
-      path: "{{ item }}"
-      state: absent
-    with_items:
-    - "{{ openshift_generated_configs_dir }}"
-  - name: Remove generated certificates
-    file:
-      path: "{{ openshift.common.config_base }}/master/{{ item }}"
-      state: absent
-    with_items:
-    - "{{ hostvars[inventory_hostname] | certificates_to_synchronize(include_keys=false) }}"
-    - "etcd.server.crt"
-    - "etcd.server.key"
-    - "master.etcd-client.crt"
-    - "master.etcd-client.key"
-    - "master.server.crt"
-    - "master.server.key"
-    - "openshift-master.crt"
-    - "openshift-master.key"
-    - "openshift-master.kubeconfig"
-  - name: Remove CA certificate
-    file:
-      path: "{{ openshift.common.config_base }}/master/{{ item }}"
-      state: absent
-    when: openshift_certificates_redeploy_ca | default(false) | bool
-    with_items:
-    - "ca.crt"
-    - "ca.key"
-    - "ca.serial.txt"
-    - "ca-bundle.crt"
-  roles:
-  - role: openshift_master_certificates
-    openshift_master_etcd_hosts: "{{ hostvars
-                                     | oo_select_keys(groups['oo_etcd_to_config'] | default([]))
-                                     | oo_collect('openshift.common.hostname')
-                                     | default(none, true) }}"
-    openshift_certificates_redeploy: true
-  - role: openshift_etcd_client_certificates
-    etcd_certificates_redeploy: true
-    etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
-    etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}"
-    etcd_cert_config_dir: "{{ openshift.common.config_base }}/master"
-    etcd_cert_prefix: "master.etcd-"
-    when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
-
-- name: Redeploy node certificates
-  hosts: oo_nodes_to_config
-  any_errors_fatal: true
-  pre_tasks:
-  - name: Remove CA certificate
-    file:
-      path: "{{ item }}"
-      state: absent
-    with_items:
-    - "{{ openshift.common.config_base }}/node/ca.crt"
-  roles:
-  - role: openshift_node_certificates
-    openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}"
-    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
-    openshift_certificates_redeploy: true
-
-- name: Restart etcd
-  hosts: oo_etcd_to_config
-  tasks:
-  - name: restart etcd
-    service:
-      name: "{{ 'etcd' if not openshift.common.is_containerized | bool else 'etcd_container' }}"
-      state: restarted
-
-- name: Stop master services
-  hosts: oo_masters_to_config
-  vars:
-    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
-  tasks:
-  - name: stop master
-    service: name={{ openshift.common.service_type }}-master state=stopped
-    when: not openshift_master_ha | bool
-  - name: stop master api
-    service: name={{ openshift.common.service_type }}-master-api state=stopped
-    when: openshift_master_ha | bool and openshift_master_cluster_method == 'native'
-  - name: stop master controllers
-    service: name={{ openshift.common.service_type }}-master-controllers state=stopped
-    when: openshift_master_ha | bool and openshift_master_cluster_method == 'native'
-
-- name: Start master services
-  hosts: oo_masters_to_config
-  serial: 1
-  vars:
-    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
-  tasks:
-  - name: start master
-    service: name={{ openshift.common.service_type }}-master state=started
-    when: not openshift_master_ha | bool
-  - name: start master api
-    service: name={{ openshift.common.service_type }}-master-api state=started
-    when: openshift_master_ha | bool and openshift_master_cluster_method == 'native'
-  - name: start master controllers
-    service: name={{ openshift.common.service_type }}-master-controllers state=started
-    when: openshift_master_ha | bool and openshift_master_cluster_method == 'native'
-
-- name: Restart masters (pacemaker)
-  hosts: oo_first_master
-  vars:
-    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
-  tasks:
-  - name: restart master
-    command: pcs resource restart master
-    when: openshift_master_ha | bool and openshift_master_cluster_method == 'pacemaker'
-
-- name: Restart nodes
-  hosts: oo_nodes_to_config
-  tasks:
-  - name: restart node
-    service: name={{ openshift.common.service_type }}-node state=restarted
-
-- name: Copy admin client config(s)
-  hosts: oo_first_master
-  tasks:
-  - name: Create temp directory for kubeconfig
-    command: mktemp -d /tmp/openshift-ansible-XXXXXX
-    register: mktemp
-    changed_when: False
-
-  - name: Copy admin client config(s)
-    command: >
-      cp {{ openshift.common.config_base }}/master//admin.kubeconfig {{ mktemp.stdout }}/admin.kubeconfig
-    changed_when: False
-
-- name: Serially drain all nodes to trigger redeployments
-  hosts: oo_nodes_to_config
-  serial: 1
-  any_errors_fatal: true
-  tasks:
-  - name: Determine if node is currently scheduleable
-    command: >
-      {{ openshift.common.client_binary }} --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig
-      get node {{ openshift.node.nodename }} -o json
-    register: node_output
-    when: openshift_certificates_redeploy_ca | default(false) | bool
-    delegate_to: "{{ groups.oo_first_master.0 }}"
-    changed_when: false
-
-  - set_fact:
-      was_schedulable: "{{ 'unschedulable' not in (node_output.stdout | from_json).spec }}"
-    when: openshift_certificates_redeploy_ca | default(false) | bool
-
-  - name: Prepare for node draining
-    command: >
-      {{ openshift.common.client_binary }} adm --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig
-      manage-node {{ openshift.node.nodename }}
-      --schedulable=false
-    delegate_to: "{{ groups.oo_first_master.0 }}"
-    when: openshift_certificates_redeploy_ca | default(false) | bool and was_schedulable | bool
-
-  - name: Drain node
-    command: >
-      {{ openshift.common.admin_binary }} --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig
-      drain {{ openshift.node.nodename }} --force --delete-local-data
-    delegate_to: "{{ groups.oo_first_master.0 }}"
-    when: openshift_certificates_redeploy_ca | default(false) | bool and was_schedulable | bool
-
-  - name: Set node schedulability
-    command: >
-      {{ openshift.common.client_binary }} adm --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig
-      manage-node {{ openshift.node.nodename }} --schedulable=true
-    delegate_to: "{{ groups.oo_first_master.0 }}"
-    when: openshift_certificates_redeploy_ca | default(false) | bool and was_schedulable | bool
-
-- name: Delete temporary directory
-  hosts: oo_first_master
-  tasks:
-  - name: Delete temp directory
-    file:
-      name: "{{ mktemp.stdout }}"
-      state: absent
-    changed_when: False
diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/ca.yml b/playbooks/common/openshift-cluster/redeploy-certificates/ca.yml
new file mode 100644
index 000000000..0b1c39ba4
--- /dev/null
+++ b/playbooks/common/openshift-cluster/redeploy-certificates/ca.yml
@@ -0,0 +1,353 @@
+---
+- name: Verify OpenShift version is greater than or equal to 1.2 or 3.2
+  hosts: oo_first_master
+  tasks:
+  - fail:
+      msg: "The current OpenShift version is less than 1.2/3.2 and does not support CA bundles."
+    when: not openshift.common.version_gte_3_2_or_1_2 | bool
+
+- name: Backup existing etcd CA certificate directories
+  hosts: oo_etcd_to_config
+  roles:
+  - etcd_common
+  tasks:
+  - name: Determine if CA certificate directory exists
+    stat:
+      path: "{{ etcd_ca_dir }}"
+    register: etcd_ca_certs_dir_stat
+  - name: Backup generated etcd certificates
+    command: >
+      tar -czf {{ etcd_conf_dir }}/etcd-ca-certificate-backup-{{ ansible_date_time.epoch }}.tgz
+      {{ etcd_ca_dir }}
+    args:
+      warn: no
+    when: etcd_ca_certs_dir_stat.stat.exists | bool
+  - name: Remove CA certificate directory
+    file:
+      path: "{{ etcd_ca_dir }}"
+      state: absent
+    when: etcd_ca_certs_dir_stat.stat.exists | bool
+
+- name: Generate new etcd CA
+  hosts: oo_first_etcd
+  roles:
+  - role: etcd_ca
+    etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
+    etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+    etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
+
+- name: Create temp directory for syncing certs
+  hosts: localhost
+  connection: local
+  become: no
+  gather_facts: no
+  tasks:
+  - name: Create local temp directory for syncing certs
+    local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX
+    register: g_etcd_mktemp
+    changed_when: false
+
+- name: Distribute etcd CA to etcd hosts
+  hosts: oo_etcd_to_config
+  vars:
+    etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+  roles:
+  - etcd_common
+  tasks:
+  - name: Create a tarball of the etcd ca certs
+    command: >
+      tar -czvf {{ etcd_conf_dir }}/{{ etcd_ca_name }}.tgz
+        -C {{ etcd_ca_dir }} .
+    args:
+      creates: "{{ etcd_conf_dir }}/{{ etcd_ca_name }}.tgz"
+      warn: no
+    delegate_to: "{{ etcd_ca_host }}"
+    run_once: true
+  - name: Retrieve etcd ca cert tarball
+    fetch:
+      src: "{{ etcd_conf_dir }}/{{ etcd_ca_name }}.tgz"
+      dest: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}/"
+      flat: yes
+      fail_on_missing: yes
+      validate_checksum: yes
+    delegate_to: "{{ etcd_ca_host }}"
+    run_once: true
+  - name: Ensure ca directory exists
+    file:
+      path: "{{ etcd_ca_dir }}"
+      state: directory
+  - name: Unarchive etcd ca cert tarballs
+    unarchive:
+      src: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}/{{ etcd_ca_name }}.tgz"
+      dest: "{{ etcd_ca_dir }}"
+  - name: Read current etcd CA
+    slurp:
+      src: "{{ etcd_conf_dir }}/ca.crt"
+    register: g_current_etcd_ca_output
+  - name: Read new etcd CA
+    slurp:
+      src: "{{ etcd_ca_dir }}/ca.crt"
+    register: g_new_etcd_ca_output
+  - copy:
+      content: "{{ (g_new_etcd_ca_output.content|b64decode) + (g_current_etcd_ca_output.content|b64decode) }}"
+      dest: "{{ item }}/ca.crt"
+    with_items:
+    - "{{ etcd_conf_dir }}"
+    - "{{ etcd_ca_dir }}"
+
+- name: Retrieve etcd CA certificate
+  hosts: oo_first_etcd
+  roles:
+  - etcd_common
+  tasks:
+  - name: Retrieve etcd CA certificate
+    fetch:
+      src: "{{ etcd_conf_dir }}/ca.crt"
+      dest: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}/"
+      flat: yes
+      fail_on_missing: yes
+      validate_checksum: yes
+
+- name: Distribute etcd CA to masters
+  hosts: oo_masters_to_config
+  vars:
+    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+  tasks:
+  - name: Deploy CA certificate, key, bundle and serial
+    copy:
+      src: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}/ca.crt"
+      dest: "{{ openshift.common.config_base }}/master/master.etcd-ca.crt"
+    when: groups.oo_etcd_to_config | default([]) | length > 0
+
+- name: Delete temporary directory on localhost
+  hosts: localhost
+  connection: local
+  become: no
+  gather_facts: no
+  tasks:
+  - file:
+      name: "{{ g_etcd_mktemp.stdout }}"
+      state: absent
+    changed_when: false
+
+- include: ../../../common/openshift-etcd/restart.yml
+
+# Update master config when ca-bundle not referenced. Services will be
+# restarted below after new CA certificate has been distributed.
+- name: Ensure ca-bundle.crt is referenced in master configuration
+  hosts: oo_masters_to_config
+  tasks:
+  - slurp:
+      src: "{{ openshift.common.config_base }}/master/master-config.yaml"
+    register: g_master_config_output
+  - modify_yaml:
+      dest: "{{ openshift.common.config_base }}/master/master-config.yaml"
+      yaml_key: kubeletClientInfo.ca
+      yaml_value: ca-bundle.crt
+    when: (g_master_config_output.content|b64decode|from_yaml).kubeletClientInfo.ca != 'ca-bundle.crt'
+  - modify_yaml:
+      dest: "{{ openshift.common.config_base }}/master/master-config.yaml"
+      yaml_key: serviceAccountConfig.masterCA
+      yaml_value: ca-bundle.crt
+    when: (g_master_config_output.content|b64decode|from_yaml).serviceAccountConfig.masterCA != 'ca-bundle.crt'
+  - modify_yaml:
+      dest: "{{ openshift.common.config_base }}/master/master-config.yaml"
+      yaml_key: oauthConfig.masterCA
+      yaml_value: ca-bundle.crt
+    when: (g_master_config_output.content|b64decode|from_yaml).oauthConfig.masterCA != 'ca-bundle.crt'
+  - modify_yaml:
+      dest: "{{ openshift.common.config_base }}/master/master-config.yaml"
+      yaml_key: servingInfo.clientCA
+      yaml_value: ca-bundle.crt
+    when: (g_master_config_output.content|b64decode|from_yaml).servingInfo.clientCA != 'ca-bundle.crt'
+
+- name: Copy current OpenShift CA to legacy directory
+  hosts: oo_masters_to_config
+  pre_tasks:
+  - name: Create legacy-ca directory
+    file:
+      path: "{{ openshift.common.config_base }}/master/legacy-ca"
+      state: directory
+      mode: 0700
+      owner: root
+      group: root
+  - command: mktemp -u XXXXXX
+    register: g_legacy_ca_mktemp
+    changed_when: false
+  # Copy CA certificate, key, serial and bundle to legacy-ca with a
+  # prefix generated by mktemp, ie. XXXXXX-ca.crt.
+  #
+  # The following roles will pick up all CA certificates matching
+  # /.*-ca.crt/ in the legacy-ca directory and ensure they are present
+  # in the OpenShift CA bundle.
+  # - openshift_ca
+  # - openshift_master_certificates
+  # - openshift_node_certificates
+  - name: Copy current OpenShift CA to legacy directory
+    copy:
+      src: "{{ openshift.common.config_base }}/master/{{ item }}"
+      dest: "{{ openshift.common.config_base }}/master/legacy-ca/{{ g_legacy_ca_mktemp.stdout }}-{{ item }}"
+      remote_src: true
+    # It is possible that redeploying failed and files may be missing.
+    # Ignore errors in this case. Files should have been copied to
+    # legacy-ca directory in previous run.
+    ignore_errors: true
+    with_items:
+    - "ca.crt"
+    - "ca.key"
+    - "ca.serial.txt"
+    - "ca-bundle.crt"
+
+- name: Generate new OpenShift CA certificate
+  hosts: oo_first_master
+  pre_tasks:
+  - name: Create temporary directory for creating new CA certificate
+    command: >
+      mktemp -d /tmp/openshift-ansible-XXXXXXX
+    register: g_new_openshift_ca_mktemp
+    changed_when: false
+  roles:
+  - role: openshift_ca
+    # Set openshift_ca_config_dir to a temporary directory where CA
+    # will be created. We'll replace the existing CA with the CA
+    # created in the temporary directory.
+    openshift_ca_config_dir: "{{ g_new_openshift_ca_mktemp.stdout }}"
+    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+    openshift_master_hostnames: "{{ hostvars
+                                    | oo_select_keys(groups['oo_masters_to_config'] | default([]))
+                                    | oo_collect('openshift.common.all_hostnames')
+                                    | oo_flatten | unique }}"
+
+- name: Create temp directory for syncing certs
+  hosts: localhost
+  connection: local
+  become: no
+  gather_facts: no
+  tasks:
+  - name: Create local temp directory for syncing certs
+    local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX
+    register: g_master_mktemp
+    changed_when: false
+
+- name: Retrieve OpenShift CA
+  hosts: oo_first_master
+  vars:
+    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+  tasks:
+  - name: Retrieve CA certificate, key, bundle and serial
+    fetch:
+      src: "{{ hostvars[openshift_ca_host].g_new_openshift_ca_mktemp.stdout }}/{{ item }}"
+      dest: "{{ hostvars['localhost'].g_master_mktemp.stdout }}/"
+      flat: yes
+      fail_on_missing: yes
+      validate_checksum: yes
+    with_items:
+    - ca.crt
+    - ca.key
+    - ca-bundle.crt
+    - ca.serial.txt
+    delegate_to: "{{ openshift_ca_host }}"
+    run_once: true
+    changed_when: false
+
+- name: Distribute OpenShift CA to masters
+  hosts: oo_masters_to_config
+  vars:
+    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+  tasks:
+  - name: Deploy CA certificate, key, bundle and serial
+    copy:
+      src: "{{ hostvars['localhost'].g_master_mktemp.stdout }}/{{ item }}"
+      dest: "{{ openshift.common.config_base }}/master/"
+    with_items:
+    - ca.crt
+    - ca.key
+    - ca-bundle.crt
+    - ca.serial.txt
+  - name: Update master client kubeconfig CA data
+    kubeclient_ca:
+      client_path: "{{ openshift.common.config_base }}/master/openshift-master.kubeconfig"
+      ca_path: "{{ openshift.common.config_base }}/master/ca-bundle.crt"
+  - name: Update admin client kubeconfig CA data
+    kubeclient_ca:
+      client_path: "{{ openshift.common.config_base }}/master/admin.kubeconfig"
+      ca_path: "{{ openshift.common.config_base }}/master/ca-bundle.crt"
+  - name: Lookup default group for ansible_ssh_user
+    command: "/usr/bin/id -g {{ ansible_ssh_user }}"
+    changed_when: false
+    register: _ansible_ssh_user_gid
+  - set_fact:
+      client_users: "{{ [ansible_ssh_user, 'root'] | unique }}"
+  - name: Create the client config dir(s)
+    file:
+      path: "~{{ item }}/.kube"
+      state: directory
+      mode: 0700
+      owner: "{{ item }}"
+      group: "{{ 'root' if item == 'root' else _ansible_ssh_user_gid.stdout  }}"
+    with_items: "{{ client_users }}"
+  - name: Copy the admin client config(s)
+    copy:
+      src: "{{ openshift.common.config_base }}/master/admin.kubeconfig"
+      dest: "~{{ item }}/.kube/config"
+      remote_src: yes
+    with_items: "{{ client_users }}"
+  - name: Update the permissions on the admin client config(s)
+    file:
+      path: "~{{ item }}/.kube/config"
+      state: file
+      mode: 0700
+      owner: "{{ item }}"
+      group: "{{ 'root' if item == 'root' else _ansible_ssh_user_gid.stdout  }}"
+    with_items: "{{ client_users }}"
+
+- include: ../../../common/openshift-master/restart.yml
+
+- name: Distribute OpenShift CA certificate to nodes
+  hosts: oo_nodes_to_config
+  vars:
+    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+  tasks:
+  - copy:
+      src: "{{ hostvars['localhost'].g_master_mktemp.stdout }}/ca-bundle.crt"
+      dest: "{{ openshift.common.config_base }}/node/ca.crt"
+  - name: Copy OpenShift CA to system CA trust
+    copy:
+      src: "{{ item.cert }}"
+      dest: "/etc/pki/ca-trust/source/anchors/{{ item.id }}-{{ item.cert | basename }}"
+      remote_src: yes
+    with_items:
+    - id: openshift
+      cert: "{{ openshift.common.config_base }}/node/ca.crt"
+    notify:
+    - update ca trust
+  - name: Update node client kubeconfig CA data
+    kubeclient_ca:
+      client_path: "{{ openshift.common.config_base }}/node/system:node:{{ openshift.common.hostname }}.kubeconfig"
+      ca_path: "{{ openshift.common.config_base }}/node/ca.crt"
+  handlers:
+  # Normally this handler would restart docker after updating ca
+  # trust. We'll do that when we restart nodes to avoid restarting
+  # docker on all nodes in parallel.
+  - name: update ca trust
+    command: update-ca-trust
+
+- name: Delete temporary directory on CA host
+  hosts: oo_first_master
+  tasks:
+  - file:
+      path: "{{ g_new_openshift_ca_mktemp.stdout }}"
+      state: absent
+
+- name: Delete temporary directory on localhost
+  hosts: localhost
+  connection: local
+  become: no
+  gather_facts: no
+  tasks:
+  - file:
+      name: "{{ g_master_mktemp.stdout }}"
+      state: absent
+    changed_when: false
+
+- include: ../../../common/openshift-node/restart.yml
diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/etcd.yml b/playbooks/common/openshift-cluster/redeploy-certificates/etcd.yml
new file mode 100644
index 000000000..2963a5940
--- /dev/null
+++ b/playbooks/common/openshift-cluster/redeploy-certificates/etcd.yml
@@ -0,0 +1,66 @@
+---
+- name: Backup and remove generated etcd certificates
+  hosts: oo_first_etcd
+  any_errors_fatal: true
+  roles:
+    - etcd_common
+  post_tasks:
+    - name: Determine if generated etcd certificates exist
+      stat:
+        path: "{{ etcd_conf_dir }}/generated_certs"
+      register: etcd_generated_certs_dir_stat
+    - name: Backup generated etcd certificates
+      command: >
+        tar -czf {{ etcd_conf_dir }}/etcd-generated-certificate-backup-{{ ansible_date_time.epoch }}.tgz
+        {{ etcd_conf_dir }}/generated_certs
+      args:
+        warn: no
+      when: etcd_generated_certs_dir_stat.stat.exists | bool
+    - name: Remove generated etcd certificates
+      file:
+        path: "{{ item }}"
+        state: absent
+      with_items:
+        - "{{ etcd_conf_dir }}/generated_certs"
+
+- name: Backup and removed deployed etcd certificates
+  hosts: oo_etcd_to_config
+  any_errors_fatal: true
+  roles:
+    - etcd_common
+  post_tasks:
+    - name: Backup etcd certificates
+      command: >
+        tar -czvf /etc/etcd/etcd-server-certificate-backup-{{ ansible_date_time.epoch }}.tgz
+        {{ etcd_conf_dir }}/ca.crt
+        {{ etcd_conf_dir }}/server.crt
+        {{ etcd_conf_dir }}/server.key
+        {{ etcd_conf_dir }}/peer.crt
+        {{ etcd_conf_dir }}/peer.key
+      args:
+        warn: no
+
+- name: Redeploy etcd certificates
+  hosts: oo_etcd_to_config
+  any_errors_fatal: true
+  roles:
+    - role: openshift_etcd_server_certificates
+      etcd_certificates_redeploy: true
+      etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+      etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
+      etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
+      openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+
+- name: Redeploy etcd client certificates for masters
+  hosts: oo_masters_to_config
+  any_errors_fatal: true
+  roles:
+    - role: openshift_etcd_client_certificates
+      etcd_certificates_redeploy: true
+      etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+      etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}"
+      etcd_cert_config_dir: "{{ openshift.common.config_base }}/master"
+      etcd_cert_prefix: "master.etcd-"
+      openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+      openshift_master_count: "{{ openshift.master.master_count | default(groups.oo_masters | length) }}"
+      when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/filter_plugins b/playbooks/common/openshift-cluster/redeploy-certificates/filter_plugins
new file mode 120000
index 000000000..b1213dedb
--- /dev/null
+++ b/playbooks/common/openshift-cluster/redeploy-certificates/filter_plugins
@@ -0,0 +1 @@
+../../../../filter_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/library b/playbooks/common/openshift-cluster/redeploy-certificates/library
new file mode 120000
index 000000000..9a53f009d
--- /dev/null
+++ b/playbooks/common/openshift-cluster/redeploy-certificates/library
@@ -0,0 +1 @@
+../../../../library
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/lookup_plugins b/playbooks/common/openshift-cluster/redeploy-certificates/lookup_plugins
new file mode 120000
index 000000000..aff753026
--- /dev/null
+++ b/playbooks/common/openshift-cluster/redeploy-certificates/lookup_plugins
@@ -0,0 +1 @@
+../../../../lookup_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/masters.yml b/playbooks/common/openshift-cluster/redeploy-certificates/masters.yml
new file mode 100644
index 000000000..f653a111f
--- /dev/null
+++ b/playbooks/common/openshift-cluster/redeploy-certificates/masters.yml
@@ -0,0 +1,45 @@
+---
+- name: Redeploy master certificates
+  hosts: oo_masters_to_config
+  any_errors_fatal: true
+  vars:
+    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+    openshift_master_count: "{{ openshift.master.master_count | default(groups.oo_masters | length) }}"
+  pre_tasks:
+  - stat:
+      path: "{{ openshift_generated_configs_dir }}"
+    register: openshift_generated_configs_dir_stat
+  - name: Backup generated certificate and config directories
+    command: >
+      tar -czvf /etc/origin/master-node-cert-config-backup-{{ ansible_date_time.epoch }}.tgz
+      {{ openshift_generated_configs_dir }}
+      {{ openshift.common.config_base }}/master
+    when: openshift_generated_configs_dir_stat.stat.exists
+    delegate_to: "{{ openshift_ca_host }}"
+    run_once: true
+  - name: Remove generated certificate directories
+    file:
+      path: "{{ item }}"
+      state: absent
+    with_items:
+    - "{{ openshift_generated_configs_dir }}"
+  - name: Remove generated certificates
+    file:
+      path: "{{ openshift.common.config_base }}/master/{{ item }}"
+      state: absent
+    with_items:
+    - "{{ hostvars[inventory_hostname] | certificates_to_synchronize(include_keys=false, include_ca=false) }}"
+    - "etcd.server.crt"
+    - "etcd.server.key"
+    - "master.server.crt"
+    - "master.server.key"
+    - "openshift-master.crt"
+    - "openshift-master.key"
+    - "openshift-master.kubeconfig"
+  roles:
+  - role: openshift_master_certificates
+    openshift_master_etcd_hosts: "{{ hostvars
+                                     | oo_select_keys(groups['oo_etcd_to_config'] | default([]))
+                                     | oo_collect('openshift.common.hostname')
+                                     | default(none, true) }}"
+    openshift_certificates_redeploy: true
diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/nodes.yml b/playbooks/common/openshift-cluster/redeploy-certificates/nodes.yml
new file mode 100644
index 000000000..4990a03f2
--- /dev/null
+++ b/playbooks/common/openshift-cluster/redeploy-certificates/nodes.yml
@@ -0,0 +1,29 @@
+---
+- name: Ensure node directory is absent from generated configs
+  hosts: oo_first_master
+  tasks:
+  # The generated configs directory (/etc/origin/generated-configs) is
+  # backed up during redeployment of the control plane certificates.
+  # We need to ensure that the generated config directory for
+  # individual nodes has been deleted before continuing, so verify
+  # that it is missing here.
+  - name: Ensure node directories and tarballs are absent from generated configs
+    shell: >
+      rm -rf {{ openshift.common.config_base }}/generated-configs/node-*
+    args:
+      warn: no
+
+- name: Redeploy node certificates
+  hosts: oo_nodes_to_config
+  pre_tasks:
+  - name: Remove CA certificate
+    file:
+      path: "{{ item }}"
+      state: absent
+    with_items:
+    - "{{ openshift.common.config_base }}/node/ca.crt"
+  roles:
+  - role: openshift_node_certificates
+    openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}"
+    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+    openshift_certificates_redeploy: true
diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/registry.yml b/playbooks/common/openshift-cluster/redeploy-certificates/registry.yml
new file mode 100644
index 000000000..18b93e1d6
--- /dev/null
+++ b/playbooks/common/openshift-cluster/redeploy-certificates/registry.yml
@@ -0,0 +1,93 @@
+---
+- name: Update registry certificates
+  hosts: oo_first_master
+  vars:
+  tasks:
+  - name: Create temp directory for kubeconfig
+    command: mktemp -d /tmp/openshift-ansible-XXXXXX
+    register: mktemp
+    changed_when: false
+
+  - name: Copy admin client config(s)
+    command: >
+      cp {{ openshift.common.config_base }}/master//admin.kubeconfig {{ mktemp.stdout }}/admin.kubeconfig
+    changed_when: false
+
+  - name: Determine if docker-registry exists
+    command: >
+      {{ openshift.common.client_binary }} get dc/docker-registry -o json
+      --config={{ mktemp.stdout }}/admin.kubeconfig
+      -n default
+    register: l_docker_registry_dc
+    failed_when: false
+    changed_when: false
+
+  - set_fact:
+      docker_registry_env_vars: "{{ ((l_docker_registry_dc.stdout | from_json)['spec']['template']['spec']['containers'][0]['env']
+                                      | oo_collect('name'))
+                                      | default([]) }}"
+      docker_registry_secrets: "{{ ((l_docker_registry_dc.stdout | from_json)['spec']['template']['spec']['volumes']
+                                     | oo_collect('secret')
+                                     | oo_collect('secretName'))
+                                     | default([]) }}"
+    changed_when: false
+    when: l_docker_registry_dc.rc == 0
+
+  # Replace dc/docker-registry environment variable certificate data if set.
+  - name: Update docker-registry environment variables
+    shell: >
+      {{ openshift.common.client_binary }} env dc/docker-registry
+      OPENSHIFT_CA_DATA="$(cat /etc/origin/master/ca.crt)"
+      OPENSHIFT_CERT_DATA="$(cat /etc/origin/master/openshift-registry.crt)"
+      OPENSHIFT_KEY_DATA="$(cat /etc/origin/master/openshift-registry.key)"
+      --config={{ mktemp.stdout }}/admin.kubeconfig
+      -n default
+    when: l_docker_registry_dc.rc == 0 and 'OPENSHIFT_CA_DATA' in docker_registry_env_vars and 'OPENSHIFT_CERT_DATA' in docker_registry_env_vars and 'OPENSHIFT_KEY_DATA' in docker_registry_env_vars
+
+  # Replace dc/docker-registry certificate secret contents if set.
+  - block:
+    - name: Retrieve registry service IP
+      command: >
+        {{ openshift.common.client_binary }} get service docker-registry
+        -o jsonpath='{.spec.clusterIP}'
+        --config={{ mktemp.stdout }}/admin.kubeconfig
+        -n default
+      register: docker_registry_service_ip
+      changed_when: false
+
+    - set_fact:
+        docker_registry_route_hostname: "{{ 'docker-registry-default.' ~ (openshift.master.default_subdomain | default('router.default.svc.cluster.local', true)) }}"
+      changed_when: false
+
+    - name: Generate registry certificate
+      command: >
+        {{ openshift.common.client_binary }} adm ca create-server-cert
+        --signer-cert={{ openshift.common.config_base }}/master/ca.crt
+        --signer-key={{ openshift.common.config_base }}/master/ca.key
+        --signer-serial={{ openshift.common.config_base }}/master/ca.serial.txt
+        --hostnames="{{ docker_registry_service_ip.stdout }},docker-registry.default.svc.cluster.local,{{ docker_registry_route_hostname }}"
+        --cert={{ openshift.common.config_base }}/master/registry.crt
+        --key={{ openshift.common.config_base }}/master/registry.key
+
+    - name: Update registry certificates secret
+      shell: >
+        {{ openshift.common.client_binary }} secret new registry-certificates
+        {{ openshift.common.config_base }}/master/registry.crt
+        {{ openshift.common.config_base }}/master/registry.key
+        --config={{ mktemp.stdout }}/admin.kubeconfig
+        -n default
+        -o json | oc replace -f -
+    when: l_docker_registry_dc.rc == 0 and 'registry-certificates' in docker_registry_secrets and 'REGISTRY_HTTP_TLS_CERTIFICATE' in docker_registry_env_vars and 'REGISTRY_HTTP_TLS_KEY' in docker_registry_env_vars
+
+  - name: Redeploy docker registry
+    command: >
+      {{ openshift.common.client_binary }} deploy dc/docker-registry
+      --latest
+      --config={{ mktemp.stdout }}/admin.kubeconfig
+      -n default
+
+  - name: Delete temp directory
+    file:
+      name: "{{ mktemp.stdout }}"
+      state: absent
+    changed_when: False
diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/roles b/playbooks/common/openshift-cluster/redeploy-certificates/roles
new file mode 120000
index 000000000..4bdbcbad3
--- /dev/null
+++ b/playbooks/common/openshift-cluster/redeploy-certificates/roles
@@ -0,0 +1 @@
+../../../../roles
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/router.yml b/playbooks/common/openshift-cluster/redeploy-certificates/router.yml
new file mode 100644
index 000000000..03d64685d
--- /dev/null
+++ b/playbooks/common/openshift-cluster/redeploy-certificates/router.yml
@@ -0,0 +1,79 @@
+---
+- name: Update router certificates
+  hosts: oo_first_master
+  vars:
+  tasks:
+  - name: Create temp directory for kubeconfig
+    command: mktemp -d /tmp/openshift-ansible-XXXXXX
+    register: mktemp
+    changed_when: false
+
+  - name: Copy admin client config(s)
+    command: >
+      cp {{ openshift.common.config_base }}/master//admin.kubeconfig {{ mktemp.stdout }}/admin.kubeconfig
+    changed_when: false
+
+  - name: Determine if router exists
+    command: >
+      {{ openshift.common.client_binary }} get dc/router -o json
+      --config={{ mktemp.stdout }}/admin.kubeconfig
+      -n default
+    register: l_router_dc
+    failed_when: false
+    changed_when: false
+
+  - set_fact:
+      router_env_vars: "{{ ((l_router_dc.stdout | from_json)['spec']['template']['spec']['containers'][0]['env']
+                             | oo_collect('name'))
+                             | default([]) }}"
+      router_secrets: "{{ ((l_router_dc.stdout | from_json)['spec']['template']['spec']['volumes']
+                            | oo_collect('secret')
+                            | oo_collect('secretName'))
+                            | default([]) }}"
+    changed_when: false
+    when: l_router_dc.rc == 0
+
+  - name: Update router environment variables
+    shell: >
+      {{ openshift.common.client_binary }} env dc/router
+      OPENSHIFT_CA_DATA="$(cat /etc/origin/master/ca.crt)"
+      OPENSHIFT_CERT_DATA="$(cat /etc/origin/master/openshift-router.crt)"
+      OPENSHIFT_KEY_DATA="$(cat /etc/origin/master/openshift-router.key)"
+      --config={{ mktemp.stdout }}/admin.kubeconfig
+      -n default
+    when: l_router_dc.rc == 0 and 'OPENSHIFT_CA_DATA' in router_env_vars and 'OPENSHIFT_CERT_DATA' in router_env_vars and 'OPENSHIFT_KEY_DATA' in router_env_vars
+
+  - block:
+    - name: Generate router certificate
+      command: >
+        {{ openshift.common.client_binary }} adm ca create-server-cert
+        --hostnames=router.default.svc,router.default.svc.cluster.local
+        --signer-cert={{ openshift.common.config_base }}/master/service-signer.crt
+        --signer-key={{ openshift.common.config_base }}/master/service-signer.key
+        --signer-serial={{ openshift.common.config_base }}/master/ca.serial.txt
+        --cert={{ mktemp.stdout }}/tls.crt
+        --key={{ mktemp.stdout }}/tls.key
+
+    - name: Update router certificates secret
+      shell: >
+        {{ openshift.common.client_binary }} secret new router-certs
+        {{ mktemp.stdout }}/tls.crt
+        {{ mktemp.stdout }}/tls.key
+        --type=kubernetes.io/tls
+        --config={{ mktemp.stdout }}/admin.kubeconfig
+        -n default
+        -o json | oc replace -f -
+    when: l_router_dc.rc == 0 and 'router-certs' in router_secrets
+
+  - name: Redeploy router
+    command: >
+      {{ openshift.common.client_binary }} deploy dc/router
+      --latest
+      --config={{ mktemp.stdout }}/admin.kubeconfig
+      -n default
+
+  - name: Delete temp directory
+    file:
+      name: "{{ mktemp.stdout }}"
+      state: absent
+    changed_when: False
diff --git a/playbooks/common/openshift-cluster/std_include.yml b/playbooks/common/openshift-cluster/std_include.yml
new file mode 100644
index 000000000..078991b12
--- /dev/null
+++ b/playbooks/common/openshift-cluster/std_include.yml
@@ -0,0 +1,42 @@
+---
+- name: Create initial host groups for localhost
+  hosts: localhost
+  connection: local
+  become: no
+  gather_facts: no
+  tags:
+  - always
+  tasks:
+  - include_vars: ../../byo/openshift-cluster/cluster_hosts.yml
+  - name: Evaluate group l_oo_all_hosts
+    add_host:
+      name: "{{ item }}"
+      groups: l_oo_all_hosts
+    with_items: "{{ g_all_hosts | default([]) }}"
+    changed_when: no
+
+- name: Create initial host groups for all hosts
+  hosts: l_oo_all_hosts
+  gather_facts: no
+  tags:
+  - always
+  tasks:
+  - include_vars: ../../byo/openshift-cluster/cluster_hosts.yml
+  - set_fact:
+      openshift_deployment_type: "{{ deployment_type }}"
+
+- include: evaluate_groups.yml
+  tags:
+  - always
+
+- include: initialize_facts.yml
+  tags:
+  - always
+
+- include: validate_hostnames.yml
+  tags:
+  - node
+
+- include: initialize_openshift_version.yml
+  tags:
+  - always
diff --git a/playbooks/common/openshift-etcd/restart.yml b/playbooks/common/openshift-etcd/restart.yml
new file mode 100644
index 000000000..196c86f28
--- /dev/null
+++ b/playbooks/common/openshift-etcd/restart.yml
@@ -0,0 +1,9 @@
+---
+- name: Restart etcd
+  hosts: oo_etcd_to_config
+  serial: 1
+  tasks:
+    - name: restart etcd
+      service:
+        name: "{{ 'etcd' if not openshift.common.is_containerized | bool else 'etcd_container' }}"
+        state: restarted
diff --git a/playbooks/common/openshift-master/restart.yml b/playbooks/common/openshift-master/restart.yml
new file mode 100644
index 000000000..b35368bf1
--- /dev/null
+++ b/playbooks/common/openshift-master/restart.yml
@@ -0,0 +1,19 @@
+---
+- include: ../../common/openshift-master/validate_restart.yml
+
+- name: Restart masters
+  hosts: oo_masters_to_config
+  vars:
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
+  serial: 1
+  handlers:
+  - include: roles/openshift_master/handlers/main.yml
+    static: yes
+  roles:
+  - openshift_facts
+  post_tasks:
+  - include: ../../common/openshift-master/restart_hosts.yml
+    when: openshift_rolling_restart_mode | default('services') == 'system'
+
+  - include: ../../common/openshift-master/restart_services.yml
+    when: openshift_rolling_restart_mode | default('services') == 'services'
diff --git a/playbooks/common/openshift-node/restart.yml b/playbooks/common/openshift-node/restart.yml
new file mode 100644
index 000000000..6e9b1cca3
--- /dev/null
+++ b/playbooks/common/openshift-node/restart.yml
@@ -0,0 +1,47 @@
+---
+- name: Restart nodes
+  hosts: oo_nodes_to_config
+  serial: "{{ openshift_restart_nodes_serial | default(1) }}"
+  tasks:
+  - name: Restart docker
+    service: name=docker state=restarted
+
+  - name: Update docker facts
+    openshift_facts:
+      role: docker
+
+  - name: Restart containerized services
+    service: name={{ item }} state=started
+    with_items:
+    - etcd_container
+    - openvswitch
+    - "{{ openshift.common.service_type }}-master"
+    - "{{ openshift.common.service_type }}-master-api"
+    - "{{ openshift.common.service_type }}-master-controllers"
+    - "{{ openshift.common.service_type }}-node"
+    failed_when: false
+    when: openshift.common.is_containerized | bool
+
+  - name: Wait for master API to come back online
+    wait_for:
+      host: "{{ openshift.common.hostname }}"
+      state: started
+      delay: 10
+      port: "{{ openshift.master.api_port }}"
+    when: inventory_hostname in groups.oo_masters_to_config
+
+  - name: restart node
+    service:
+      name: "{{ openshift.common.service_type }}-node"
+      state: restarted
+
+  - name: Wait for node to be ready
+    command: >
+      {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} get node {{ openshift.common.hostname | lower }} --no-headers
+    register: node_output
+    delegate_to: "{{ groups.oo_first_master.0 }}"
+    when: inventory_hostname in groups.oo_nodes_to_config
+    until: "{{ node_output.stdout.split()[1].startswith('Ready')}}"
+    # Give the node two minutes to come back online.
+    retries: 24
+    delay: 5