summaryrefslogtreecommitdiffstats
path: root/playbooks/adhoc/upgrades/upgrade.yml
diff options
context:
space:
mode:
authorBrenton Leanhardt <bleanhar@redhat.com>2015-11-06 16:24:52 -0500
committerBrenton Leanhardt <bleanhar@redhat.com>2015-11-06 16:24:52 -0500
commit4517f7e1b4f189f54a616f4979579b7694a55ec7 (patch)
tree2a1666bc96c1c8d2fe94544e3ff5f095db2256d1 /playbooks/adhoc/upgrades/upgrade.yml
parent509e39d48ac205d740bb25004db666ad3b66d556 (diff)
parente38eae135ac4957576b263c3cc8f50e9805ea937 (diff)
downloadopenshift-4517f7e1b4f189f54a616f4979579b7694a55ec7.tar.gz
openshift-4517f7e1b4f189f54a616f4979579b7694a55ec7.tar.bz2
openshift-4517f7e1b4f189f54a616f4979579b7694a55ec7.tar.xz
openshift-4517f7e1b4f189f54a616f4979579b7694a55ec7.zip
Merge pull request #822 from detiber/upgradeFix2
Upgrade enhancements
Diffstat (limited to 'playbooks/adhoc/upgrades/upgrade.yml')
-rw-r--r--playbooks/adhoc/upgrades/upgrade.yml343
1 files changed, 261 insertions, 82 deletions
diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml
index 09f991b1d..09c472e58 100644
--- a/playbooks/adhoc/upgrades/upgrade.yml
+++ b/playbooks/adhoc/upgrades/upgrade.yml
@@ -1,35 +1,61 @@
---
-- name: Update deployment type
- hosts: OSEv3
+- name: Load master facts
+ hosts: masters
roles:
- openshift_facts
- post_tasks: # technically tasks are run after roles, but post_tasks is a bit more explicit.
- - openshift_facts:
- role: common
- local_facts:
- deployment_type: "{{ deployment_type }}"
- name: Verify upgrade can proceed
- hosts: masters
+ hosts: masters[0]
+ vars:
+ openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+ gather_facts: no
+ tasks:
+ # Pacemaker is currently the only supported upgrade path for multiple masters
+ - fail:
+ msg: "openshift_master_cluster_method must be set to 'pacemaker'"
+ when: openshift_master_ha | bool and ((openshift_master_cluster_method is not defined) or (openshift_master_cluster_method is defined and openshift_master_cluster_method != "pacemaker"))
+
+- name: Run pre-upgrade checks on first master
+ hosts: masters[0]
+ tasks:
+ # If this script errors out ansible will show the default stdout/stderr
+ # which contains details for the user:
+ - script: files/pre-upgrade-check
+
+- name: Evaluate etcd_hosts
+ hosts: localhost
tasks:
- # Checking the global deployment type rather than host facts, this is about
- # what the user is requesting.
- - fail: msg="Deployment type enterprise not supported for upgrade"
- when: deployment_type == "enterprise"
+ - name: Evaluate etcd hosts
+ add_host:
+ name: "{{ groups.masters.0 }}"
+ groups: etcd_hosts
+ when: hostvars[groups.masters.0].openshift.master.embedded_etcd | bool
+ - name: Evaluate etcd hosts
+ add_host:
+ name: "{{ item }}"
+ groups: etcd_hosts
+ with_items: groups.etcd
+ when: not hostvars[groups.masters.0].openshift.master.embedded_etcd | bool
- name: Backup etcd
- hosts: masters
+ hosts: etcd_hosts
vars:
embedded_etcd: "{{ openshift.master.embedded_etcd }}"
timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
roles:
- openshift_facts
tasks:
+
- stat: path=/var/lib/openshift
register: var_lib_openshift
+
+ - stat: path=/var/lib/origin
+ register: var_lib_origin
+
- name: Create origin symlink if necessary
file: src=/var/lib/openshift/ dest=/var/lib/origin state=link
- when: var_lib_openshift.stat.exists == True
+ when: var_lib_openshift.stat.exists == True and var_lib_origin.stat.exists == False
+
- name: Check available disk space for etcd backup
# We assume to be using the data dir for all backups.
shell: >
@@ -43,62 +69,71 @@
when: embedded_etcd | bool
- name: Abort if insufficient disk space for etcd backup
- fail: msg="{{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup, {{ avail_disk.stdout }} Kb available."
+ fail:
+ msg: >
+ {{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup,
+ {{ avail_disk.stdout }} Kb available.
when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int)
+
- name: Install etcd (for etcdctl)
- yum: pkg=etcd state=latest
+ yum:
+ pkg: etcd
+ state: latest
+
- name: Generate etcd backup
- command: etcdctl backup --data-dir={{ openshift.master.etcd_data_dir }} --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}
+ command: >
+ etcdctl backup --data-dir={{ openshift.master.etcd_data_dir }}
+ --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}
+
- name: Display location of etcd backup
- debug: msg="Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}"
+ debug:
+ msg: "Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}"
-- name: Upgrade base package on masters
- hosts: masters
+- name: Update deployment type
+ hosts: OSEv3
roles:
- openshift_facts
- vars:
- openshift_version: "{{ openshift_pkg_version | default('') }}"
- tasks:
- - name: Upgrade base package
- yum: pkg={{ openshift.common.service_type }}{{ openshift_version }} state=latest
+ post_tasks:
+ - openshift_facts:
+ role: common
+ local_facts:
+ deployment_type: "{{ deployment_type }}"
+
-- name: Evaluate oo_first_master
- hosts: localhost
- vars:
- g_masters_group: "{{ 'masters' }}"
- tasks:
- - name: Evaluate oo_first_master
- add_host:
- name: "{{ groups[g_masters_group][0] }}"
- groups: oo_first_master
- ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
- ansible_sudo: "{{ g_sudo | default(omit) }}"
- when: g_masters_group in groups and (groups[g_masters_group] | length) > 0
-
-# TODO: ideally we would check the new version, without installing it. (some
-# kind of yum repoquery? would need to handle openshift -> atomic-openshift
-# package rename)
- name: Perform upgrade version checking
- hosts: oo_first_master
+ hosts: masters[0]
tasks:
- - name: Determine new version
- command: >
- rpm -q --queryformat '%{version}' {{ openshift.common.service_type }}
- register: _new_version
+ - name: Clean yum cache
+ command: yum clean all
-- name: Ensure AOS 3.0.2 or Origin 1.0.6
- hosts: oo_first_master
- tasks:
- fail: This playbook requires Origin 1.0.6 or Atomic OpenShift 3.0.2 or later
- when: _new_version.stdout | version_compare('1.0.6','<') or ( _new_version.stdout | version_compare('3.0','>=' and _new_version.stdout | version_compare('3.0.2','<') )
+ - name: Determine available versions
+ script: files/versions.sh {{ openshift.common.service_type }} openshift
+ register: g_versions_result
+
+ - set_fact:
+ g_aos_versions: "{{ g_versions_result.stdout | from_yaml }}"
+
+ - set_fact:
+ g_new_version: "{{ g_aos_versions.curr_version.split('-', 1).0 if g_aos_versions.avail_version is none else g_aos_versions.avail_version.split('-', 1).0 }}"
+
+ - fail:
+ msg: This playbook requires Origin 1.0.6 or later
+ when: deployment_type == 'origin' and g_aos_versions.curr_version | version_compare('1.0.6','<')
+
+ - fail:
+ msg: This playbook requires Atomic OpenShift 3.0.2 or later
+ when: deployment_type in ['openshift-enterprise', 'atomic-openshift'] and g_aos_versions.curr_version | version_compare('3.0.2','<')
+
+ - fail:
+ msg: Atomic OpenShift 3.1 packages not found
+ when: deployment_type in ['openshift-enterprise', 'atomic-openshift'] and g_aos_versions.curr_version | version_compare('3.0.2.900','<') and (g_aos_versions.avail_version is none or g_aos_versions.avail_version | version_compare('3.0.2.900','<'))
+ # Deployment type 'enterprise' is no longer valid if we're upgrading to 3.1 or beyond.
+ # (still valid for 3.0.x to 3.0.y however) Using the global deployment_type here as
+ # we're checking what was requested by the upgrade, not the current type on the system.
+ - fail:
+ msg: "Deployment type enterprise not supported for upgrade"
+ when: deployment_type == "enterprise" and g_aos_versions.curr_version | version_compare('3.1', '>=')
-- name: Verify upgrade can proceed
- hosts: oo_first_master
- tasks:
- # Checking the global deployment type rather than host facts, this is about
- # what the user is requesting.
- - fail: msg="Deployment type 'enterprise' must be updated to 'openshift-enterprise' for upgrade to proceed"
- when: deployment_type == "enterprise" and (_new_version.stdout | version_compare('1.0.7', '>=') or _new_version.stdout | version_compare('3.1', '>='))
- name: Upgrade masters
hosts: masters
@@ -106,16 +141,138 @@
openshift_version: "{{ openshift_pkg_version | default('') }}"
tasks:
- name: Upgrade to latest available kernel
- yum: pkg=kernel state=latest
- - name: display just the deployment_type variable for the current host
- debug:
- var: hostvars[inventory_hostname]
+ yum:
+ pkg: kernel
+ state: latest
+
- name: Upgrade master packages
command: yum update -y {{ openshift.common.service_type }}-master{{ openshift_version }}
- - name: Upgrade master configuration.
- openshift_upgrade_config: from_version=3.0 to_version=3.1 role=master
- - name: Restart master services
- service: name="{{ openshift.common.service_type}}-master" state=restarted
+
+ - name: Ensure python-yaml present for config upgrade
+ yum:
+ pkg: python-yaml
+ state: installed
+
+ - name: Upgrade master configuration
+ openshift_upgrade_config:
+ from_version: '3.0'
+ to_version: '3.1'
+ role: master
+ config_base: "{{ hostvars[inventory_hostname].openshift.common.config_base }}"
+ when: deployment_type in ['openshift-enterprise', 'atomic-enterprise'] and g_aos_versions.curr_version | version_compare('3.1', '>=')
+
+ - set_fact:
+ master_certs_missing: True
+ master_cert_subdir: master-{{ openshift.common.hostname }}
+ master_cert_config_dir: "{{ openshift.common.config_base }}/master"
+
+- name: Create temp directory for syncing certs
+ hosts: localhost
+ gather_facts: no
+ tasks:
+ - name: Create local temp directory for syncing certs
+ local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX
+ register: g_master_mktemp
+ changed_when: False
+
+- name: Generate missing master certificates
+ hosts: masters[0]
+ vars:
+ master_hostnames: "{{ hostvars
+ | oo_select_keys(groups.masters)
+ | oo_collect('openshift.common.all_hostnames')
+ | oo_flatten | unique }}"
+ master_generated_certs_dir: "{{ openshift.common.config_base }}/generated-configs"
+ masters_needing_certs: "{{ hostvars
+ | oo_select_keys(groups.masters)
+ | difference([groups.masters.0]) }}"
+ sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
+ openshift_deployment_type: "{{ deployment_type }}"
+ roles:
+ - openshift_master_certificates
+ post_tasks:
+ - name: Remove generated etcd client certs when using external etcd
+ file:
+ path: "{{ master_generated_certs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}"
+ state: absent
+ when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
+ with_nested:
+ - masters_needing_certs
+ - - master.etcd-client.crt
+ - master.etcd-client.key
+
+ - name: Create a tarball of the master certs
+ command: >
+ tar -czvf {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz
+ -C {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }} .
+ with_items: masters_needing_certs
+
+ - name: Retrieve the master cert tarball from the master
+ fetch:
+ src: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz"
+ dest: "{{ sync_tmpdir }}/"
+ flat: yes
+ fail_on_missing: yes
+ validate_checksum: yes
+ with_items: masters_needing_certs
+
+- name: Sync certs and restart masters post configuration change
+ hosts: masters
+ vars:
+ sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
+ openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+ tasks:
+ - name: Unarchive the tarball on the master
+ unarchive:
+ src: "{{ sync_tmpdir }}/{{ master_cert_subdir }}.tgz"
+ dest: "{{ master_cert_config_dir }}"
+ when: inventory_hostname != groups.masters.0
+
+ - name: Restart master services
+ service: name="{{ openshift.common.service_type}}-master" state=restarted
+ when: not openshift_master_ha | bool
+
+- name: Destroy cluster
+ hosts: masters[0]
+ vars:
+ openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+ openshift_deployment_type: "{{ deployment_type }}"
+ pre_tasks:
+ - name: Check for configured cluster
+ stat:
+ path: /etc/corosync/corosync.conf
+ register: corosync_conf
+ when: openshift_master_ha | bool
+ - name: Destroy cluster
+ command: pcs cluster destroy --all
+ when: openshift_master_ha | bool and corosync_conf.stat.exists == true
+
+- name: Start pcsd on masters
+ hosts: masters
+ vars:
+ openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+ tasks:
+ - name: Start pcsd
+ service: name=pcsd state=started
+ when: openshift_master_ha | bool
+
+- name: Re-create cluster
+ hosts: masters[0]
+ vars:
+ openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+ openshift_deployment_type: "{{ deployment_type }}"
+ omc_cluster_hosts: "{{ groups.masters | join(' ') }}"
+ roles:
+ - role: openshift_master_cluster
+ when: openshift_master_ha | bool
+
+- name: Delete temporary directory on localhost
+ hosts: localhost
+ gather_facts: no
+ tasks:
+ - file: name={{ g_master_mktemp.stdout }} state=absent
+ changed_when: False
+
- name: Upgrade nodes
hosts: nodes
@@ -129,17 +286,17 @@
- name: Restart node services
service: name="{{ openshift.common.service_type }}-node" state=restarted
-- name: Update cluster policy
- hosts: oo_first_master
+- name: Update cluster policy and policy bindings
+ hosts: masters[0]
+ vars:
+ origin_reconcile_bindings: "{{ deployment_type == 'origin' and g_new_version | version_compare('1.0.6', '>') }}"
+ ent_reconcile_bindings: "{{ deployment_type in ['openshift-enterprise', 'atomic-enterprise'] and g_new_version | version_compare('3.0.2','>') }}"
tasks:
- name: oadm policy reconcile-cluster-roles --confirm
command: >
{{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig
policy reconcile-cluster-roles --confirm
-- name: Update cluster policy bindings
- hosts: oo_first_master
- tasks:
- name: oadm policy reconcile-cluster-role-bindings --confirm
command: >
{{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig
@@ -148,12 +305,40 @@
--exclude-groups=system:unauthenticated
--exclude-users=system:anonymous
--additive-only=true --confirm
- when: ( _new_version.stdout | version_compare('1.0.6', '>') and _new_version.stdout | version_compare('3.0','<') ) or _new_version.stdout | version_compare('3.0.2','>')
+ when: origin_reconcile_bindings | bool or ent_reconcile_bindings | bool
+
-- name: Upgrade default router
- hosts: oo_first_master
+- name: Restart masters post reconcile
+ hosts: masters
vars:
- - router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + _new_version.stdout ) }}"
+ openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+ tasks:
+ - name: Restart master services
+ service: name="{{ openshift.common.service_type}}-master" state=restarted
+ when: not openshift_master_ha | bool
+
+- name: Restart cluster post reconcile
+ hosts: masters[0]
+ vars:
+ openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+ tasks:
+ - name: Restart master cluster
+ command: pcs resource restart master
+ when: openshift_master_ha | bool
+ - name: Wait for the clustered master service to be available
+ wait_for:
+ host: "{{ openshift_master_cluster_vip }}"
+ port: 8443
+ state: started
+ timeout: 180
+ delay: 90
+ when: openshift_master_ha | bool
+
+- name: Upgrade default router and registry
+ hosts: masters[0]
+ vars:
+ - registry_image: "{{ openshift.master.registry_url | replace( '${component}', 'docker-registry' ) | replace ( '${version}', 'v' + g_new_version ) }}"
+ - router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + g_new_version ) }}"
- oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig"
tasks:
- name: Check for default router
@@ -188,12 +373,6 @@
{{ oc_cmd }} patch dc/router -p
'{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}"}]}}}}'
-- name: Upgrade default
- hosts: oo_first_master
- vars:
- - registry_image: "{{ openshift.master.registry_url | replace( '${component}', 'docker-registry' ) | replace ( '${version}', 'v' + _new_version.stdout ) }}"
- - oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig"
- tasks:
- name: Check for default registry
command: >
{{ oc_cmd }} get -n default dc/docker-registry
@@ -207,7 +386,7 @@
'{"spec":{"template":{"spec":{"containers":[{"name":"registry","image":"{{ registry_image }}"}]}}}}'
- name: Update image streams and templates
- hosts: oo_first_master
+ hosts: masters[0]
vars:
openshift_examples_import_command: "update"
openshift_deployment_type: "{{ deployment_type }}"