diff options
Diffstat (limited to 'playbooks/common/openshift-cluster/upgrades')
11 files changed, 368 insertions, 32 deletions
diff --git a/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check b/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check index b5459f312..e5c958ebb 100644 --- a/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check +++ b/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check @@ -111,13 +111,16 @@ def print_validation_header(): overwhelming the user. """ print """\ -At least one port name does not validate. Valid port names: +At least one port name is invalid and must be corrected before upgrading. +Please update or remove any resources with invalid port names. - * must be less that 16 chars + Valid port names must: + + * be less that 16 characters * have at least one letter - * only a-z0-9- - * do not start or end with - - * Dashes may not be next to eachother ('--') + * contain only a-z0-9- + * not start or end with - + * not contain dashes next to each other ('--') """ @@ -142,9 +145,9 @@ def main(): # Where the magic happens first_error = True for kind, path in [ + ('deploymentconfigs', ("spec", "template", "spec", "containers")), ('replicationcontrollers', ("spec", "template", "spec", "containers")), - ('pods', ("spec", "containers")), - ('deploymentconfigs', ("spec", "template", "spec", "containers"))]: + ('pods', ("spec", "containers"))]: for item in list_items(kind): namespace = item["metadata"]["namespace"] item_name = item["metadata"]["name"] diff --git a/playbooks/common/openshift-cluster/upgrades/files/versions.sh b/playbooks/common/openshift-cluster/upgrades/files/versions.sh index c7c966b60..3a1a8ebb1 100644 --- a/playbooks/common/openshift-cluster/upgrades/files/versions.sh +++ b/playbooks/common/openshift-cluster/upgrades/files/versions.sh @@ -1,9 +1,8 @@ #!/bin/bash -yum_installed=$(yum list installed "$@" 2>&1 | tail -n +2 | grep -v 'Installed Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'Error:' | awk '{ print $2 }' | tr '\n' ' ') - -yum_available=$(yum list available -q "$@" 2>&1 | tail -n +2 | grep -v 'Available Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'el7ose' | grep -v 'Error:' | awk '{ print $2 }' | tr '\n' ' ') +yum_installed=$(yum list installed -e 0 -q "$@" 2>&1 | tail -n +2 | awk '{ print $2 }' | sort -r | tr '\n' ' ') +yum_available=$(yum list available -e 0 -q "$@" 2>&1 | tail -n +2 | grep -v 'el7ose' | awk '{ print $2 }' | sort -r | tr '\n' ' ') echo "---" echo "curr_version: ${yum_installed}" diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_minor/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/upgrade.yml index 9f7e49b93..63c8ef756 100644 --- a/playbooks/common/openshift-cluster/upgrades/v3_0_minor/upgrade.yml +++ b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/upgrade.yml @@ -11,7 +11,7 @@ openshift_version: "{{ openshift_pkg_version | default('') }}" tasks: - name: Upgrade master packages - yum: pkg={{ openshift.common.service_type }}-master{{ openshift_version }} state=latest + action: "{{ ansible_pkg_mgr }} name={{ openshift.common.service_type }}-master{{ openshift_version }} state=latest" - name: Restart master services service: name="{{ openshift.common.service_type}}-master" state=restarted @@ -21,7 +21,7 @@ openshift_version: "{{ openshift_pkg_version | default('') }}" tasks: - name: Upgrade node packages - yum: pkg={{ openshift.common.service_type }}-node{{ openshift_version }} state=latest + action: "{{ ansible_pkg_mgr }} name={{ openshift.common.service_type }}-node{{ openshift_version }} state=latest" - name: Restart node services service: name="{{ openshift.common.service_type }}-node" state=restarted diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml index 0309e8a77..8ec379109 100644 --- a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml +++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml @@ -12,6 +12,8 @@ - name: Evaluate additional groups for upgrade hosts: localhost + connection: local + become: no tasks: - name: Evaluate etcd_hosts_to_backup add_host: @@ -27,6 +29,7 @@ hosts: oo_first_master vars: openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + target_version: "{{ '1.1' if deployment_type == 'origin' else '3.1' }}" gather_facts: no tasks: # Pacemaker is currently the only supported upgrade path for multiple masters @@ -43,8 +46,8 @@ - fail: msg: > openshift_pkg_version is {{ openshift_pkg_version }} which is not a - valid version for a 3.1 upgrade - when: openshift_pkg_version is defined and openshift_pkg_version.split('-',1).1 | version_compare('3.0.2.900','<') + valid version for a {{ target_version }} upgrade + when: openshift_pkg_version is defined and openshift_pkg_version.split('-',1).1 | version_compare(target_version ,'<') # If this script errors out ansible will show the default stdout/stderr # which contains details for the user: @@ -53,9 +56,11 @@ - name: Verify upgrade can proceed hosts: oo_masters_to_config:oo_nodes_to_config + vars: + target_version: "{{ '1.1' if deployment_type == 'origin' else '3.1' }}" tasks: - - name: Clean yum cache - command: yum clean all + - name: Clean package cache + command: "{{ ansible_pkg_mgr }} clean all" - set_fact: g_new_service_name: "{{ 'origin' if deployment_type =='origin' else 'atomic-openshift' }}" @@ -75,8 +80,8 @@ when: deployment_type == 'origin' and g_aos_versions.curr_version | version_compare('1.0.6','<') - fail: - msg: Atomic OpenShift 3.1 packages not found - when: g_aos_versions.curr_version | version_compare('3.0.2.900','<') and (g_aos_versions.avail_version is none or g_aos_versions.avail_version | version_compare('3.0.2.900','<')) + msg: Upgrade packages not found + when: (g_aos_versions.avail_version | default(g_aos_versions.curr_version, true) | version_compare(target_version, '<')) - set_fact: pre_upgrade_complete: True @@ -87,6 +92,8 @@ ############################################################################## - name: Gate on pre-upgrade checks hosts: localhost + connection: local + become: no vars: pre_upgrade_hosts: "{{ groups.oo_masters_to_config | union(groups.oo_nodes_to_config) }}" tasks: @@ -149,9 +156,7 @@ when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int) - name: Install etcd (for etcdctl) - yum: - pkg: etcd - state: latest + action: "{{ ansible_pkg_mgr }} name=etcd state=latest" - name: Generate etcd backup command: > @@ -171,6 +176,8 @@ ############################################################################## - name: Gate on etcd backup hosts: localhost + connection: local + become: no tasks: - set_fact: etcd_backup_completed: "{{ hostvars @@ -189,6 +196,8 @@ ############################################################################### - name: Create temp directory for syncing certs hosts: localhost + connection: local + become: no gather_facts: no tasks: - name: Create local temp directory for syncing certs @@ -222,17 +231,14 @@ openshift_version: "{{ openshift_pkg_version | default('') }}" tasks: - name: Upgrade to latest available kernel - yum: - pkg: kernel - state: latest + action: "{{ ansible_pkg_mgr}} name=kernel state=latest" - name: Upgrade master packages - command: yum update -y {{ openshift.common.service_type }}-master{{ openshift_version }} + command: "{{ ansible_pkg_mgr}} update -y {{ openshift.common.service_type }}-master{{ openshift_version }}" - name: Ensure python-yaml present for config upgrade - yum: - pkg: PyYAML - state: installed + action: "{{ ansible_pkg_mgr }} name=PyYAML state=present" + when: not openshift.common.is_atomic | bool - name: Upgrade master configuration openshift_upgrade_config: @@ -242,7 +248,31 @@ config_base: "{{ hostvars[inventory_hostname].openshift.common.config_base }}" - set_fact: - master_certs_missing: True + openshift_master_certs_no_etcd: + - admin.crt + - master.kubelet-client.crt + - "{{ 'master.proxy-client.crt' if openshift.common.version_greater_than_3_1_or_1_1 else omit }}" + - master.server.crt + - openshift-master.crt + - openshift-registry.crt + - openshift-router.crt + - etcd.server.crt + openshift_master_certs_etcd: + - master.etcd-client.crt + + - set_fact: + openshift_master_certs: "{{ (openshift_master_certs_no_etcd | union(openshift_master_certs_etcd)) if (groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config) else openshift_master_certs_no_etcd }}" + + - name: Check status of master certificates + stat: + path: "{{ openshift.common.config_base }}/master/{{ item }}" + with_items: openshift_master_certs + register: g_master_cert_stat_result + + - set_fact: + master_certs_missing: "{{ False in (g_master_cert_stat_result.results + | oo_collect(attribute='stat.exists') + | list ) }}" master_cert_subdir: master-{{ openshift.common.hostname }} master_cert_config_dir: "{{ openshift.common.config_base }}/master" @@ -256,8 +286,8 @@ | oo_flatten | unique }}" master_generated_certs_dir: "{{ openshift.common.config_base }}/generated-configs" masters_needing_certs: "{{ hostvars - | oo_select_keys(groups.oo_masters_to_config) - | difference([groups.oo_first_master.0]) }}" + | oo_select_keys(groups['oo_masters_to_config'] | difference(groups['oo_first_master'])) + | oo_filter_list(filter_attr='master_certs_missing') }}" sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" openshift_deployment_type: "{{ deployment_type }}" roles: @@ -339,6 +369,8 @@ - name: Delete temporary directory on localhost hosts: localhost + connection: local + become: no gather_facts: no tasks: - file: name={{ g_master_mktemp.stdout }} state=absent @@ -357,6 +389,8 @@ ############################################################################## - name: Gate on master update hosts: localhost + connection: local + become: no tasks: - set_fact: master_update_completed: "{{ hostvars @@ -380,7 +414,7 @@ - openshift_facts tasks: - name: Upgrade node packages - command: yum update -y {{ openshift.common.service_type }}-node{{ openshift_version }} + command: "{{ ansible_pkg_mgr }} update -y {{ openshift.common.service_type }}-node{{ openshift_version }}" - name: Restart node service service: name="{{ openshift.common.service_type }}-node" state=restarted @@ -388,6 +422,24 @@ - name: Ensure node service enabled service: name="{{ openshift.common.service_type }}-node" state=started enabled=yes + - name: Install Ceph storage plugin dependencies + action: "{{ ansible_pkg_mgr }} name=ceph-common state=present" + + - name: Install GlusterFS storage plugin dependencies + action: "{{ ansible_pkg_mgr }} name=glusterfs-fuse state=present" + + - name: Set sebooleans to allow gluster storage plugin access from containers + seboolean: + name: "{{ item }}" + state: yes + persistent: yes + when: ansible_selinux and ansible_selinux.status == "enabled" + with_items: + - virt_use_fusefs + - virt_sandbox_use_fusefs + register: sebool_result + failed_when: "'state' not in sebool_result and 'msg' in sebool_result and 'SELinux boolean {{ item }} does not exist' not in sebool_result.msg" + - set_fact: node_update_complete: True @@ -397,6 +449,8 @@ ############################################################################## - name: Gate on nodes update hosts: localhost + connection: local + become: no tasks: - set_fact: node_update_completed: "{{ hostvars @@ -464,6 +518,8 @@ ############################################################################## - name: Gate on reconcile hosts: localhost + connection: local + become: no tasks: - set_fact: reconcile_completed: "{{ hostvars diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_minor/filter_plugins b/playbooks/common/openshift-cluster/upgrades/v3_1_minor/filter_plugins new file mode 120000 index 000000000..27ddaa18b --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_1_minor/filter_plugins @@ -0,0 +1 @@ +../../../../../filter_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_minor/library b/playbooks/common/openshift-cluster/upgrades/v3_1_minor/library new file mode 120000 index 000000000..53bed9684 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_1_minor/library @@ -0,0 +1 @@ +../library
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_minor/lookup_plugins b/playbooks/common/openshift-cluster/upgrades/v3_1_minor/lookup_plugins new file mode 120000 index 000000000..cf407f69b --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_1_minor/lookup_plugins @@ -0,0 +1 @@ +../../../../../lookup_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_minor/post.yml b/playbooks/common/openshift-cluster/upgrades/v3_1_minor/post.yml new file mode 100644 index 000000000..d8336fcae --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_1_minor/post.yml @@ -0,0 +1,50 @@ +--- +############################################################################### +# Post upgrade - Upgrade default router, default registry and examples +############################################################################### +- name: Upgrade default router and default registry + hosts: oo_first_master + vars: + openshift_deployment_type: "{{ deployment_type }}" + registry_image: "{{ openshift.master.registry_url | replace( '${component}', 'docker-registry' ) | replace ( '${version}', 'v' + g_new_version ) }}" + router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + g_new_version ) }}" + oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig" + roles: + # Create the new templates shipped in 3.1.z, existing templates are left + # unmodified. This prevents the subsequent role definition for + # openshift_examples from failing when trying to replace templates that do + # not already exist. We could have potentially done a replace --force to + # create and update in one step. + - openshift_examples + # Update the existing templates + - role: openshift_examples + openshift_examples_import_command: replace + pre_tasks: + - name: Check for default router + command: > + {{ oc_cmd }} get -n default dc/router + register: _default_router + failed_when: false + changed_when: false + + - name: Check for default registry + command: > + {{ oc_cmd }} get -n default dc/docker-registry + register: _default_registry + failed_when: false + changed_when: false + + - name: Update router image to current version + when: _default_router.rc == 0 + command: > + {{ oc_cmd }} patch dc/router -p + '{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}"}]}}}}' + --api-version=v1 + + - name: Update registry image to current version + when: _default_registry.rc == 0 + command: > + {{ oc_cmd }} patch dc/docker-registry -p + '{"spec":{"template":{"spec":{"containers":[{"name":"registry","image":"{{ registry_image }}"}]}}}}' + --api-version=v1 + diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_minor/pre.yml b/playbooks/common/openshift-cluster/upgrades/v3_1_minor/pre.yml new file mode 100644 index 000000000..91780de09 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_1_minor/pre.yml @@ -0,0 +1,87 @@ +--- +############################################################################### +# Evaluate host groups and gather facts +############################################################################### +- name: Load openshift_facts + hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config:oo_lb_to_config + roles: + - openshift_facts + +############################################################################### +# Pre-upgrade checks +############################################################################### +- name: Verify upgrade can proceed + hosts: oo_first_master + vars: + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + target_version: "{{ '1.1.1' if deployment_type == 'origin' else '3.1.1' }}" + gather_facts: no + tasks: + - fail: + msg: > + This upgrade is only supported for origin, openshift-enterprise, and online + deployment types + when: deployment_type not in ['origin','openshift-enterprise', 'online'] + + - fail: + msg: > + openshift_pkg_version is {{ openshift_pkg_version }} which is not a + valid version for a {{ target_version }} upgrade + when: openshift_pkg_version is defined and openshift_pkg_version.split('-',1).1 | version_compare(target_version ,'<') + +- name: Verify upgrade can proceed + hosts: oo_masters_to_config:oo_nodes_to_config + vars: + target_version: "{{ '1.1.1' if deployment_type == 'origin' else '3.1.1' }}" + tasks: + - name: Clean package cache + command: "{{ ansible_pkg_mgr }} clean all" + + - set_fact: + g_new_service_name: "{{ 'origin' if deployment_type =='origin' else 'atomic-openshift' }}" + + - name: Determine available versions + script: ../files/versions.sh {{ g_new_service_name }} openshift + register: g_versions_result + + - set_fact: + g_aos_versions: "{{ g_versions_result.stdout | from_yaml }}" + + - set_fact: + g_new_version: "{{ g_aos_versions.curr_version.split('-', 1).0 if g_aos_versions.avail_version is none else g_aos_versions.avail_version.split('-', 1).0 }}" + + - fail: + msg: This playbook requires Origin 1.1 or later + when: deployment_type == 'origin' and g_aos_versions.curr_version | version_compare('1.1','<') + + - fail: + msg: This playbook requires Atomic Enterprise Platform/OpenShift Enterprise 3.1 or later + when: deployment_type == 'atomic-openshift' and g_aos_versions.curr_version | version_compare('3.1','<') + + - fail: + msg: Upgrade packages not found + when: (g_aos_versions.avail_version | default(g_aos_versions.curr_version, true) | version_compare(target_version, '<')) + + - set_fact: + pre_upgrade_complete: True + + +############################################################################## +# Gate on pre-upgrade checks +############################################################################## +- name: Gate on pre-upgrade checks + hosts: localhost + connection: local + become: no + vars: + pre_upgrade_hosts: "{{ groups.oo_masters_to_config | union(groups.oo_nodes_to_config) }}" + tasks: + - set_fact: + pre_upgrade_completed: "{{ hostvars + | oo_select_keys(pre_upgrade_hosts) + | oo_collect('inventory_hostname', {'pre_upgrade_complete': true}) }}" + - set_fact: + pre_upgrade_failed: "{{ pre_upgrade_hosts | difference(pre_upgrade_completed) }}" + - fail: + msg: "Upgrade cannot continue. The following hosts did not complete pre-upgrade checks: {{ pre_upgrade_failed | join(',') }}" + when: pre_upgrade_failed | length > 0 diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_minor/roles b/playbooks/common/openshift-cluster/upgrades/v3_1_minor/roles new file mode 120000 index 000000000..6bc1a7aef --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_1_minor/roles @@ -0,0 +1 @@ +../../../../../roles
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_minor/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_1_minor/upgrade.yml new file mode 100644 index 000000000..81dbba1e3 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_1_minor/upgrade.yml @@ -0,0 +1,137 @@ +--- +############################################################################### +# The restart playbook should be run after this playbook completes. +############################################################################### + +############################################################################### +# Upgrade Masters +############################################################################### +- name: Upgrade master packages and configuration + hosts: oo_masters_to_config + vars: + openshift_version: "{{ openshift_pkg_version | default('') }}" + tasks: + - name: Upgrade master packages + command: "{{ ansible_pkg_mgr}} update -y {{ openshift.common.service_type }}-master{{ openshift_version }}" + + - name: Ensure python-yaml present for config upgrade + action: "{{ ansible_pkg_mgr }} name=PyYAML state=present" + when: not openshift.common.is_atomic | bool + +# Currently 3.1.1 does not have any new configuration settings +# +# - name: Upgrade master configuration +# openshift_upgrade_config: +# from_version: '3.0' +# to_version: '3.1' +# role: master +# config_base: "{{ hostvars[inventory_hostname].openshift.common.config_base }}" + +- name: Set master update status to complete + hosts: oo_masters_to_config + tasks: + - set_fact: + master_update_complete: True + +############################################################################## +# Gate on master update complete +############################################################################## +- name: Gate on master update + hosts: localhost + connection: local + become: no + tasks: + - set_fact: + master_update_completed: "{{ hostvars + | oo_select_keys(groups.oo_masters_to_config) + | oo_collect('inventory_hostname', {'master_update_complete': true}) }}" + - set_fact: + master_update_failed: "{{ groups.oo_masters_to_config | difference(master_update_completed) }}" + - fail: + msg: "Upgrade cannot continue. The following masters did not finish updating: {{ master_update_failed | join(',') }}" + when: master_update_failed | length > 0 + +############################################################################### +# Upgrade Nodes +############################################################################### +- name: Upgrade nodes + hosts: oo_nodes_to_config + vars: + openshift_version: "{{ openshift_pkg_version | default('') }}" + roles: + - openshift_facts + tasks: + - name: Upgrade node packages + command: "{{ ansible_pkg_mgr }} update -y {{ openshift.common.service_type }}-node{{ openshift_version }}" + + - name: Restart node service + service: name="{{ openshift.common.service_type }}-node" state=restarted + + - set_fact: + node_update_complete: True + +############################################################################## +# Gate on nodes update +############################################################################## +- name: Gate on nodes update + hosts: localhost + connection: local + become: no + tasks: + - set_fact: + node_update_completed: "{{ hostvars + | oo_select_keys(groups.oo_nodes_to_config) + | oo_collect('inventory_hostname', {'node_update_complete': true}) }}" + - set_fact: + node_update_failed: "{{ groups.oo_nodes_to_config | difference(node_update_completed) }}" + - fail: + msg: "Upgrade cannot continue. The following nodes did not finish updating: {{ node_update_failed | join(',') }}" + when: node_update_failed | length > 0 + +############################################################################### +# Reconcile Cluster Roles and Cluster Role Bindings +############################################################################### +- name: Reconcile Cluster Roles and Cluster Role Bindings + hosts: oo_masters_to_config + vars: + origin_reconcile_bindings: "{{ deployment_type == 'origin' and g_new_version | version_compare('1.0.6', '>') }}" + ent_reconcile_bindings: true + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + tasks: + - name: Reconcile Cluster Roles + command: > + {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig + policy reconcile-cluster-roles --confirm + run_once: true + + - name: Reconcile Cluster Role Bindings + command: > + {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig + policy reconcile-cluster-role-bindings + --exclude-groups=system:authenticated + --exclude-groups=system:unauthenticated + --exclude-users=system:anonymous + --additive-only=true --confirm + when: origin_reconcile_bindings | bool or ent_reconcile_bindings | bool + run_once: true + + - set_fact: + reconcile_complete: True + +############################################################################## +# Gate on reconcile +############################################################################## +- name: Gate on reconcile + hosts: localhost + connection: local + become: no + tasks: + - set_fact: + reconcile_completed: "{{ hostvars + | oo_select_keys(groups.oo_masters_to_config) + | oo_collect('inventory_hostname', {'reconcile_complete': true}) }}" + - set_fact: + reconcile_failed: "{{ groups.oo_masters_to_config | difference(reconcile_completed) }}" + - fail: + msg: "Upgrade cannot continue. The following masters did not finish reconciling: {{ reconcile_failed | join(',') }}" + when: reconcile_failed | length > 0 |