diff options
Diffstat (limited to 'playbooks/common/openshift-cluster')
50 files changed, 1121 insertions, 835 deletions
diff --git a/playbooks/common/openshift-cluster/additional_config.yml b/playbooks/common/openshift-cluster/additional_config.yml index 26b31d313..c0ea93d2c 100644 --- a/playbooks/common/openshift-cluster/additional_config.yml +++ b/playbooks/common/openshift-cluster/additional_config.yml @@ -1,3 +1,4 @@ +--- - name: Additional master configuration hosts: oo_first_master vars: @@ -11,6 +12,8 @@ - role: openshift_examples registry_url: "{{ openshift.master.registry_url }}" when: openshift.common.install_examples | bool + - role: openshift_hosted_templates + registry_url: "{{ openshift.master.registry_url }}" - role: openshift_manageiq when: openshift.common.use_manageiq | bool - role: cockpit diff --git a/playbooks/common/openshift-cluster/config.yml b/playbooks/common/openshift-cluster/config.yml index d6a99fcda..0f226f5f9 100644 --- a/playbooks/common/openshift-cluster/config.yml +++ b/playbooks/common/openshift-cluster/config.yml @@ -12,8 +12,10 @@ - node - include: initialize_openshift_version.yml + tags: + - always -- name: Set oo_options +- name: Set oo_option facts hosts: oo_all_hosts tags: - always diff --git a/playbooks/common/openshift-cluster/enable_dnsmasq.yml b/playbooks/common/openshift-cluster/enable_dnsmasq.yml index 4cfe8617e..ca5177852 100644 --- a/playbooks/common/openshift-cluster/enable_dnsmasq.yml +++ b/playbooks/common/openshift-cluster/enable_dnsmasq.yml @@ -59,7 +59,7 @@ vars: openshift_deployment_type: "{{ deployment_type }}" roles: - - openshift_node_dnsmasq + - openshift_node_dnsmasq post_tasks: - modify_yaml: dest: "{{ openshift.common.config_base }}/node/node-config.yaml" diff --git a/playbooks/common/openshift-cluster/evaluate_groups.yml b/playbooks/common/openshift-cluster/evaluate_groups.yml index b3e02fb97..45a4875a3 100644 --- a/playbooks/common/openshift-cluster/evaluate_groups.yml +++ b/playbooks/common/openshift-cluster/evaluate_groups.yml @@ -7,27 +7,27 @@ tasks: - fail: msg: This playbook requires g_etcd_hosts to be set - when: g_etcd_hosts is not defined + when: "{{ g_etcd_hosts is not defined }}" - fail: msg: This playbook requires g_master_hosts or g_new_master_hosts to be set - when: g_master_hosts is not defined and g_new_master_hosts is not defined + when: "{{ g_master_hosts is not defined and g_new_master_hosts is not defined }}" - fail: msg: This playbook requires g_node_hosts or g_new_node_hosts to be set - when: g_node_hosts is not defined and g_new_node_hosts is not defined + when: "{{ g_node_hosts is not defined and g_new_node_hosts is not defined }}" - fail: msg: This playbook requires g_lb_hosts to be set - when: g_lb_hosts is not defined + when: "{{ g_lb_hosts is not defined }}" - fail: msg: This playbook requires g_nfs_hosts to be set - when: g_nfs_hosts is not defined + when: "{{ g_nfs_hosts is not defined }}" - fail: msg: The nfs group must be limited to one host - when: (groups[g_nfs_hosts] | default([])) | length > 1 + when: "{{ (groups[g_nfs_hosts] | default([])) | length > 1 }}" - name: Evaluate oo_all_hosts add_host: @@ -36,6 +36,7 @@ ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" ansible_become: "{{ g_sudo | default(omit) }}" with_items: "{{ g_all_hosts | default([]) }}" + changed_when: no - name: Evaluate oo_masters add_host: @@ -44,6 +45,7 @@ ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" ansible_become: "{{ g_sudo | default(omit) }}" with_items: "{{ g_master_hosts | union(g_new_master_hosts) | default([]) }}" + changed_when: no - name: Evaluate oo_etcd_to_config add_host: @@ -52,6 +54,7 @@ ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" ansible_become: "{{ g_sudo | default(omit) }}" with_items: "{{ g_etcd_hosts | default([]) }}" + changed_when: no - name: Evaluate oo_masters_to_config add_host: @@ -60,6 +63,7 @@ ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" ansible_become: "{{ g_sudo | default(omit) }}" with_items: "{{ g_new_master_hosts | default(g_master_hosts | default([], true), true) }}" + changed_when: no - name: Evaluate oo_nodes_to_config add_host: @@ -68,23 +72,26 @@ ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" ansible_become: "{{ g_sudo | default(omit) }}" with_items: "{{ g_new_node_hosts | default(g_node_hosts | default([], true), true) }}" + changed_when: no # Skip adding the master to oo_nodes_to_config when g_new_node_hosts is - - name: Evaluate oo_nodes_to_config + - name: Add master to oo_nodes_to_config add_host: name: "{{ item }}" groups: oo_nodes_to_config ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" ansible_become: "{{ g_sudo | default(omit) }}" with_items: "{{ g_master_hosts | default([]) }}" - when: g_nodeonmaster | default(false) | bool and not g_new_node_hosts | default(false) | bool + when: "{{ g_nodeonmaster | default(false) | bool and not g_new_node_hosts | default(false) | bool }}" + changed_when: no - name: Evaluate oo_first_etcd add_host: name: "{{ g_etcd_hosts[0] }}" groups: oo_first_etcd ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" - when: g_etcd_hosts|length > 0 + when: "{{ g_etcd_hosts|length > 0 }}" + changed_when: no - name: Evaluate oo_first_master add_host: @@ -92,7 +99,8 @@ groups: oo_first_master ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" ansible_become: "{{ g_sudo | default(omit) }}" - when: g_master_hosts|length > 0 + when: "{{ g_master_hosts|length > 0 }}" + changed_when: no - name: Evaluate oo_lb_to_config add_host: @@ -101,6 +109,7 @@ ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" ansible_become: "{{ g_sudo | default(omit) }}" with_items: "{{ g_lb_hosts | default([]) }}" + changed_when: no - name: Evaluate oo_nfs_to_config add_host: @@ -109,3 +118,4 @@ ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" ansible_become: "{{ g_sudo | default(omit) }}" with_items: "{{ g_nfs_hosts | default([]) }}" + changed_when: no diff --git a/playbooks/common/openshift-cluster/initialize_facts.yml b/playbooks/common/openshift-cluster/initialize_facts.yml index 04dde632b..18f99728c 100644 --- a/playbooks/common/openshift-cluster/initialize_facts.yml +++ b/playbooks/common/openshift-cluster/initialize_facts.yml @@ -1,7 +1,11 @@ --- +- name: Ensure that all non-node hosts are accessible + hosts: oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config:oo_nfs_to_config + any_errors_fatal: true + tasks: + - name: Initialize host facts hosts: oo_all_hosts - any_errors_fatal: true roles: - openshift_facts tasks: @@ -11,3 +15,5 @@ hostname: "{{ openshift_hostname | default(None) }}" - set_fact: openshift_docker_hosted_registry_network: "{{ hostvars[groups.oo_first_master.0].openshift.common.portal_net }}" + - set_fact: + openshift_deployment_type: "{{ deployment_type }}" diff --git a/playbooks/common/openshift-cluster/initialize_openshift_version.yml b/playbooks/common/openshift-cluster/initialize_openshift_version.yml index 7112a6084..a1bd1bd92 100644 --- a/playbooks/common/openshift-cluster/initialize_openshift_version.yml +++ b/playbooks/common/openshift-cluster/initialize_openshift_version.yml @@ -1,5 +1,22 @@ --- # NOTE: requires openshift_facts be run +- hosts: l_oo_all_hosts + gather_facts: no + tasks: + # See: + # https://bugzilla.redhat.com/show_bug.cgi?id=1395047 + # https://bugzilla.redhat.com/show_bug.cgi?id=1282961 + # https://github.com/openshift/openshift-ansible/issues/1138 + - name: Check for bad combinations of yum and subscription-manager + command: > + {{ repoquery_cmd }} --installed --qf '%{version}' "yum" + register: yum_ver_test + changed_when: false + when: not openshift.common.is_atomic | bool + - fail: + msg: Incompatible versions of yum and subscription-manager found. You may need to update yum and yum-utils. + when: "not openshift.common.is_atomic | bool and 'Plugin \"search-disabled-repos\" requires API 2.7. Supported API is 2.6.' in yum_ver_test.stdout" + - name: Determine openshift_version to configure on first master hosts: oo_first_master roles: diff --git a/playbooks/common/openshift-cluster/openshift_hosted.yml b/playbooks/common/openshift-cluster/openshift_hosted.yml index 4aca4daf4..ec5b18389 100644 --- a/playbooks/common/openshift-cluster/openshift_hosted.yml +++ b/playbooks/common/openshift-cluster/openshift_hosted.yml @@ -19,112 +19,29 @@ openshift_hosted_router_registryurl: "{{ hostvars[groups.oo_first_master.0].openshift.master.registry_url }}" openshift_hosted_registry_registryurl: "{{ hostvars[groups.oo_first_master.0].openshift.master.registry_url }}" when: "'master' in hostvars[groups.oo_first_master.0].openshift and 'registry_url' in hostvars[groups.oo_first_master.0].openshift.master" + - set_fact: + logging_hostname: "{{ openshift_hosted_logging_hostname | default('kibana.' ~ (openshift_master_default_subdomain | default('router.default.svc.cluster.local', true))) }}" + logging_ops_hostname: "{{ openshift_hosted_logging_ops_hostname | default('kibana-ops.' ~ (openshift_master_default_subdomain | default('router.default.svc.cluster.local', true))) }}" + logging_master_public_url: "{{ openshift_hosted_logging_master_public_url | default(openshift.master.public_api_url) }}" + logging_elasticsearch_cluster_size: "{{ openshift_hosted_logging_elasticsearch_cluster_size | default(1) }}" + logging_elasticsearch_ops_cluster_size: "{{ openshift_hosted_logging_elasticsearch_ops_cluster_size | default(1) }}" roles: - - role: openshift_cli - - role: openshift_hosted_facts - - role: openshift_projects - # TODO: Move standard project definitions to openshift_hosted/vars/main.yml - # Vars are not accessible in meta/main.yml in ansible-1.9.x - openshift_projects: "{{ openshift_additional_projects | default({}) | oo_merge_dicts({'default':{'default_node_selector':''},'openshift-infra':{'default_node_selector':''},'logging':{'default_node_selector':''}}) }}" - - role: openshift_serviceaccounts - openshift_serviceaccounts_names: - - router - openshift_serviceaccounts_namespace: default - openshift_serviceaccounts_sccs: - - hostnetwork - when: openshift.common.version_gte_3_2_or_1_2 - - role: openshift_serviceaccounts - openshift_serviceaccounts_names: - - router - - registry - openshift_serviceaccounts_namespace: default - openshift_serviceaccounts_sccs: - - privileged - when: not openshift.common.version_gte_3_2_or_1_2 - role: openshift_hosted - - role: openshift_metrics - when: openshift.hosted.metrics.deploy | bool - - role: cockpit-ui - when: openshift.common.deployment_subtype == 'registry' + - role: openshift_hosted_metrics + when: openshift_hosted_metrics_deploy | default(false) | bool + - role: openshift_hosted_logging + when: openshift_hosted_logging_deploy | default(false) | bool + openshift_hosted_logging_hostname: "{{ logging_hostname }}" + openshift_hosted_logging_ops_hostname: "{{ logging_ops_hostname }}" + openshift_hosted_logging_master_public_url: "{{ logging_master_public_url }}" + openshift_hosted_logging_elasticsearch_cluster_size: "{{ logging_elasticsearch_cluster_size }}" + openshift_hosted_logging_elasticsearch_pvc_dynamic: "{{ 'true' if openshift_hosted_logging_storage_kind | default(none) == 'dynamic' else '' }}" + openshift_hosted_logging_elasticsearch_pvc_size: "{{ openshift.hosted.logging.storage.volume.size if openshift_hosted_logging_storage_kind | default(none) in ['dynamic','nfs'] else '' }}" + openshift_hosted_logging_elasticsearch_pvc_prefix: "{{ 'logging-es' if openshift_hosted_logging_storage_kind | default(none) == 'dynamic' else '' }}" + openshift_hosted_logging_elasticsearch_ops_cluster_size: "{{ logging_elasticsearch_ops_cluster_size }}" + openshift_hosted_logging_elasticsearch_ops_pvc_dynamic: "{{ 'true' if openshift_hosted_logging_storage_kind | default(none) == 'dynamic' else '' }}" + openshift_hosted_logging_elasticsearch_ops_pvc_size: "{{ openshift.hosted.logging.storage.volume.size if openshift_hosted_logging_storage_kind | default(none) in ['dynamic','nfs' ] else '' }}" + openshift_hosted_logging_elasticsearch_ops_pvc_prefix: "{{ 'logging-es' if openshift_hosted_logging_storage_kind | default(none) =='dynamic' else '' }}" -- name: Configure CA certificate for secure registry - hosts: oo_nodes_to_config - tags: - - hosted - tasks: - - name: Create temp directory for kubeconfig - command: mktemp -d /tmp/openshift-ansible-XXXXXX - register: mktemp - when: openshift.common.deployment_subtype == 'registry' - changed_when: false - delegate_to: "{{ groups.oo_first_master.0 }}" - run_once: true - - set_fact: - openshift_hosted_kubeconfig: "{{ mktemp.stdout }}/admin.kubeconfig" - when: openshift.common.deployment_subtype == 'registry' - delegate_to: "{{ groups.oo_first_master.0 }}" - run_once: true - - name: Copy the admin client config(s) - command: > - cp {{ openshift.common.config_base }}/master/admin.kubeconfig {{ openshift_hosted_kubeconfig }} - when: openshift.common.deployment_subtype == 'registry' - changed_when: false - delegate_to: "{{ groups.oo_first_master.0 }}" - run_once: true - - name: Retrieve docker-registry route - command: > - {{ openshift.common.client_binary }} get route docker-registry - --template='{{ '{{' }} .spec.host {{ '}}' }}' - --config={{ openshift_hosted_kubeconfig }} - -n default - register: docker_registry_route - when: openshift.common.deployment_subtype == 'registry' - changed_when: false - delegate_to: "{{ groups.oo_first_master.0 }}" - run_once: true - - name: Retrieve registry service IP - command: > - {{ openshift.common.client_binary }} get service docker-registry - --template='{{ '{{' }} .spec.clusterIP {{ '}}' }}' - --config={{ openshift_hosted_kubeconfig }} - -n default - register: docker_registry_service_ip - when: openshift.common.deployment_subtype == 'registry' - changed_when: false - delegate_to: "{{ groups.oo_first_master.0 }}" - run_once: true - - name: Create registry CA directories - file: - path: "/etc/docker/certs.d/{{ item }}" - state: directory - with_items: - - "{{ docker_registry_service_ip.stdout }}:5000" - - "{{ docker_registry_route.stdout }}" - - "docker-registry.default.svc.cluster.local:5000" - when: openshift.common.deployment_subtype == 'registry' - - name: Copy CA to registry CA directories - copy: - src: "{{ openshift.common.config_base }}/node/ca.crt" - dest: "/etc/docker/certs.d/{{ item }}" - remote_src: yes - force: yes - with_items: - - "{{ docker_registry_service_ip.stdout }}:5000" - - "{{ docker_registry_route.stdout }}" - - "docker-registry.default.svc.cluster.local:5000" - when: openshift.common.deployment_subtype == 'registry' - notify: - - Restart docker - - name: Delete temp directory - file: - name: "{{ mktemp.stdout }}" - state: absent - when: openshift.common.deployment_subtype == 'registry' - changed_when: False - delegate_to: "{{ groups.oo_first_master.0 }}" - run_once: true - handlers: - - name: Restart docker - service: - name: docker - state: restarted + - role: cockpit-ui + when: ( openshift.common.version_gte_3_3_or_1_3 | bool ) and ( openshift_hosted_manage_registry | default(true) | bool ) and not (openshift.docker.hosted_registry_insecure | default(false) | bool) diff --git a/playbooks/common/openshift-cluster/openshift_metrics.yml b/playbooks/common/openshift-cluster/openshift_metrics.yml new file mode 100644 index 000000000..9f38ceea6 --- /dev/null +++ b/playbooks/common/openshift-cluster/openshift_metrics.yml @@ -0,0 +1,5 @@ +--- +- name: OpenShift Metrics + hosts: oo_first_master + roles: + - openshift_metrics diff --git a/playbooks/common/openshift-cluster/redeploy-certificates.yml b/playbooks/common/openshift-cluster/redeploy-certificates.yml index 4996c56a7..6e3e04a6b 100644 --- a/playbooks/common/openshift-cluster/redeploy-certificates.yml +++ b/playbooks/common/openshift-cluster/redeploy-certificates.yml @@ -204,7 +204,7 @@ cp {{ openshift.common.config_base }}/master//admin.kubeconfig {{ mktemp.stdout }}/admin.kubeconfig changed_when: False -- name: Serially evacuate all nodes to trigger redeployments +- name: Serially drain all nodes to trigger redeployments hosts: oo_nodes_to_config serial: 1 any_errors_fatal: true @@ -222,25 +222,25 @@ was_schedulable: "{{ 'unschedulable' not in (node_output.stdout | from_json).spec }}" when: openshift_certificates_redeploy_ca | default(false) | bool - - name: Prepare for node evacuation + - name: Prepare for node draining command: > - {{ openshift.common.admin_binary }} --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig + {{ openshift.common.client_binary }} adm --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig manage-node {{ openshift.node.nodename }} --schedulable=false delegate_to: "{{ groups.oo_first_master.0 }}" when: openshift_certificates_redeploy_ca | default(false) | bool and was_schedulable | bool - - name: Evacuate node + - name: Drain node command: > - {{ openshift.common.admin_binary }} --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig + {{ openshift.common.client_binary }} adm --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig manage-node {{ openshift.node.nodename }} - --evacuate --force + {{ openshift.common.evacuate_or_drain }} --force delegate_to: "{{ groups.oo_first_master.0 }}" when: openshift_certificates_redeploy_ca | default(false) | bool and was_schedulable | bool - name: Set node schedulability command: > - {{ openshift.common.admin_binary }} --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig + {{ openshift.common.client_binary }} adm --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig manage-node {{ openshift.node.nodename }} --schedulable=true delegate_to: "{{ groups.oo_first_master.0 }}" when: openshift_certificates_redeploy_ca | default(false) | bool and was_schedulable | bool diff --git a/playbooks/common/openshift-cluster/upgrades/cleanup_unused_images.yml b/playbooks/common/openshift-cluster/upgrades/cleanup_unused_images.yml new file mode 100644 index 000000000..6e953be69 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/cleanup_unused_images.yml @@ -0,0 +1,22 @@ +--- +- name: Check Docker image count + shell: "docker images -aq | wc -l" + register: docker_image_count + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- debug: var=docker_image_count.stdout + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- name: Remove unused Docker images for Docker 1.10+ migration + shell: "docker rmi `docker images -aq`" + # Will fail on images still in use: + failed_when: false + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- name: Check Docker image count + shell: "docker images -aq | wc -l" + register: docker_image_count + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- debug: var=docker_image_count.stdout + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool diff --git a/playbooks/common/openshift-cluster/upgrades/containerized_node_upgrade.yml b/playbooks/common/openshift-cluster/upgrades/containerized_node_upgrade.yml index 32a3636aa..9f7961614 100644 --- a/playbooks/common/openshift-cluster/upgrades/containerized_node_upgrade.yml +++ b/playbooks/common/openshift-cluster/upgrades/containerized_node_upgrade.yml @@ -1,11 +1,14 @@ -- include_vars: ../../../../roles/openshift_node/vars/main.yml +--- +# This is a hack to allow us to use systemd_units.yml, but skip the handlers which +# restart services. We will unconditionally restart all containerized services +# because we have to unconditionally restart Docker: +- set_fact: + skip_node_svc_handlers: True - name: Update systemd units include: ../../../../roles/openshift_node/tasks/systemd_units.yml openshift_version={{ openshift_image_tag }} -- name: Verifying the correct version was configured - shell: grep {{ verify_upgrade_version }} {{ item }} - with_items: - - /etc/sysconfig/openvswitch - - /etc/sysconfig/{{ openshift.common.service_type }}* - when: verify_upgrade_version is defined +# This is a no-op because of skip_node_svc_handlers, but lets us trigger it before end of +# play when the node has already been marked schedulable again. (this would look strange +# in logs otherwise) +- meta: flush_handlers diff --git a/playbooks/common/openshift-cluster/upgrades/create_service_signer_cert.yml b/playbooks/common/openshift-cluster/upgrades/create_service_signer_cert.yml index e8a20aa2b..23cf8cf76 100644 --- a/playbooks/common/openshift-cluster/upgrades/create_service_signer_cert.yml +++ b/playbooks/common/openshift-cluster/upgrades/create_service_signer_cert.yml @@ -9,6 +9,7 @@ local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX register: local_cert_sync_tmpdir changed_when: false + when: not (hostvars[groups.oo_first_master.0].service_signer_cert_stat.stat.exists | bool) - name: Create service signer certificate hosts: oo_first_master @@ -17,16 +18,18 @@ command: mktemp -d /tmp/openshift-ansible-XXXXXXX register: remote_cert_create_tmpdir changed_when: false + when: not (hostvars[groups.oo_first_master.0].service_signer_cert_stat.stat.exists | bool) - name: Create service signer certificate command: > - {{ openshift.common.admin_binary }} ca create-signer-cert - --cert=service-signer.crt - --key=service-signer.key - --name=openshift-service-serving-signer - --serial=service-signer.serial.txt + {{ openshift.common.client_binary }} adm ca create-signer-cert + --cert="{{ remote_cert_create_tmpdir.stdout }}/"service-signer.crt + --key="{{ remote_cert_create_tmpdir.stdout }}/"service-signer.key + --name="{{ remote_cert_create_tmpdir.stdout }}/"openshift-service-serving-signer + --serial="{{ remote_cert_create_tmpdir.stdout }}/"service-signer.serial.txt args: chdir: "{{ remote_cert_create_tmpdir.stdout }}/" + when: not (hostvars[groups.oo_first_master.0].service_signer_cert_stat.stat.exists | bool) - name: Retrieve service signer certificate fetch: @@ -38,12 +41,14 @@ with_items: - "service-signer.crt" - "service-signer.key" + when: not (hostvars[groups.oo_first_master.0].service_signer_cert_stat.stat.exists | bool) - name: Delete remote temp directory file: name: "{{ remote_cert_create_tmpdir.stdout }}" state: absent changed_when: false + when: not (hostvars[groups.oo_first_master.0].service_signer_cert_stat.stat.exists | bool) - name: Deploy service signer certificate hosts: oo_masters_to_config @@ -55,6 +60,7 @@ with_items: - "service-signer.crt" - "service-signer.key" + when: not (hostvars[groups.oo_first_master.0].service_signer_cert_stat.stat.exists | bool) - name: Delete local temp directory hosts: localhost @@ -67,3 +73,4 @@ name: "{{ local_cert_sync_tmpdir.stdout }}" state: absent changed_when: false + when: not (hostvars[groups.oo_first_master.0].service_signer_cert_stat.stat.exists | bool) diff --git a/playbooks/common/openshift-cluster/upgrades/docker/restart.yml b/playbooks/common/openshift-cluster/upgrades/docker/restart.yml new file mode 100644 index 000000000..1b418920f --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/docker/restart.yml @@ -0,0 +1,27 @@ +--- +- name: Restart docker + service: name=docker state=restarted + +- name: Update docker facts + openshift_facts: + role: docker + +- name: Restart containerized services + service: name={{ item }} state=started + with_items: + - etcd_container + - openvswitch + - "{{ openshift.common.service_type }}-master" + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + failed_when: false + when: openshift.common.is_containerized | bool + +- name: Wait for master API to come back online + wait_for: + host: "{{ openshift.common.hostname }}" + state: started + delay: 10 + port: "{{ openshift.master.api_port }}" + when: inventory_hostname in groups.oo_masters_to_config diff --git a/playbooks/common/openshift-cluster/upgrades/docker/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/docker/upgrade.yml index 417096dd0..17f8fc6e9 100644 --- a/playbooks/common/openshift-cluster/upgrades/docker/upgrade.yml +++ b/playbooks/common/openshift-cluster/upgrades/docker/upgrade.yml @@ -20,7 +20,7 @@ - debug: var=docker_image_count.stdout - name: Remove all containers and images - script: nuke_images.sh docker + script: nuke_images.sh register: nuke_images_result when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool @@ -35,32 +35,7 @@ - service: name=docker state=stopped - name: Upgrade Docker - action: "{{ ansible_pkg_mgr }} name=docker{{ '-' + docker_version }} state=present" + package: name=docker{{ '-' + docker_version }} state=present -- service: name=docker state=started - -- name: Update docker facts - openshift_facts: - role: docker - -- name: Restart containerized services - service: name={{ item }} state=started - with_items: - - etcd_container - - openvswitch - - "{{ openshift.common.service_type }}-master" - - "{{ openshift.common.service_type }}-master-api" - - "{{ openshift.common.service_type }}-master-controllers" - - "{{ openshift.common.service_type }}-node" - failed_when: false - when: openshift.common.is_containerized | bool - -- name: Wait for master API to come back online - become: no - local_action: - module: wait_for - host="{{ inventory_hostname }}" - state=started - delay=10 - port="{{ openshift.master.api_port }}" - when: inventory_hostname in groups.oo_masters_to_config +- include: restart.yml + when: not skip_docker_restart | default(False) | bool diff --git a/playbooks/common/openshift-cluster/upgrades/docker/upgrade_check.yml b/playbooks/common/openshift-cluster/upgrades/docker/upgrade_check.yml index 8002af4fc..b2a2eac9a 100644 --- a/playbooks/common/openshift-cluster/upgrades/docker/upgrade_check.yml +++ b/playbooks/common/openshift-cluster/upgrades/docker/upgrade_check.yml @@ -1,7 +1,7 @@ --- # This snippet determines if a Docker upgrade is required by checking the inventory -# variables, the available packages, and sets l_docker_version to True if so. +# variables, the available packages, and sets l_docker_upgrade to True if so. - set_fact: docker_upgrade: True @@ -9,6 +9,8 @@ - name: Check if Docker is installed command: rpm -q docker + args: + warn: no register: pkg_check failed_when: pkg_check.rc > 1 changed_when: no @@ -22,22 +24,24 @@ command: > {{ repoquery_cmd }} --qf '%{version}' "docker" register: avail_docker_version + # Don't expect docker rpm to be available on hosts that don't already have it installed: + when: pkg_check.rc == 0 failed_when: false changed_when: false - fail: - msg: This playbook requires access to Docker 1.10 or later - # Disable the 1.10 requirement if the user set a specific Docker version - when: docker_version is not defined and (docker_upgrade is not defined or docker_upgrade | bool == True) and (avail_docker_version.stdout == "" or avail_docker_version.stdout | version_compare('1.10','<')) + msg: This playbook requires access to Docker 1.12 or later + # Disable the 1.12 requirement if the user set a specific Docker version + when: docker_version is not defined and (docker_upgrade is not defined or docker_upgrade | bool == True) and (pkg_check.rc == 0 and (avail_docker_version.stdout == "" or avail_docker_version.stdout | version_compare('1.12','<'))) # Default l_docker_upgrade to False, we'll set to True if an upgrade is required: - set_fact: l_docker_upgrade: False -# Make sure a docker_verison is set if none was requested: +# Make sure a docker_version is set if none was requested: - set_fact: docker_version: "{{ avail_docker_version.stdout }}" - when: docker_version is not defined + when: pkg_check.rc == 0 and docker_version is not defined - name: Flag for Docker upgrade if necessary set_fact: @@ -46,6 +50,5 @@ - name: Flag to delete all images prior to upgrade if crossing Docker 1.10 boundary set_fact: - docker_upgrade_nuke_images: True + docker_upgrade_nuke_images: True when: l_docker_upgrade | bool and docker_upgrade_nuke_images is not defined and curr_docker_version.stdout | version_compare('1.10','<') and docker_version | version_compare('1.10','>=') - diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml new file mode 100644 index 000000000..d0eadf1fc --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml @@ -0,0 +1,94 @@ +--- +- name: Backup etcd + hosts: etcd_hosts_to_backup + vars: + embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" + timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" + etcdctl_command: "{{ 'etcdctl' if not openshift.common.is_containerized or embedded_etcd else 'docker exec etcd_container etcdctl' }}" + roles: + - openshift_facts + tasks: + # Ensure we persist the etcd role for this host in openshift_facts + - openshift_facts: + role: etcd + local_facts: {} + when: "'etcd' not in openshift" + + - stat: path=/var/lib/openshift + register: var_lib_openshift + + - stat: path=/var/lib/origin + register: var_lib_origin + + - name: Create origin symlink if necessary + file: src=/var/lib/openshift/ dest=/var/lib/origin state=link + when: var_lib_openshift.stat.exists == True and var_lib_origin.stat.exists == False + + # TODO: replace shell module with command and update later checks + # We assume to be using the data dir for all backups. + - name: Check available disk space for etcd backup + shell: df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1 + register: avail_disk + + # TODO: replace shell module with command and update later checks + - name: Check current embedded etcd disk usage + shell: du -k {{ openshift.etcd.etcd_data_dir }} | tail -n 1 | cut -f1 + register: etcd_disk_usage + when: embedded_etcd | bool + + - name: Abort if insufficient disk space for etcd backup + fail: + msg: > + {{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup, + {{ avail_disk.stdout }} Kb available. + when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int) + + # For non containerized and non embedded we should have the correct version of + # etcd installed already. So don't do anything. + # + # For embedded or containerized we need to use the latest because OCP 3.3 uses + # a version of etcd that can only be backed up with etcd-3.x and if it's + # containerized then etcd version may be newer than that on the host so + # upgrade it. + # + # On atomic we have neither yum nor dnf so ansible throws a hard to debug error + # if you use package there, like this: "Could not find a module for unknown." + # see https://bugzilla.redhat.com/show_bug.cgi?id=1408668 + # + # TODO - We should refactor all containerized backups to use the containerized + # version of etcd to perform the backup rather than relying on the host's + # binaries. Until we do that we'll continue to have problems backing up etcd + # when atomic host has an older version than the version that's running in the + # container whether that's embedded or not + - name: Install latest etcd for containerized or embedded + package: + name: etcd + state: latest + when: ( embedded_etcd | bool or openshift.common.is_containerized ) and not openshift.common.is_atomic + + - name: Generate etcd backup + command: > + {{ etcdctl_command }} backup --data-dir={{ openshift.etcd.etcd_data_dir }} + --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ backup_tag | default('') }}{{ timestamp }} + + - set_fact: + etcd_backup_complete: True + + - name: Display location of etcd backup + debug: + msg: "Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ backup_tag | default('') }}{{ timestamp }}" + +- name: Gate on etcd backup + hosts: localhost + connection: local + become: no + tasks: + - set_fact: + etcd_backup_completed: "{{ hostvars + | oo_select_keys(groups.etcd_hosts_to_backup) + | oo_collect('inventory_hostname', {'etcd_backup_complete': true}) }}" + - set_fact: + etcd_backup_failed: "{{ groups.etcd_hosts_to_backup | difference(etcd_backup_completed) }}" + - fail: + msg: "Upgrade cannot continue. The following hosts did not complete etcd backup: {{ etcd_backup_failed | join(',') }}" + when: etcd_backup_failed | length > 0 diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/containerized_tasks.yml b/playbooks/common/openshift-cluster/upgrades/etcd/containerized_tasks.yml new file mode 100644 index 000000000..5f8b59e17 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/containerized_tasks.yml @@ -0,0 +1,46 @@ +--- +- name: Verify cluster is healthy pre-upgrade + command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health" + +- name: Get current image + shell: grep 'ExecStart=' /etc/systemd/system/etcd_container.service | awk '{print $NF}' + register: current_image + +- name: Set new_etcd_image + set_fact: + new_etcd_image: "{{ current_image.stdout | regex_replace('/etcd.*$','/etcd:' ~ upgrade_version ) }}" + +- name: Pull new etcd image + command: "docker pull {{ new_etcd_image }}" + +- name: Update to latest etcd image + replace: + dest: /etc/systemd/system/etcd_container.service + regexp: "{{ current_image.stdout }}$" + replace: "{{ new_etcd_image }}" + +- name: Restart etcd_container + systemd: + name: etcd_container + daemon_reload: yes + state: restarted + +## TODO: probably should just move this into the backup playbooks, also this +## will fail on atomic host. We need to revisit how to do etcd backups there as +## the container may be newer than etcdctl on the host. Assumes etcd3 obsoletes etcd (7.3.1) +- name: Upgrade etcd for etcdctl when not atomic + package: name=etcd state=latest + when: not openshift.common.is_atomic | bool + +- name: Verify cluster is healthy + command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health" + register: etcdctl + until: etcdctl.rc == 0 + retries: 3 + delay: 10 + +- name: Store new etcd_image + openshift_facts: + role: etcd + local_facts: + etcd_image: "{{ new_etcd_image }}" diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/fedora_tasks.yml b/playbooks/common/openshift-cluster/upgrades/etcd/fedora_tasks.yml new file mode 100644 index 000000000..30232110e --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/fedora_tasks.yml @@ -0,0 +1,23 @@ +--- +# F23 GA'd with etcd 2.0, currently has 2.2 in updates +# F24 GA'd with etcd-2.2, currently has 2.2 in updates +# F25 Beta currently has etcd 3.0 +- name: Verify cluster is healthy pre-upgrade + command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health" + +- name: Update etcd + package: + name: "etcd" + state: "latest" + +- name: Restart etcd + service: + name: etcd + state: restarted + +- name: Verify cluster is healthy + command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health" + register: etcdctl + until: etcdctl.rc == 0 + retries: 3 + delay: 10 diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/files/etcdctl.sh b/playbooks/common/openshift-cluster/upgrades/etcd/files/etcdctl.sh new file mode 120000 index 000000000..641e04e44 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/files/etcdctl.sh @@ -0,0 +1 @@ +../roles/etcd/files/etcdctl.sh
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/filter_plugins b/playbooks/common/openshift-cluster/upgrades/etcd/filter_plugins new file mode 120000 index 000000000..27ddaa18b --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/filter_plugins @@ -0,0 +1 @@ +../../../../../filter_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/lookup_plugins b/playbooks/common/openshift-cluster/upgrades/etcd/lookup_plugins new file mode 120000 index 000000000..cf407f69b --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/lookup_plugins @@ -0,0 +1 @@ +../../../../../lookup_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/main.yml b/playbooks/common/openshift-cluster/upgrades/etcd/main.yml new file mode 100644 index 000000000..8268adc2e --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/main.yml @@ -0,0 +1,44 @@ +--- +# For 1.4/3.4 we want to upgrade everyone to etcd-3.0. etcd docs say to +# upgrade from 2.0.x to 2.1.x to 2.2.x to 2.3.x to 3.0.x. While this is a tedius +# task for RHEL and CENTOS it's simply not possible in Fedora unless you've +# mirrored packages on your own because only the GA and latest versions are +# available in the repos. So for Fedora we'll simply skip this, sorry. + +- include: ../../evaluate_groups.yml + tags: + - always + +# We use two groups one for hosts we're upgrading which doesn't include embedded etcd +# The other for backing up which includes the embedded etcd host, there's no need to +# upgrade embedded etcd that just happens when the master is updated. +- name: Evaluate additional groups for etcd + hosts: localhost + connection: local + become: no + tasks: + - name: Evaluate etcd_hosts_to_upgrade + add_host: + name: "{{ item }}" + groups: etcd_hosts_to_upgrade + with_items: "{{ groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else [] }}" + - name: Evaluate etcd_hosts_to_backup + add_host: + name: "{{ item }}" + groups: etcd_hosts_to_backup + with_items: "{{ groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else groups.oo_first_master }}" + +- name: Backup etcd before upgrading anything + include: backup.yml + vars: + backup_tag: "pre-upgrade-" + when: openshift_etcd_backup | default(true) | bool + +- name: Drop etcdctl profiles + hosts: etcd_hosts_to_upgrade + tasks: + - include: roles/etcd/tasks/etcdctl.yml + +- name: Perform etcd upgrade + include: ./upgrade.yml + when: openshift_etcd_upgrade | default(true) | bool diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/rhel_tasks.yml b/playbooks/common/openshift-cluster/upgrades/etcd/rhel_tasks.yml new file mode 100644 index 000000000..3a972e8ab --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/rhel_tasks.yml @@ -0,0 +1,20 @@ +--- +- name: Verify cluster is healthy pre-upgrade + command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health" + +- name: Update etcd RPM + package: + name: etcd-{{ upgrade_version }}* + state: latest + +- name: Restart etcd + service: + name: etcd + state: restarted + +- name: Verify cluster is healthy + command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health" + register: etcdctl + until: etcdctl.rc == 0 + retries: 3 + delay: 10 diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/roles b/playbooks/common/openshift-cluster/upgrades/etcd/roles new file mode 120000 index 000000000..6bc1a7aef --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/roles @@ -0,0 +1 @@ +../../../../../roles
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml new file mode 100644 index 000000000..0f8d94737 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml @@ -0,0 +1,94 @@ +--- +- name: Determine etcd version + hosts: etcd_hosts_to_upgrade + tasks: + - name: Record RPM based etcd version + command: rpm -qa --qf '%{version}' etcd\* + args: + warn: no + register: etcd_rpm_version + failed_when: false + when: not openshift.common.is_containerized | bool + - name: Record containerized etcd version + command: docker exec etcd_container rpm -qa --qf '%{version}' etcd\* + register: etcd_container_version + failed_when: false + when: openshift.common.is_containerized | bool + +# I really dislike this copy/pasta but I wasn't able to find a way to get it to loop +# through hosts, then loop through tasks only when appropriate +- name: Upgrade to 2.1 + hosts: etcd_hosts_to_upgrade + serial: 1 + vars: + upgrade_version: '2.1' + tasks: + - include: rhel_tasks.yml + when: etcd_rpm_version.stdout | default('99') | version_compare('2.1','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool + +- name: Upgrade RPM hosts to 2.2 + hosts: etcd_hosts_to_upgrade + serial: 1 + vars: + upgrade_version: '2.2' + tasks: + - include: rhel_tasks.yml + when: etcd_rpm_version.stdout | default('99') | version_compare('2.2','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool + +- name: Upgrade containerized hosts to 2.2.5 + hosts: etcd_hosts_to_upgrade + serial: 1 + vars: + upgrade_version: 2.2.5 + tasks: + - include: containerized_tasks.yml + when: etcd_container_version.stdout | default('99') | version_compare('2.2','<') and openshift.common.is_containerized | bool + +- name: Upgrade RPM hosts to 2.3 + hosts: etcd_hosts_to_upgrade + serial: 1 + vars: + upgrade_version: '2.3' + tasks: + - include: rhel_tasks.yml + when: etcd_rpm_version.stdout | default('99') | version_compare('2.3','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool + +- name: Upgrade containerized hosts to 2.3.7 + hosts: etcd_hosts_to_upgrade + serial: 1 + vars: + upgrade_version: 2.3.7 + tasks: + - include: containerized_tasks.yml + when: etcd_container_version.stdout | default('99') | version_compare('2.3','<') and openshift.common.is_containerized | bool + +- name: Upgrade RPM hosts to 3.0 + hosts: etcd_hosts_to_upgrade + serial: 1 + vars: + upgrade_version: '3.0' + tasks: + - include: rhel_tasks.yml + when: etcd_rpm_version.stdout | default('99') | version_compare('3.0','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool + +- name: Upgrade containerized hosts to etcd3 image + hosts: etcd_hosts_to_upgrade + serial: 1 + vars: + upgrade_version: 3.0.15 + tasks: + - include: containerized_tasks.yml + when: etcd_container_version.stdout | default('99') | version_compare('3.0','<') and openshift.common.is_containerized | bool + +- name: Upgrade fedora to latest + hosts: etcd_hosts_to_upgrade + serial: 1 + tasks: + - include: fedora_tasks.yml + when: ansible_distribution == 'Fedora' and not openshift.common.is_containerized | bool + +- name: Backup etcd + include: backup.yml + vars: + backup_tag: "post-3.0-" + when: openshift_etcd_backup | default(true) | bool diff --git a/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check b/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check deleted file mode 100644 index e5c958ebb..000000000 --- a/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python -""" -Pre-upgrade checks that must be run on a master before proceeding with upgrade. -""" -# This is a script not a python module: -# pylint: disable=invalid-name - -# NOTE: This script should not require any python libs other than what is -# in the standard library. - -__license__ = "ASL 2.0" - -import json -import os -import subprocess -import re - -# The maximum length of container.ports.name -ALLOWED_LENGTH = 15 -# The valid structure of container.ports.name -ALLOWED_CHARS = re.compile('^[a-z0-9][a-z0-9\\-]*[a-z0-9]$') -AT_LEAST_ONE_LETTER = re.compile('[a-z]') -# look at OS_PATH for the full path. Default ot 'oc' -OC_PATH = os.getenv('OC_PATH', 'oc') - - -def validate(value): - """ - validate verifies that value matches required conventions - - Rules of container.ports.name validation: - - * must be less that 16 chars - * at least one letter - * only a-z0-9- - * hyphens can not be leading or trailing or next to each other - - :Parameters: - - `value`: Value to validate - """ - if len(value) > ALLOWED_LENGTH: - return False - - if '--' in value: - return False - - # We search since it can be anywhere - if not AT_LEAST_ONE_LETTER.search(value): - return False - - # We match because it must start at the beginning - if not ALLOWED_CHARS.match(value): - return False - return True - - -def list_items(kind): - """ - list_items returns a list of items from the api - - :Parameters: - - `kind`: Kind of item to access - """ - response = subprocess.check_output([OC_PATH, 'get', '--all-namespaces', '-o', 'json', kind]) - items = json.loads(response) - return items.get("items", []) - - -def get(obj, *paths): - """ - Gets an object - - :Parameters: - - `obj`: A dictionary structure - - `path`: All other non-keyword arguments - """ - ret_obj = obj - for path in paths: - if ret_obj.get(path, None) is None: - return [] - ret_obj = ret_obj[path] - return ret_obj - - -# pylint: disable=too-many-arguments -def pretty_print_errors(namespace, kind, item_name, container_name, invalid_label, port_name, valid): - """ - Prints out results in human friendly way. - - :Parameters: - - `namespace`: Namespace of the resource - - `kind`: Kind of the resource - - `item_name`: Name of the resource - - `container_name`: Name of the container. May be "" when kind=Service. - - `port_name`: Name of the port - - `invalid_label`: The label of the invalid port. Port.name/targetPort - - `valid`: True if the port is valid - """ - if not valid: - if len(container_name) > 0: - print('%s/%s -n %s (Container="%s" %s="%s")' % ( - kind, item_name, namespace, container_name, invalid_label, port_name)) - else: - print('%s/%s -n %s (%s="%s")' % ( - kind, item_name, namespace, invalid_label, port_name)) - - -def print_validation_header(): - """ - Prints the error header. Should run on the first error to avoid - overwhelming the user. - """ - print """\ -At least one port name is invalid and must be corrected before upgrading. -Please update or remove any resources with invalid port names. - - Valid port names must: - - * be less that 16 characters - * have at least one letter - * contain only a-z0-9- - * not start or end with - - * not contain dashes next to each other ('--') -""" - - -def main(): - """ - main is the main entry point to this script - """ - try: - # the comma at the end suppresses the newline - print "Checking for oc ...", - subprocess.check_output([OC_PATH, 'whoami']) - print "found" - except: - print( - 'Unable to run "%s whoami"\n' - 'Please ensure OpenShift is running, and "oc" is on your system ' - 'path.\n' - 'You can override the path with the OC_PATH environment variable.' - % OC_PATH) - raise SystemExit(1) - - # Where the magic happens - first_error = True - for kind, path in [ - ('deploymentconfigs', ("spec", "template", "spec", "containers")), - ('replicationcontrollers', ("spec", "template", "spec", "containers")), - ('pods', ("spec", "containers"))]: - for item in list_items(kind): - namespace = item["metadata"]["namespace"] - item_name = item["metadata"]["name"] - for container in get(item, *path): - container_name = container["name"] - for port in get(container, "ports"): - port_name = port.get("name", None) - if not port_name: - # Unnamed ports are OK - continue - valid = validate(port_name) - if not valid and first_error: - first_error = False - print_validation_header() - pretty_print_errors( - namespace, kind, item_name, - container_name, "Port.name", port_name, valid) - - # Services follow a different flow - for item in list_items('services'): - namespace = item["metadata"]["namespace"] - item_name = item["metadata"]["name"] - for port in get(item, "spec", "ports"): - port_name = port.get("targetPort", None) - if isinstance(port_name, int) or port_name is None: - # Integer only or unnamed ports are OK - continue - valid = validate(port_name) - if not valid and first_error: - first_error = False - print_validation_header() - pretty_print_errors( - namespace, "services", item_name, "", - "targetPort", port_name, valid) - - # If we had at least 1 error then exit with 1 - if not first_error: - raise SystemExit(1) - - -if __name__ == '__main__': - main() - diff --git a/playbooks/common/openshift-cluster/upgrades/files/rpm_versions.sh b/playbooks/common/openshift-cluster/upgrades/files/rpm_versions.sh deleted file mode 100644 index 7bf249742..000000000 --- a/playbooks/common/openshift-cluster/upgrades/files/rpm_versions.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -if [ `which dnf 2> /dev/null` ]; then - installed=$(dnf repoquery --installed --latest-limit 1 -d 0 --qf '%{version}-%{release}' "${@}" 2> /dev/null) - available=$(dnf repoquery --available --latest-limit 1 -d 0 --qf '%{version}-%{release}' "${@}" 2> /dev/null) -else - installed=$(repoquery --plugins --pkgnarrow=installed --qf '%{version}-%{release}' "${@}" 2> /dev/null) - available=$(repoquery --plugins --pkgnarrow=available --qf '%{version}-%{release}' "${@}" 2> /dev/null) -fi - -echo "---" -echo "curr_version: ${installed}" -echo "avail_version: ${available}" diff --git a/playbooks/common/openshift-cluster/upgrades/init.yml b/playbooks/common/openshift-cluster/upgrades/init.yml new file mode 100644 index 000000000..8cac2fb3b --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/init.yml @@ -0,0 +1,48 @@ +--- +- hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - include_vars: ../../../byo/openshift-cluster/cluster_hosts.yml + - add_host: + name: "{{ item }}" + groups: l_oo_all_hosts + with_items: "{{ g_all_hosts | default([]) }}" + +- hosts: l_oo_all_hosts + gather_facts: no + tasks: + - include_vars: ../../../byo/openshift-cluster/cluster_hosts.yml + +- include: ../evaluate_groups.yml + vars: + # Do not allow adding hosts during upgrade. + g_new_master_hosts: [] + g_new_node_hosts: [] + openshift_cluster_id: "{{ cluster_id | default('default') }}" + openshift_deployment_type: "{{ deployment_type }}" + +- name: Set oo_options + hosts: oo_all_hosts + tasks: + - set_fact: + openshift_docker_additional_registries: "{{ lookup('oo_option', 'docker_additional_registries') }}" + when: openshift_docker_additional_registries is not defined + - set_fact: + openshift_docker_insecure_registries: "{{ lookup('oo_option', 'docker_insecure_registries') }}" + when: openshift_docker_insecure_registries is not defined + - set_fact: + openshift_docker_blocked_registries: "{{ lookup('oo_option', 'docker_blocked_registries') }}" + when: openshift_docker_blocked_registries is not defined + - set_fact: + openshift_docker_options: "{{ lookup('oo_option', 'docker_options') }}" + when: openshift_docker_options is not defined + - set_fact: + openshift_docker_log_driver: "{{ lookup('oo_option', 'docker_log_driver') }}" + when: openshift_docker_log_driver is not defined + - set_fact: + openshift_docker_log_options: "{{ lookup('oo_option', 'docker_log_options') }}" + when: openshift_docker_log_options is not defined + +- include: ../initialize_facts.yml diff --git a/playbooks/common/openshift-cluster/upgrades/initialize_nodes_to_upgrade.yml b/playbooks/common/openshift-cluster/upgrades/initialize_nodes_to_upgrade.yml new file mode 100644 index 000000000..4e375ac26 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/initialize_nodes_to_upgrade.yml @@ -0,0 +1,40 @@ +--- +- name: Filter list of nodes to be upgraded if necessary + hosts: oo_first_master + tasks: + - name: Retrieve list of openshift nodes matching upgrade label + command: > + {{ openshift.common.client_binary }} + get nodes + --config={{ openshift.common.config_base }}/master/admin.kubeconfig + --selector={{ openshift_upgrade_nodes_label }} + -o jsonpath='{.items[*].metadata.name}' + register: matching_nodes + changed_when: false + when: openshift_upgrade_nodes_label is defined + + - set_fact: + nodes_to_upgrade: "{{ matching_nodes.stdout.split(' ') }}" + when: openshift_upgrade_nodes_label is defined + + # We got a list of nodes with the label, now we need to match these with inventory hosts + # using their openshift.common.hostname fact. + - name: Map labelled nodes to inventory hosts + add_host: + name: "{{ item }}" + groups: temp_nodes_to_upgrade + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_become: "{{ g_sudo | default(omit) }}" + with_items: " {{ groups['oo_nodes_to_config'] }}" + when: openshift_upgrade_nodes_label is defined and hostvars[item].openshift.common.hostname in nodes_to_upgrade + changed_when: false + + # Build up the oo_nodes_to_upgrade group, use the list filtered by label if + # present, otherwise hit all nodes: + - name: Evaluate oo_nodes_to_upgrade + add_host: + name: "{{ item }}" + groups: oo_nodes_to_upgrade + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_become: "{{ g_sudo | default(omit) }}" + with_items: "{{ groups['temp_nodes_to_upgrade'] | default(groups['oo_nodes_to_config']) }}" diff --git a/playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py b/playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py index 9a065fd1c..673f11889 100755 --- a/playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py +++ b/playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py @@ -17,6 +17,7 @@ requirements: [ ] EXAMPLES = ''' ''' + def modify_api_levels(level_list, remove, ensure, msg_prepend='', msg_append=''): """ modify_api_levels """ @@ -62,7 +63,6 @@ def upgrade_master_3_0_to_3_1(ansible_module, config_base, backup): config = yaml.safe_load(master_cfg_file.read()) master_cfg_file.close() - # Remove unsupported api versions and ensure supported api versions from # master config unsupported_levels = ['v1beta1', 'v1beta2', 'v1beta3'] @@ -118,7 +118,7 @@ def main(): # redefined-outer-name global module - module = AnsibleModule( + module = AnsibleModule( # noqa: F405 argument_spec=dict( config_base=dict(required=True), from_version=dict(required=True, choices=['3.0']), @@ -146,13 +146,14 @@ def main(): # ignore broad-except error to avoid stack trace to ansible user # pylint: disable=broad-except - except Exception, e: + except Exception as e: return module.fail_json(msg=str(e)) + # ignore pylint errors related to the module_utils import -# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import +# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import, wrong-import-position # import module snippets -from ansible.module_utils.basic import * +from ansible.module_utils.basic import * # noqa: E402,F403 if __name__ == '__main__': main() diff --git a/playbooks/common/openshift-cluster/upgrades/openvswitch-avoid-oom.conf b/playbooks/common/openshift-cluster/upgrades/openvswitch-avoid-oom.conf new file mode 120000 index 000000000..514526fe2 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/openvswitch-avoid-oom.conf @@ -0,0 +1 @@ +../../../../roles/openshift_node/templates/openvswitch-avoid-oom.conf
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/post.yml b/playbooks/common/openshift-cluster/upgrades/post_control_plane.yml index e43954453..2bbcbe1f8 100644 --- a/playbooks/common/openshift-cluster/upgrades/post.yml +++ b/playbooks/common/openshift-cluster/upgrades/post_control_plane.yml @@ -17,10 +17,14 @@ # not already exist. We could have potentially done a replace --force to # create and update in one step. - openshift_examples + - openshift_hosted_templates # Update the existing templates - role: openshift_examples registry_url: "{{ openshift.master.registry_url }}" openshift_examples_import_command: replace + - role: openshift_hosted_templates + registry_url: "{{ openshift.master.registry_url }}" + openshift_hosted_templates_import_command: replace pre_tasks: - name: Collect all routers command: > @@ -41,7 +45,7 @@ {{ oc_cmd }} patch dc/{{ item['labels']['deploymentconfig'] }} -n {{ item['namespace'] }} -p '{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}","livenessProbe":{"tcpSocket":null,"httpGet":{"path": "/healthz", "port": 1936, "host": "localhost", "scheme": "HTTP"},"initialDelaySeconds":10,"timeoutSeconds":1}}]}}}}' --api-version=v1 - with_items: haproxy_routers + with_items: "{{ haproxy_routers }}" - name: Check for default registry command: > diff --git a/playbooks/common/openshift-cluster/upgrades/pre.yml b/playbooks/common/openshift-cluster/upgrades/pre.yml deleted file mode 100644 index 42a24eaf8..000000000 --- a/playbooks/common/openshift-cluster/upgrades/pre.yml +++ /dev/null @@ -1,311 +0,0 @@ ---- -############################################################################### -# Evaluate host groups and gather facts -############################################################################### - -- include: ../initialize_facts.yml - -- name: Update repos and initialize facts on all hosts - hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config:oo_lb_to_config - roles: - - openshift_repos - -- name: Set openshift_no_proxy_internal_hostnames - hosts: oo_masters_to_config:oo_nodes_to_config - tasks: - - set_fact: - openshift_no_proxy_internal_hostnames: "{{ hostvars | oo_select_keys(groups['oo_nodes_to_config'] - | union(groups['oo_masters_to_config']) - | union(groups['oo_etcd_to_config'] | default([]))) - | oo_collect('openshift.common.hostname') | default([]) | join (',') - }}" - when: "{{ (openshift_http_proxy is defined or openshift_https_proxy is defined) and - openshift_generate_no_proxy_hosts | default(True) | bool }}" - -- name: Evaluate additional groups for upgrade - hosts: localhost - connection: local - become: no - tasks: - - name: Evaluate etcd_hosts_to_backup - add_host: - name: "{{ item }}" - groups: etcd_hosts_to_backup - with_items: groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else groups.oo_first_master - -############################################################################### -# Pre-upgrade checks -############################################################################### -- name: Verify upgrade can proceed on first master - hosts: oo_first_master - vars: - g_pacemaker_upgrade_url_segment: "{{ 'org/latest' if deployment_type =='origin' else '.com/enterprise/3.1' }}" - gather_facts: no - tasks: - - fail: - msg: > - This upgrade is only supported for atomic-enterprise, origin, openshift-enterprise, and online - deployment types - when: deployment_type not in ['atomic-enterprise', 'origin','openshift-enterprise', 'online'] - - - fail: - msg: > - This upgrade does not support Pacemaker: - https://docs.openshift.{{ g_pacemaker_upgrade_url_segment }}/install_config/upgrading/pacemaker_to_native_ha.html - when: openshift.master.cluster_method is defined and openshift.master.cluster_method == 'pacemaker' - - # Error out in situations where the user has older versions specified in their - # inventory in any of the openshift_release, openshift_image_tag, and - # openshift_pkg_version variables. These must be removed or updated to proceed - # with upgrade. - # TODO: Should we block if you're *over* the next major release version as well? - - fail: - msg: > - openshift_pkg_version is {{ openshift_pkg_version }} which is not a - valid version for a {{ openshift_upgrade_target }} upgrade - when: openshift_pkg_version is defined and openshift_pkg_version.split('-',1).1 | version_compare(openshift_upgrade_target ,'<') - - - fail: - msg: > - openshift_image_tag is {{ openshift_image_tag }} which is not a - valid version for a {{ openshift_upgrade_target }} upgrade - when: openshift_image_tag is defined and openshift_image_tag.split('v',1).1 | version_compare(openshift_upgrade_target ,'<') - - - set_fact: - openshift_release: "{{ openshift_release[1:] }}" - when: openshift_release is defined and openshift_release[0] == 'v' - - - fail: - msg: > - openshift_release is {{ openshift_release }} which is not a - valid release for a {{ openshift_upgrade_target }} upgrade - when: openshift_release is defined and not openshift_release | version_compare(openshift_upgrade_target ,'=') - -- include: ../../../common/openshift-cluster/initialize_openshift_version.yml - vars: - # Request specific openshift_release and let the openshift_version role handle converting this - # to a more specific version, respecting openshift_image_tag and openshift_pkg_version if - # defined, and overriding the normal behavior of protecting the installed version - openshift_release: "{{ openshift_upgrade_target }}" - openshift_protect_installed_version: False - # Docker role (a dependency) should be told not to do anything to installed version - # of docker, we handle this separately during upgrade. (the inventory may have a - # docker_version defined, we don't want to actually do it until later) - docker_protect_installed_version: True - -- name: Verify master processes - hosts: oo_masters_to_config - roles: - - openshift_facts - tasks: - - openshift_facts: - role: master - local_facts: - ha: "{{ groups.oo_masters_to_config | length > 1 }}" - - - name: Ensure Master is running - service: - name: "{{ openshift.common.service_type }}-master" - state: started - enabled: yes - when: openshift.master.ha is defined and not openshift.master.ha | bool and openshift.common.is_containerized | bool - - - name: Ensure HA Master is running - service: - name: "{{ openshift.common.service_type }}-master-api" - state: started - enabled: yes - when: openshift.master.ha is defined and openshift.master.ha | bool and openshift.common.is_containerized | bool - - - name: Ensure HA Master is running - service: - name: "{{ openshift.common.service_type }}-master-controllers" - state: started - enabled: yes - when: openshift.master.ha is defined and openshift.master.ha | bool and openshift.common.is_containerized | bool - -- name: Verify node processes - hosts: oo_nodes_to_config - roles: - - openshift_facts - - openshift_docker_facts - tasks: - - name: Ensure Node is running - service: - name: "{{ openshift.common.service_type }}-node" - state: started - enabled: yes - when: openshift.common.is_containerized | bool - -- name: Verify upgrade targets - hosts: oo_masters_to_config:oo_nodes_to_config - vars: - openshift_docker_hosted_registry_network: "{{ hostvars[groups.oo_first_master.0].openshift.common.portal_net }}" - pre_tasks: - - fail: - msg: Verify OpenShift is already installed - when: openshift.common.version is not defined - - - fail: - msg: Verify the correct version was found - when: verify_upgrade_version is defined and openshift_version != verify_upgrade_version - - - name: Clean package cache - command: "{{ ansible_pkg_mgr }} clean all" - when: not openshift.common.is_atomic | bool - - - set_fact: - g_new_service_name: "{{ 'origin' if deployment_type =='origin' else 'atomic-openshift' }}" - when: not openshift.common.is_containerized | bool - - - name: Verify containers are available for upgrade - command: > - docker pull {{ openshift.common.cli_image }}:{{ openshift_image_tag }} - register: pull_result - changed_when: "'Downloaded newer image' in pull_result.stdout" - when: openshift.common.is_containerized | bool - - - name: Check latest available OpenShift RPM version - command: > - {{ repoquery_cmd }} --qf '%{version}' "{{ openshift.common.service_type }}" - failed_when: false - changed_when: false - register: avail_openshift_version - when: not openshift.common.is_containerized | bool - - - name: Verify OpenShift RPMs are available for upgrade - fail: - msg: "OpenShift {{ avail_openshift_version.stdout }} is available, but {{ openshift_upgrade_target }} or greater is required" - when: not openshift.common.is_containerized | bool and not avail_openshift_version | skipped and avail_openshift_version.stdout | default('0.0', True) | version_compare(openshift_release, '<') - - - fail: - msg: "This upgrade playbook must be run against OpenShift {{ openshift_upgrade_min }} or later" - when: deployment_type == 'origin' and openshift.common.version | version_compare(openshift_upgrade_min,'<') - -- name: Verify docker upgrade targets - hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config - tasks: - # Only check if docker upgrade is required if docker_upgrade is not - # already set to False. - - include: docker/upgrade_check.yml - when: docker_upgrade is not defined or docker_upgrade | bool and not openshift.common.is_atomic | bool - - # Additional checks for Atomic hosts: - - - name: Determine available Docker - shell: "rpm -q --queryformat '---\ncurr_version: %{VERSION}\navail_version: \n' docker" - register: g_atomic_docker_version_result - when: openshift.common.is_atomic | bool - - - set_fact: - l_docker_version: "{{ g_atomic_docker_version_result.stdout | from_yaml }}" - when: openshift.common.is_atomic | bool - - - fail: - msg: This playbook requires access to Docker 1.10 or later - when: openshift.common.is_atomic | bool and l_docker_version.avail_version | default(l_docker_version.curr_version, true) | version_compare('1.10','<') - - - set_fact: - pre_upgrade_complete: True - - -############################################################################## -# Gate on pre-upgrade checks -############################################################################## -- name: Gate on pre-upgrade checks - hosts: localhost - connection: local - become: no - vars: - pre_upgrade_hosts: "{{ groups.oo_masters_to_config | union(groups.oo_nodes_to_config) }}" - tasks: - - set_fact: - pre_upgrade_completed: "{{ hostvars - | oo_select_keys(pre_upgrade_hosts) - | oo_collect('inventory_hostname', {'pre_upgrade_complete': true}) }}" - - set_fact: - pre_upgrade_failed: "{{ pre_upgrade_hosts | difference(pre_upgrade_completed) }}" - - fail: - msg: "Upgrade cannot continue. The following hosts did not complete pre-upgrade checks: {{ pre_upgrade_failed | join(',') }}" - when: pre_upgrade_failed | length > 0 - -############################################################################### -# Backup etcd -############################################################################### -- name: Backup etcd - hosts: etcd_hosts_to_backup - vars: - embedded_etcd: "{{ hostvars[groups.oo_first_master.0].openshift.master.embedded_etcd }}" - timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" - roles: - - openshift_facts - tasks: - # Ensure we persist the etcd role for this host in openshift_facts - - openshift_facts: - role: etcd - local_facts: {} - when: "'etcd' not in openshift" - - - stat: path=/var/lib/openshift - register: var_lib_openshift - - - stat: path=/var/lib/origin - register: var_lib_origin - - - name: Create origin symlink if necessary - file: src=/var/lib/openshift/ dest=/var/lib/origin state=link - when: var_lib_openshift.stat.exists == True and var_lib_origin.stat.exists == False - - # TODO: replace shell module with command and update later checks - # We assume to be using the data dir for all backups. - - name: Check available disk space for etcd backup - shell: df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1 - register: avail_disk - - # TODO: replace shell module with command and update later checks - - name: Check current embedded etcd disk usage - shell: du -k {{ openshift.etcd.etcd_data_dir }} | tail -n 1 | cut -f1 - register: etcd_disk_usage - when: embedded_etcd | bool - - - name: Abort if insufficient disk space for etcd backup - fail: - msg: > - {{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup, - {{ avail_disk.stdout }} Kb available. - when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int) - - - name: Install etcd (for etcdctl) - action: "{{ ansible_pkg_mgr }} name=etcd state=latest" - when: not openshift.common.is_atomic | bool - - - name: Generate etcd backup - command: > - etcdctl backup --data-dir={{ openshift.etcd.etcd_data_dir }} - --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }} - - - set_fact: - etcd_backup_complete: True - - - name: Display location of etcd backup - debug: - msg: "Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}" - - -############################################################################## -# Gate on etcd backup -############################################################################## -- name: Gate on etcd backup - hosts: localhost - connection: local - become: no - tasks: - - set_fact: - etcd_backup_completed: "{{ hostvars - | oo_select_keys(groups.etcd_hosts_to_backup) - | oo_collect('inventory_hostname', {'etcd_backup_complete': true}) }}" - - set_fact: - etcd_backup_failed: "{{ groups.etcd_hosts_to_backup | difference(etcd_backup_completed) }}" - - fail: - msg: "Upgrade cannot continue. The following hosts did not complete etcd backup: {{ etcd_backup_failed | join(',') }}" - when: etcd_backup_failed | length > 0 diff --git a/playbooks/common/openshift-cluster/upgrades/pre/gate_checks.yml b/playbooks/common/openshift-cluster/upgrades/pre/gate_checks.yml new file mode 100644 index 000000000..8ecae4539 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/pre/gate_checks.yml @@ -0,0 +1,6 @@ +--- +- name: Flag pre-upgrade checks complete for hosts without errors + hosts: oo_masters_to_config:oo_nodes_to_upgrade:oo_etcd_to_config + tasks: + - set_fact: + pre_upgrade_complete: True diff --git a/playbooks/common/openshift-cluster/upgrades/pre/roles b/playbooks/common/openshift-cluster/upgrades/pre/roles new file mode 120000 index 000000000..415645be6 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/pre/roles @@ -0,0 +1 @@ +../../../../../roles/
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/pre/verify_control_plane_running.yml b/playbooks/common/openshift-cluster/upgrades/pre/verify_control_plane_running.yml new file mode 100644 index 000000000..06eb5f936 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/pre/verify_control_plane_running.yml @@ -0,0 +1,31 @@ +--- +- name: Verify master processes + hosts: oo_masters_to_config + roles: + - openshift_facts + tasks: + - openshift_facts: + role: master + local_facts: + ha: "{{ groups.oo_masters_to_config | length > 1 }}" + + - name: Ensure Master is running + service: + name: "{{ openshift.common.service_type }}-master" + state: started + enabled: yes + when: openshift.master.ha is defined and not openshift.master.ha | bool and openshift.common.is_containerized | bool + + - name: Ensure HA Master is running + service: + name: "{{ openshift.common.service_type }}-master-api" + state: started + enabled: yes + when: openshift.master.ha is defined and openshift.master.ha | bool and openshift.common.is_containerized | bool + + - name: Ensure HA Master is running + service: + name: "{{ openshift.common.service_type }}-master-controllers" + state: started + enabled: yes + when: openshift.master.ha is defined and openshift.master.ha | bool and openshift.common.is_containerized | bool diff --git a/playbooks/common/openshift-cluster/upgrades/pre/verify_docker_upgrade_targets.yml b/playbooks/common/openshift-cluster/upgrades/pre/verify_docker_upgrade_targets.yml new file mode 100644 index 000000000..7646e0fa6 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/pre/verify_docker_upgrade_targets.yml @@ -0,0 +1,23 @@ +--- +- name: Verify docker upgrade targets + hosts: oo_masters_to_config:oo_nodes_to_upgrade:oo_etcd_to_config + tasks: + # Only check if docker upgrade is required if docker_upgrade is not + # already set to False. + - include: ../docker/upgrade_check.yml + when: docker_upgrade is not defined or docker_upgrade | bool and not openshift.common.is_atomic | bool + + # Additional checks for Atomic hosts: + + - name: Determine available Docker + shell: "rpm -q --queryformat '---\ncurr_version: %{VERSION}\navail_version: \n' docker" + register: g_atomic_docker_version_result + when: openshift.common.is_atomic | bool + + - set_fact: + l_docker_version: "{{ g_atomic_docker_version_result.stdout | from_yaml }}" + when: openshift.common.is_atomic | bool + + - fail: + msg: This playbook requires access to Docker 1.12 or later + when: openshift.common.is_atomic | bool and l_docker_version.avail_version | default(l_docker_version.curr_version, true) | version_compare('1.12','<') diff --git a/playbooks/common/openshift-cluster/upgrades/pre/verify_inventory_vars.yml b/playbooks/common/openshift-cluster/upgrades/pre/verify_inventory_vars.yml new file mode 100644 index 000000000..9a959a959 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/pre/verify_inventory_vars.yml @@ -0,0 +1,37 @@ +--- +- name: Verify upgrade can proceed on first master + hosts: oo_first_master + gather_facts: no + tasks: + - fail: + msg: > + This upgrade is only supported for origin, openshift-enterprise, and online + deployment types + when: deployment_type not in ['origin','openshift-enterprise', 'online'] + + # Error out in situations where the user has older versions specified in their + # inventory in any of the openshift_release, openshift_image_tag, and + # openshift_pkg_version variables. These must be removed or updated to proceed + # with upgrade. + # TODO: Should we block if you're *over* the next major release version as well? + - fail: + msg: > + openshift_pkg_version is {{ openshift_pkg_version }} which is not a + valid version for a {{ openshift_upgrade_target }} upgrade + when: openshift_pkg_version is defined and openshift_pkg_version.split('-',1).1 | version_compare(openshift_upgrade_target ,'<') + + - fail: + msg: > + openshift_image_tag is {{ openshift_image_tag }} which is not a + valid version for a {{ openshift_upgrade_target }} upgrade + when: openshift_image_tag is defined and openshift_image_tag.split('v',1).1 | version_compare(openshift_upgrade_target ,'<') + + - set_fact: + openshift_release: "{{ openshift_release[1:] }}" + when: openshift_release is defined and openshift_release[0] == 'v' + + - fail: + msg: > + openshift_release is {{ openshift_release }} which is not a + valid release for a {{ openshift_upgrade_target }} upgrade + when: openshift_release is defined and not openshift_release | version_compare(openshift_upgrade_target ,'=') diff --git a/playbooks/common/openshift-cluster/upgrades/pre/verify_nodes_running.yml b/playbooks/common/openshift-cluster/upgrades/pre/verify_nodes_running.yml new file mode 100644 index 000000000..354af3cde --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/pre/verify_nodes_running.yml @@ -0,0 +1,13 @@ +--- +- name: Verify node processes + hosts: oo_nodes_to_config + roles: + - openshift_facts + - openshift_docker_facts + tasks: + - name: Ensure Node is running + service: + name: "{{ openshift.common.service_type }}-node" + state: started + enabled: yes + when: openshift.common.is_containerized | bool diff --git a/playbooks/common/openshift-cluster/upgrades/pre/verify_upgrade_targets.yml b/playbooks/common/openshift-cluster/upgrades/pre/verify_upgrade_targets.yml new file mode 100644 index 000000000..9632626a4 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/pre/verify_upgrade_targets.yml @@ -0,0 +1,45 @@ +--- +- name: Verify upgrade targets + hosts: oo_masters_to_config:oo_nodes_to_upgrade + vars: + openshift_docker_hosted_registry_network: "{{ hostvars[groups.oo_first_master.0].openshift.common.portal_net }}" + pre_tasks: + - fail: + msg: Verify OpenShift is already installed + when: openshift.common.version is not defined + + - fail: + msg: Verify the correct version was found + when: verify_upgrade_version is defined and openshift_version != verify_upgrade_version + + - name: Clean package cache + command: "{{ ansible_pkg_mgr }} clean all" + when: not openshift.common.is_atomic | bool + + - set_fact: + g_new_service_name: "{{ 'origin' if deployment_type =='origin' else 'atomic-openshift' }}" + when: not openshift.common.is_containerized | bool + + - name: Verify containers are available for upgrade + command: > + docker pull {{ openshift.common.cli_image }}:{{ openshift_image_tag }} + register: pull_result + changed_when: "'Downloaded newer image' in pull_result.stdout" + when: openshift.common.is_containerized | bool + + - name: Check latest available OpenShift RPM version + command: > + {{ repoquery_cmd }} --qf '%{version}' "{{ openshift.common.service_type }}" + failed_when: false + changed_when: false + register: avail_openshift_version + when: not openshift.common.is_containerized | bool + + - name: Verify OpenShift RPMs are available for upgrade + fail: + msg: "OpenShift {{ avail_openshift_version.stdout }} is available, but {{ openshift_upgrade_target }} or greater is required" + when: not openshift.common.is_containerized | bool and not avail_openshift_version | skipped and avail_openshift_version.stdout | default('0.0', True) | version_compare(openshift_release, '<') + + - fail: + msg: "This upgrade playbook must be run against OpenShift {{ openshift_upgrade_min }} or later" + when: deployment_type == 'origin' and openshift.common.version | version_compare(openshift_upgrade_min,'<') diff --git a/playbooks/common/openshift-cluster/upgrades/rpm_upgrade.yml b/playbooks/common/openshift-cluster/upgrades/rpm_upgrade.yml index f5e4d807e..df2b664d4 100644 --- a/playbooks/common/openshift-cluster/upgrades/rpm_upgrade.yml +++ b/playbooks/common/openshift-cluster/upgrades/rpm_upgrade.yml @@ -1,10 +1,8 @@ +--- # We verified latest rpm available is suitable, so just yum update. - name: Upgrade packages - action: "{{ ansible_pkg_mgr }} name={{ openshift.common.service_type }}-{{ component }}{{ openshift_pkg_version }} state=present" + package: "name={{ openshift.common.service_type }}-{{ component }}{{ openshift_pkg_version }} state=present" - name: Ensure python-yaml present for config upgrade - action: "{{ ansible_pkg_mgr }} name=PyYAML state=present" + package: name=PyYAML state=present when: not openshift.common.is_atomic | bool - -- name: Restart node service - service: name="{{ openshift.common.service_type }}-node" state=restarted diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml index 8a2784fb4..6950b6166 100644 --- a/playbooks/common/openshift-cluster/upgrades/upgrade.yml +++ b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml @@ -1,50 +1,45 @@ --- ############################################################################### -# The restart playbook should be run after this playbook completes. +# Upgrade Masters ############################################################################### - -# Separate step so we can execute in parallel and clear out anything unused -# before we get into the serialized upgrade process which will then remove -# remaining images if possible. -- name: Cleanup unused Docker images - hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config +- name: Evaluate additional groups for upgrade + hosts: localhost + connection: local + become: no tasks: - - name: Check Docker image count - shell: "docker images -aq | wc -l" - register: docker_image_count - when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool - - - debug: var=docker_image_count.stdout - when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool - - - name: Remove unused Docker images for Docker 1.10+ migration - shell: "docker rmi `docker images -aq`" - # Will fail on images still in use: - failed_when: false - when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool - - - name: Check Docker image count - shell: "docker images -aq | wc -l" - register: docker_image_count - when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + - name: Evaluate etcd_hosts_to_backup + add_host: + name: "{{ item }}" + groups: etcd_hosts_to_backup + with_items: "{{ groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else groups.oo_first_master }}" + +# If facts cache were for some reason deleted, this fact may not be set, and if not set +# it will always default to true. This causes problems for the etcd data dir fact detection +# so we must first make sure this is set correctly before attempting the backup. +- name: Set master embedded_etcd fact + hosts: oo_masters_to_config + roles: + - openshift_facts + tasks: + - openshift_facts: + role: master + local_facts: + embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" + debug_level: "{{ openshift_master_debug_level | default(openshift.common.debug_level | default(2)) }}" - - debug: var=docker_image_count.stdout - when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool +- name: Upgrade and backup etcd + include: ./etcd/main.yml -############################################################################### -# Upgrade Masters -############################################################################### - name: Upgrade master packages hosts: oo_masters_to_config - handlers: - - include: ../../../../roles/openshift_master/handlers/main.yml - static: yes roles: - openshift_facts tasks: - include: rpm_upgrade.yml component=master when: not openshift.common.is_containerized | bool +# Create service signer cert when missing. Service signer certificate +# is added to master config in the master config hook for v3_3. - name: Determine if service signer cert must be created hosts: oo_first_master tasks: @@ -54,10 +49,15 @@ register: service_signer_cert_stat changed_when: false -# Create service signer cert when missing. Service signer certificate -# is added to master config in the master config hook for v3_3. - include: create_service_signer_cert.yml - when: not (hostvars[groups.oo_first_master.0].service_signer_cert_stat.stat.exists | bool) + +# Set openshift_master_facts separately. In order to reconcile +# admission_config's, we currently must run openshift_master_facts and +# then run openshift_facts. +- name: Set OpenShift master facts + hosts: oo_masters_to_config + roles: + - openshift_master_facts - name: Upgrade master config and systemd units hosts: oo_masters_to_config @@ -66,7 +66,11 @@ static: yes roles: - openshift_facts - tasks: + post_tasks: + - include_vars: ../../../../roles/openshift_master_facts/vars/main.yml + + - include: upgrade_scheduler.yml + - include: "{{ master_config_hook }}" when: master_config_hook is defined @@ -75,13 +79,6 @@ - name: Update systemd units include: ../../../../roles/openshift_master/tasks/systemd_units.yml -# - name: Upgrade master configuration -# openshift_upgrade_config: -# from_version: '3.1' -# to_version: '3.2' -# role: master -# config_base: "{{ hostvars[inventory_hostname].openshift.common.config_base }}" - - name: Check for ca-bundle.crt stat: path: "{{ openshift.common.config_base }}/master/ca-bundle.crt" @@ -131,6 +128,10 @@ msg: "Upgrade cannot continue. The following masters did not finish updating: {{ master_update_failed | join(',') }}" when: master_update_failed | length > 0 +# We are now ready to restart master services (or entire system +# depending on openshift_rolling_restart_mode): +- include: ../../openshift-master/restart.yml + ############################################################################### # Reconcile Cluster Roles, Cluster Role Bindings and Security Context Constraints ############################################################################### @@ -143,23 +144,19 @@ origin_reconcile_bindings: "{{ deployment_type == 'origin' and openshift_version | version_compare('1.0.6', '>') }}" ent_reconcile_bindings: true openshift_docker_hosted_registry_network: "{{ hostvars[groups.oo_first_master.0].openshift.common.portal_net }}" - # Similar to pre.yml, we don't want to upgrade docker during the openshift_cli role, - # it will be updated when we perform node upgrade. - docker_protect_installed_version: True + # Another spot where we assume docker is running and do not want to accidentally trigger an unsafe + # restart. + skip_docker_role: True tasks: - - name: Verifying the correct commandline tools are available - shell: grep {{ verify_upgrade_version }} {{ openshift.common.admin_binary}} - when: openshift.common.is_containerized | bool and verify_upgrade_version is defined - - name: Reconcile Cluster Roles command: > - {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig + {{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig policy reconcile-cluster-roles --additive-only=true --confirm run_once: true - name: Reconcile Cluster Role Bindings command: > - {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig + {{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig policy reconcile-cluster-role-bindings --exclude-groups=system:authenticated --exclude-groups=system:authenticated:oauth @@ -169,78 +166,19 @@ when: origin_reconcile_bindings | bool or ent_reconcile_bindings | bool run_once: true - - name: Reconcile Security Context Constraints + - name: Reconcile Jenkins Pipeline Role Bindings command: > - {{ openshift.common.admin_binary}} policy reconcile-sccs --confirm --additive-only=true + {{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig policy reconcile-cluster-role-bindings system:build-strategy-jenkinspipeline --confirm run_once: true + when: openshift.common.version_gte_3_4_or_1_4 | bool - - set_fact: - reconcile_complete: True - -############################################################################### -# Upgrade Nodes -############################################################################### - -# Here we handle all tasks that might require a node evac. (upgrading docker, and the node service) -- name: Perform upgrades that may require node evacuation - hosts: oo_masters_to_config:oo_etcd_to_config:oo_nodes_to_config - serial: 1 - any_errors_fatal: true - roles: - - openshift_facts - handlers: - - include: ../../../../roles/openshift_node/handlers/main.yml - static: yes - tasks: - # TODO: To better handle re-trying failed upgrades, it would be nice to check if the node - # or docker actually needs an upgrade before proceeding. Perhaps best to save this until - # we merge upgrade functionality into the base roles and a normal config.yml playbook run. - - name: Determine if node is currently scheduleable + - name: Reconcile Security Context Constraints command: > - {{ openshift.common.client_binary }} get node {{ openshift.node.nodename }} -o json - register: node_output - delegate_to: "{{ groups.oo_first_master.0 }}" - changed_when: false - when: inventory_hostname in groups.oo_nodes_to_config + {{ openshift.common.client_binary }} adm policy reconcile-sccs --confirm --additive-only=true + run_once: true - set_fact: - was_schedulable: "{{ 'unschedulable' not in (node_output.stdout | from_json).spec }}" - when: inventory_hostname in groups.oo_nodes_to_config - - - name: Mark unschedulable if host is a node - command: > - {{ openshift.common.admin_binary }} manage-node {{ openshift.node.nodename }} --schedulable=false - delegate_to: "{{ groups.oo_first_master.0 }}" - when: inventory_hostname in groups.oo_nodes_to_config - - - name: Evacuate Node for Kubelet upgrade - command: > - {{ openshift.common.admin_binary }} manage-node {{ openshift.node.nodename }} --evacuate --force - delegate_to: "{{ groups.oo_first_master.0 }}" - when: inventory_hostname in groups.oo_nodes_to_config - - - include: docker/upgrade.yml - when: l_docker_upgrade is defined and l_docker_upgrade | bool and not openshift.common.is_atomic | bool - - include: "{{ node_config_hook }}" - when: node_config_hook is defined and inventory_hostname in groups.oo_nodes_to_config - - - include: rpm_upgrade.yml - vars: - component: "node" - openshift_version: "{{ openshift_pkg_version | default('') }}" - when: inventory_hostname in groups.oo_nodes_to_config and not openshift.common.is_containerized | bool - - - include: containerized_node_upgrade.yml - when: inventory_hostname in groups.oo_nodes_to_config and openshift.common.is_containerized | bool - - - meta: flush_handlers - - - name: Set node schedulability - command: > - {{ openshift.common.admin_binary }} manage-node {{ openshift.node.nodename }} --schedulable=true - delegate_to: "{{ groups.oo_first_master.0 }}" - when: inventory_hostname in groups.oo_nodes_to_config and was_schedulable | bool - + reconcile_complete: True ############################################################################## # Gate on reconcile @@ -259,3 +197,13 @@ - fail: msg: "Upgrade cannot continue. The following masters did not finish reconciling: {{ reconcile_failed | join(',') }}" when: reconcile_failed | length > 0 + +- name: Upgrade Docker on dedicated containerized etcd hosts + hosts: oo_etcd_to_config:!oo_nodes_to_upgrade + serial: 1 + any_errors_fatal: true + roles: + - openshift_facts + tasks: + - include: docker/upgrade.yml + when: l_docker_upgrade is defined and l_docker_upgrade | bool and not openshift.common.is_atomic | bool diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml b/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml new file mode 100644 index 000000000..86b344d7a --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml @@ -0,0 +1,98 @@ +--- +- name: Drain and upgrade nodes + hosts: oo_nodes_to_upgrade + # This var must be set with -e on invocation, as it is not a per-host inventory var + # and is evaluated early. Values such as "20%" can also be used. + serial: "{{ openshift_upgrade_nodes_serial | default(1) }}" + any_errors_fatal: true + roles: + - openshift_facts + - docker + handlers: + - include: ../../../../roles/openshift_node/handlers/main.yml + static: yes + pre_tasks: + # TODO: To better handle re-trying failed upgrades, it would be nice to check if the node + # or docker actually needs an upgrade before proceeding. Perhaps best to save this until + # we merge upgrade functionality into the base roles and a normal config.yml playbook run. + - name: Determine if node is currently scheduleable + command: > + {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} get node {{ openshift.node.nodename | lower }} -o json + register: node_output + delegate_to: "{{ groups.oo_first_master.0 }}" + changed_when: false + when: inventory_hostname in groups.oo_nodes_to_upgrade + + - set_fact: + was_schedulable: "{{ 'unschedulable' not in (node_output.stdout | from_json).spec }}" + when: inventory_hostname in groups.oo_nodes_to_upgrade + + - name: Mark unschedulable if host is a node + command: > + {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --schedulable=false + delegate_to: "{{ groups.oo_first_master.0 }}" + when: inventory_hostname in groups.oo_nodes_to_upgrade + # NOTE: There is a transient "object has been modified" error here, allow a couple + # retries for a more reliable upgrade. + register: node_unsched + until: node_unsched.rc == 0 + retries: 3 + delay: 1 + + - name: Drain Node for Kubelet upgrade + command: > + {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} {{ openshift.common.evacuate_or_drain }} --force + delegate_to: "{{ groups.oo_first_master.0 }}" + when: inventory_hostname in groups.oo_nodes_to_upgrade + + tasks: + + - include: docker/upgrade.yml + vars: + # We will restart Docker ourselves after everything is ready: + skip_docker_restart: True + when: l_docker_upgrade is defined and l_docker_upgrade | bool and not openshift.common.is_atomic | bool + + - include: "{{ node_config_hook }}" + when: node_config_hook is defined and inventory_hostname in groups.oo_nodes_to_upgrade + + - include: rpm_upgrade.yml + vars: + component: "node" + openshift_version: "{{ openshift_pkg_version | default('') }}" + when: inventory_hostname in groups.oo_nodes_to_upgrade and not openshift.common.is_containerized | bool + + - name: Remove obsolete docker-sdn-ovs.conf + file: path=/etc/systemd/system/docker.service.d/docker-sdn-ovs.conf state=absent + when: (deployment_type == 'openshift-enterprise' and openshift_release | version_compare('3.4', '>=')) or (deployment_type == 'origin' and openshift_release | version_compare('1.4', '>=')) + + - include: containerized_node_upgrade.yml + when: inventory_hostname in groups.oo_nodes_to_upgrade and openshift.common.is_containerized | bool + + - name: Ensure containerized services stopped before Docker restart + service: name={{ item }} state=stopped + with_items: + - etcd_container + - openvswitch + - "{{ openshift.common.service_type }}-master" + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + failed_when: false + when: openshift.common.is_containerized | bool + + # Mandatory Docker restart, ensure all containerized services are running: + - include: docker/restart.yml + + - name: Restart rpm node service + service: name="{{ openshift.common.service_type }}-node" state=restarted + when: inventory_hostname in groups.oo_nodes_to_upgrade and not openshift.common.is_containerized | bool + - name: Set node schedulability + command: > + {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --schedulable=true + delegate_to: "{{ groups.oo_first_master.0 }}" + when: inventory_hostname in groups.oo_nodes_to_upgrade and was_schedulable | bool + register: node_sched + until: node_sched.rc == 0 + retries: 3 + delay: 1 diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade_scheduler.yml b/playbooks/common/openshift-cluster/upgrades/upgrade_scheduler.yml new file mode 100644 index 000000000..88f2ddc78 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/upgrade_scheduler.yml @@ -0,0 +1,166 @@ +--- +# Upgrade predicates +- vars: + prev_predicates: "{{ lookup('openshift_master_facts_default_predicates', short_version=openshift_upgrade_min, deployment_type=openshift_deployment_type) }}" + prev_predicates_no_region: "{{ lookup('openshift_master_facts_default_predicates', short_version=openshift_upgrade_min, deployment_type=openshift_deployment_type, regions_enabled=False) }}" + default_predicates_no_region: "{{ lookup('openshift_master_facts_default_predicates', regions_enabled=False) }}" + # older_predicates are the set of predicates that have previously been + # hard-coded into openshift_facts + older_predicates: + - - name: MatchNodeSelector + - name: PodFitsResources + - name: PodFitsPorts + - name: NoDiskConflict + - name: NoVolumeZoneConflict + - name: MaxEBSVolumeCount + - name: MaxGCEPDVolumeCount + - name: Region + argument: + serviceAffinity: + labels: + - region + - - name: MatchNodeSelector + - name: PodFitsResources + - name: PodFitsPorts + - name: NoDiskConflict + - name: NoVolumeZoneConflict + - name: Region + argument: + serviceAffinity: + labels: + - region + - - name: MatchNodeSelector + - name: PodFitsResources + - name: PodFitsPorts + - name: NoDiskConflict + - name: Region + argument: + serviceAffinity: + labels: + - region + # older_predicates_no_region are the set of predicates that have previously + # been hard-coded into openshift_facts, with the Region predicate removed + older_predicates_no_region: + - - name: MatchNodeSelector + - name: PodFitsResources + - name: PodFitsPorts + - name: NoDiskConflict + - name: NoVolumeZoneConflict + - name: MaxEBSVolumeCount + - name: MaxGCEPDVolumeCount + - - name: MatchNodeSelector + - name: PodFitsResources + - name: PodFitsPorts + - name: NoDiskConflict + - name: NoVolumeZoneConflict + - - name: MatchNodeSelector + - name: PodFitsResources + - name: PodFitsPorts + - name: NoDiskConflict + block: + + # Handle case where openshift_master_predicates is defined + - block: + - debug: + msg: "WARNING: openshift_master_scheduler_predicates is set to defaults from an earlier release of OpenShift current defaults are: {{ openshift_master_scheduler_default_predicates }}" + when: "{{ openshift_master_scheduler_predicates in older_predicates + older_predicates_no_region + [prev_predicates] + [prev_predicates_no_region] }}" + + - debug: + msg: "WARNING: openshift_master_scheduler_predicates does not match current defaults of: {{ openshift_master_scheduler_default_predicates }}" + when: "{{ openshift_master_scheduler_predicates != openshift_master_scheduler_default_predicates }}" + when: "{{ openshift_master_scheduler_predicates | default(none) is not none }}" + + # Handle cases where openshift_master_predicates is not defined + - block: + - debug: + msg: "WARNING: existing scheduler config does not match previous known defaults automated upgrade of scheduler config is disabled.\nexisting scheduler predicates: {{ openshift_master_scheduler_current_predicates }}\ncurrent scheduler default predicates are: {{ openshift_master_scheduler_default_predicates }}" + when: "{{ openshift_master_scheduler_current_predicates != openshift_master_scheduler_default_predicates and + openshift_master_scheduler_current_predicates not in older_predicates + [prev_predicates] }}" + + - set_fact: + openshift_upgrade_scheduler_predicates: "{{ openshift_master_scheduler_default_predicates }}" + when: "{{ openshift_master_scheduler_current_predicates != openshift_master_scheduler_default_predicates and + openshift_master_scheduler_current_predicates in older_predicates + [prev_predicates] }}" + + - set_fact: + openshift_upgrade_scheduler_predicates: "{{ default_predicates_no_region }}" + when: "{{ openshift_master_scheduler_current_predicates != default_predicates_no_region and + openshift_master_scheduler_current_predicates in older_predicates_no_region + [prev_predicates_no_region] }}" + + when: "{{ openshift_master_scheduler_predicates | default(none) is none }}" + + +# Upgrade priorities +- vars: + prev_priorities: "{{ lookup('openshift_master_facts_default_priorities', short_version=openshift_upgrade_min, deployment_type=openshift_deployment_type) }}" + prev_priorities_no_zone: "{{ lookup('openshift_master_facts_default_priorities', short_version=openshift_upgrade_min, deployment_type=openshift_deployment_type, zones_enabled=False) }}" + default_priorities_no_zone: "{{ lookup('openshift_master_facts_default_priorities', zones_enabled=False) }}" + # older_priorities are the set of priorities that have previously been + # hard-coded into openshift_facts + older_priorities: + - - name: LeastRequestedPriority + weight: 1 + - name: SelectorSpreadPriority + weight: 1 + - name: Zone + weight: 2 + argument: + serviceAntiAffinity: + label: zone + # older_priorities_no_region are the set of priorities that have previously + # been hard-coded into openshift_facts, with the Zone priority removed + older_priorities_no_zone: + - - name: LeastRequestedPriority + weight: 1 + - name: SelectorSpreadPriority + weight: 1 + block: + + # Handle case where openshift_master_priorities is defined + - block: + - debug: + msg: "WARNING: openshift_master_scheduler_priorities is set to defaults from an earlier release of OpenShift current defaults are: {{ openshift_master_scheduler_default_priorities }}" + when: "{{ openshift_master_scheduler_priorities in older_priorities + older_priorities_no_zone + [prev_priorities] + [prev_priorities_no_zone] }}" + + - debug: + msg: "WARNING: openshift_master_scheduler_priorities does not match current defaults of: {{ openshift_master_scheduler_default_priorities }}" + when: "{{ openshift_master_scheduler_priorities != openshift_master_scheduler_default_priorities }}" + when: "{{ openshift_master_scheduler_priorities | default(none) is not none }}" + + # Handle cases where openshift_master_priorities is not defined + - block: + - debug: + msg: "WARNING: existing scheduler config does not match previous known defaults automated upgrade of scheduler config is disabled.\nexisting scheduler priorities: {{ openshift_master_scheduler_current_priorities }}\ncurrent scheduler default priorities are: {{ openshift_master_scheduler_default_priorities }}" + when: "{{ openshift_master_scheduler_current_priorities != openshift_master_scheduler_default_priorities and + openshift_master_scheduler_current_priorities not in older_priorities + [prev_priorities] }}" + + - set_fact: + openshift_upgrade_scheduler_priorities: "{{ openshift_master_scheduler_default_priorities }}" + when: "{{ openshift_master_scheduler_current_priorities != openshift_master_scheduler_default_priorities and + openshift_master_scheduler_current_priorities in older_priorities + [prev_priorities] }}" + + - set_fact: + openshift_upgrade_scheduler_priorities: "{{ default_priorities_no_zone }}" + when: "{{ openshift_master_scheduler_current_priorities != default_priorities_no_zone and + openshift_master_scheduler_current_priorities in older_priorities_no_zone + [prev_priorities_no_zone] }}" + + when: "{{ openshift_master_scheduler_priorities | default(none) is none }}" + + +# Update scheduler +- vars: + scheduler_config: + kind: Policy + apiVersion: v1 + predicates: "{{ openshift_upgrade_scheduler_predicates + | default(openshift_master_scheduler_current_predicates) }}" + priorities: "{{ openshift_upgrade_scheduler_priorities + | default(openshift_master_scheduler_current_priorities) }}" + block: + - name: Update scheduler config + copy: + content: "{{ scheduler_config | to_nice_json }}" + dest: "{{ openshift_master_scheduler_conf }}" + backup: true + when: "{{ openshift_upgrade_scheduler_predicates is defined or + openshift_upgrade_scheduler_priorities is defined }}" diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/nuke_images.sh b/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/nuke_images.sh deleted file mode 120000 index 49a51bba9..000000000 --- a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/nuke_images.sh +++ /dev/null @@ -1 +0,0 @@ -../files/nuke_images.sh
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/v3_3/master_config_upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_3/master_config_upgrade.yml index 684eea343..68c71a132 100644 --- a/playbooks/common/openshift-cluster/upgrades/v3_3/master_config_upgrade.yml +++ b/playbooks/common/openshift-cluster/upgrades/v3_3/master_config_upgrade.yml @@ -48,3 +48,19 @@ dest: "{{ openshift.common.config_base}}/master/master-config.yaml" yaml_key: 'controllerConfig.servicesServingCert.signer.keyFile' yaml_value: service-signer.key + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'admissionConfig.pluginConfig' + yaml_value: "{{ openshift.master.admission_plugin_config }}" + when: "{{ 'admission_plugin_config' in openshift.master }}" + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'admissionConfig.pluginOrderOverride' + yaml_value: + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'kubernetesMasterConfig.admissionConfig' + yaml_value: diff --git a/playbooks/common/openshift-cluster/upgrades/v3_3/node_config_upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_3/node_config_upgrade.yml index 8f64636ae..89b524f14 100644 --- a/playbooks/common/openshift-cluster/upgrades/v3_3/node_config_upgrade.yml +++ b/playbooks/common/openshift-cluster/upgrades/v3_3/node_config_upgrade.yml @@ -18,4 +18,3 @@ dest: "{{ openshift.common.config_base}}/node/node-config.yaml" yaml_key: 'masterClientConnectionOverrides.qps' yaml_value: 20 - diff --git a/playbooks/common/openshift-cluster/upgrades/v3_4/master_config_upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_4/master_config_upgrade.yml new file mode 100644 index 000000000..43c2ffcd4 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_4/master_config_upgrade.yml @@ -0,0 +1,16 @@ +--- +- modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'admissionConfig.pluginConfig' + yaml_value: "{{ openshift.master.admission_plugin_config }}" + when: "{{ 'admission_plugin_config' in openshift.master }}" + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'admissionConfig.pluginOrderOverride' + yaml_value: + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'kubernetesMasterConfig.admissionConfig' + yaml_value: diff --git a/playbooks/common/openshift-cluster/validate_hostnames.yml b/playbooks/common/openshift-cluster/validate_hostnames.yml index 50e25984f..48cc03b19 100644 --- a/playbooks/common/openshift-cluster/validate_hostnames.yml +++ b/playbooks/common/openshift-cluster/validate_hostnames.yml @@ -11,6 +11,6 @@ failed_when: false - name: Warn user about bad openshift_hostname values pause: - prompt: "The hostname \"{{ openshift.common.hostname }}\" for \"{{ ansible_nodename }}\" doesn't resolve to an ip address owned by this host. Please set openshift_hostname variable to a hostname that when resolved on the host in question resolves to an IP address matching an interface on this host. This host will fail liveness checks for pods utilizing hostPorts, press ENTER to continue or CTRL-C to abort." - seconds: "{{ 10 if openshift_override_hostname_check | default(false) | bool else omit }}" + prompt: "The hostname \"{{ openshift.common.hostname }}\" for \"{{ ansible_nodename }}\" doesn't resolve to an ip address owned by this host. Please set openshift_hostname variable to a hostname that when resolved on the host in question resolves to an IP address matching an interface on this host. This host will fail liveness checks for pods utilizing hostPorts, press ENTER to continue or CTRL-C to abort." + seconds: "{{ 10 if openshift_override_hostname_check | default(false) | bool else omit }}" when: lookupip.stdout not in ansible_all_ipv4_addresses diff --git a/playbooks/common/openshift-cluster/verify_ansible_version.yml b/playbooks/common/openshift-cluster/verify_ansible_version.yml deleted file mode 100644 index 2a143b065..000000000 --- a/playbooks/common/openshift-cluster/verify_ansible_version.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- hosts: localhost - connection: local - become: no - gather_facts: no - tasks: - - name: Verify Ansible version is greater than or equal to 2.1.0.0 - fail: - msg: "Unsupported ansible version: {{ ansible_version.full }} found" - when: not ansible_version.full | version_compare('2.1.0.0', 'ge') |