diff options
Diffstat (limited to 'roles/openshift_node_upgrade/tasks')
14 files changed, 447 insertions, 0 deletions
diff --git a/roles/openshift_node_upgrade/tasks/config/configure-node-settings.yml b/roles/openshift_node_upgrade/tasks/config/configure-node-settings.yml new file mode 100644 index 000000000..527580481 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/config/configure-node-settings.yml @@ -0,0 +1,16 @@ +--- +- name: Configure Node settings + lineinfile: + dest: /etc/sysconfig/{{ openshift.common.service_type }}-node + regexp: "{{ item.regex }}" + line: "{{ item.line }}" + create: true + with_items: + - regex: '^OPTIONS=' + line: "OPTIONS=--loglevel={{ openshift_node_debug_level }}" + - regex: '^CONFIG_FILE=' + line: "CONFIG_FILE={{ openshift.common.config_base }}/node/node-config.yaml" + - regex: '^IMAGE_VERSION=' + line: "IMAGE_VERSION={{ openshift_image_tag }}" + notify: + - restart node diff --git a/roles/openshift_node_upgrade/tasks/config/configure-proxy-settings.yml b/roles/openshift_node_upgrade/tasks/config/configure-proxy-settings.yml new file mode 100644 index 000000000..d60794305 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/config/configure-proxy-settings.yml @@ -0,0 +1,17 @@ +--- +- name: Configure Proxy Settings + lineinfile: + dest: /etc/sysconfig/{{ openshift.common.service_type }}-node + regexp: "{{ item.regex }}" + line: "{{ item.line }}" + create: true + with_items: + - regex: '^HTTP_PROXY=' + line: "HTTP_PROXY={{ openshift.common.http_proxy | default('') }}" + - regex: '^HTTPS_PROXY=' + line: "HTTPS_PROXY={{ openshift.common.https_proxy | default('') }}" + - regex: '^NO_PROXY=' + line: "NO_PROXY={{ openshift.common.no_proxy | default([]) }},{{ openshift.common.portal_net }},{{ hostvars[groups.oo_first_master.0].openshift.master.sdn_cluster_network_cidr }}" + when: ('http_proxy' in openshift.common and openshift.common.http_proxy != '') + notify: + - restart node diff --git a/roles/openshift_node_upgrade/tasks/config/install-node-deps-docker-service-file.yml b/roles/openshift_node_upgrade/tasks/config/install-node-deps-docker-service-file.yml new file mode 100644 index 000000000..ee91a88ab --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/config/install-node-deps-docker-service-file.yml @@ -0,0 +1,8 @@ +--- +- name: Install Node dependencies docker service file + template: + dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node-dep.service" + src: openshift.docker.node.dep.service + notify: + - reload systemd units + - restart node diff --git a/roles/openshift_node_upgrade/tasks/config/install-node-docker-service-file.yml b/roles/openshift_node_upgrade/tasks/config/install-node-docker-service-file.yml new file mode 100644 index 000000000..f92ff79b5 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/config/install-node-docker-service-file.yml @@ -0,0 +1,8 @@ +--- +- name: Install Node docker service file + template: + dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node.service" + src: openshift.docker.node.service + notify: + - reload systemd units + - restart node diff --git a/roles/openshift_node_upgrade/tasks/config/install-ovs-docker-service-file.yml b/roles/openshift_node_upgrade/tasks/config/install-ovs-docker-service-file.yml new file mode 100644 index 000000000..c2c5ea1d4 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/config/install-ovs-docker-service-file.yml @@ -0,0 +1,8 @@ +--- +- name: Install OpenvSwitch docker service file + template: + dest: "/etc/systemd/system/openvswitch.service" + src: openvswitch.docker.service + notify: + - reload systemd units + - restart openvswitch diff --git a/roles/openshift_node_upgrade/tasks/config/install-ovs-service-env-file.yml b/roles/openshift_node_upgrade/tasks/config/install-ovs-service-env-file.yml new file mode 100644 index 000000000..1d75a3355 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/config/install-ovs-service-env-file.yml @@ -0,0 +1,8 @@ +--- +- name: Create the openvswitch service env file + template: + src: openvswitch.sysconfig.j2 + dest: /etc/sysconfig/openvswitch + notify: + - reload systemd units + - restart openvswitch diff --git a/roles/openshift_node_upgrade/tasks/config/workaround-bz1331590-ovs-oom-fix.yml b/roles/openshift_node_upgrade/tasks/config/workaround-bz1331590-ovs-oom-fix.yml new file mode 100644 index 000000000..5df1abc79 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/config/workaround-bz1331590-ovs-oom-fix.yml @@ -0,0 +1,13 @@ +--- +# May be a temporary workaround. +# https://bugzilla.redhat.com/show_bug.cgi?id=1331590 +- name: Create OpenvSwitch service.d directory + file: path=/etc/systemd/system/openvswitch.service.d/ state=directory + +- name: Install OpenvSwitch service OOM fix + template: + dest: "/etc/systemd/system/openvswitch.service.d/01-avoid-oom.conf" + src: openvswitch-avoid-oom.conf + notify: + - reload systemd units + - restart openvswitch diff --git a/roles/openshift_node_upgrade/tasks/containerized_node_upgrade.yml b/roles/openshift_node_upgrade/tasks/containerized_node_upgrade.yml new file mode 100644 index 000000000..07b0ac715 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/containerized_node_upgrade.yml @@ -0,0 +1,14 @@ +--- +# This is a hack to allow us to use systemd_units.yml, but skip the handlers which +# restart services. We will unconditionally restart all containerized services +# because we have to unconditionally restart Docker: +- set_fact: + skip_node_svc_handlers: True + +- name: Update systemd units + include: systemd_units.yml + +# This is a no-op because of skip_node_svc_handlers, but lets us trigger it before end of +# play when the node has already been marked schedulable again. (this would look strange +# in logs otherwise) +- meta: flush_handlers diff --git a/roles/openshift_node_upgrade/tasks/docker/upgrade.yml b/roles/openshift_node_upgrade/tasks/docker/upgrade.yml new file mode 100644 index 000000000..ebe87d6fd --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/docker/upgrade.yml @@ -0,0 +1,40 @@ +--- +# input variables: +# - openshift.common.service_type +# - openshift.common.is_containerized +# - docker_upgrade_nuke_images +# - docker_version +# - skip_docker_restart + +- name: Check Docker image count + shell: "docker images -aq | wc -l" + register: docker_image_count + +- debug: var=docker_image_count.stdout + +# TODO(jchaloup): put all docker_upgrade_nuke_images into a block with only one condition +- name: Remove all containers and images + script: nuke_images.sh + register: nuke_images_result + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- name: Check Docker image count + shell: "docker images -aq | wc -l" + register: docker_image_count + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- debug: var=docker_image_count.stdout + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- service: + name: docker + state: stopped + register: l_openshift_node_upgrade_docker_stop_result + until: not l_openshift_node_upgrade_docker_stop_result | failed + retries: 3 + delay: 30 + +- name: Upgrade Docker + package: name=docker{{ '-' + docker_version }} state=present + +# starting docker happens back in ../main.yml where it calls ../restart.yml diff --git a/roles/openshift_node_upgrade/tasks/main.yml b/roles/openshift_node_upgrade/tasks/main.yml new file mode 100644 index 000000000..6bcf3072d --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/main.yml @@ -0,0 +1,179 @@ +--- +# input variables: +# - l_docker_upgrade +# - openshift.common.is_atomic +# - node_config_hook +# - openshift_pkg_version +# - openshift.common.is_containerized +# - deployment_type +# - openshift_release + +# tasks file for openshift_node_upgrade + +- include: registry_auth.yml + +- name: Stop node and openvswitch services + service: + name: "{{ item }}" + state: stopped + with_items: + - "{{ openshift.common.service_type }}-node" + - openvswitch + failed_when: false + +- name: Stop additional containerized services + service: + name: "{{ item }}" + state: stopped + with_items: + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-master-api" + - etcd_container + failed_when: false + when: openshift.common.is_containerized | bool + +- name: Pre-pull node image + command: > + docker pull {{ openshift.node.node_image }}:{{ openshift_image_tag }} + register: pull_result + changed_when: "'Downloaded newer image' in pull_result.stdout" + when: openshift.common.is_containerized | bool + +- name: Pre-pull openvswitch image + command: > + docker pull {{ openshift.node.ovs_image }}:{{ openshift_image_tag }} + register: pull_result + changed_when: "'Downloaded newer image' in pull_result.stdout" + when: + - openshift.common.is_containerized | bool + - openshift_use_openshift_sdn | bool + +- include: docker/upgrade.yml + vars: + # We will restart Docker ourselves after everything is ready: + skip_docker_restart: True + when: + - l_docker_upgrade is defined + - l_docker_upgrade | bool + +- include: "{{ node_config_hook }}" + when: node_config_hook is defined + +- include: rpm_upgrade.yml + vars: + component: "node" + openshift_version: "{{ openshift_pkg_version | default('') }}" + when: not openshift.common.is_containerized | bool + +- name: Remove obsolete docker-sdn-ovs.conf + file: + path: "/etc/systemd/system/docker.service.d/docker-sdn-ovs.conf" + state: absent + when: (deployment_type == 'openshift-enterprise' and openshift_release | version_compare('3.4', '>=')) + or (deployment_type == 'origin' and openshift_release | version_compare('1.4', '>=')) + +- include: containerized_node_upgrade.yml + when: openshift.common.is_containerized | bool + +- name: Ensure containerized services stopped before Docker restart + service: + name: "{{ item }}" + state: stopped + with_items: + - etcd_container + - openvswitch + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + failed_when: false + when: openshift.common.is_containerized | bool + +- name: Stop rpm based services + service: + name: "{{ item }}" + state: stopped + with_items: + - "{{ openshift.common.service_type }}-node" + - openvswitch + failed_when: false + when: not openshift.common.is_containerized | bool + +- name: Upgrade openvswitch + package: + name: openvswitch + state: latest + when: not openshift.common.is_containerized | bool + +- name: Update oreg value + yedit: + src: "{{ openshift.common.config_base }}/node/node-config.yaml" + key: 'imageConfig.format' + value: "{{ oreg_url | default(oreg_url_node) }}" + when: oreg_url is defined or oreg_url_node is defined + +# https://docs.openshift.com/container-platform/3.4/admin_guide/overcommit.html#disabling-swap-memory +- name: Check for swap usage + command: grep "^[^#].*swap" /etc/fstab + # grep: match any lines which don't begin with '#' and contain 'swap' + changed_when: false + failed_when: false + register: swap_result + + # Disable Swap Block +- block: + + - name: Disable swap + command: swapoff --all + + - name: Remove swap entries from /etc/fstab + replace: + dest: /etc/fstab + regexp: '(^[^#].*swap.*)' + replace: '# \1' + backup: yes + + - name: Add notice about disabling swap + lineinfile: + dest: /etc/fstab + line: '# OpenShift-Ansible Installer disabled swap per overcommit guidelines' + state: present + + when: + - swap_result.stdout_lines | length > 0 + - openshift_disable_swap | default(true) | bool + # End Disable Swap Block + +- name: Reset selinux context + command: restorecon -RF {{ openshift_node_data_dir }}/openshift.local.volumes + when: + - ansible_selinux is defined + - ansible_selinux.status == 'enabled' + +- name: Apply 3.6 dns config changes + yedit: + src: /etc/origin/node/node-config.yaml + key: "{{ item.key }}" + value: "{{ item.value }}" + with_items: + - key: "dnsBindAddress" + value: "127.0.0.1:53" + - key: "dnsRecursiveResolvConf" + value: "/etc/origin/node/resolv.conf" + +# Restart all services +- include: restart.yml + +- name: Wait for node to be ready + oc_obj: + state: list + kind: node + name: "{{ openshift.common.hostname | lower }}" + register: node_output + delegate_to: "{{ groups.oo_first_master.0 }}" + until: node_output.results.returncode == 0 and node_output.results.results[0].status.conditions | selectattr('type', 'match', '^Ready$') | map(attribute='status') | join | bool == True + # Give the node two minutes to come back online. + retries: 24 + delay: 5 + +- include_role: + name: openshift_node_dnsmasq diff --git a/roles/openshift_node_upgrade/tasks/registry_auth.yml b/roles/openshift_node_upgrade/tasks/registry_auth.yml new file mode 100644 index 000000000..de396fb4b --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/registry_auth.yml @@ -0,0 +1,24 @@ +--- +- name: Check for credentials file for registry auth + stat: + path: "{{ oreg_auth_credentials_path }}" + when: oreg_auth_user is defined + register: node_oreg_auth_credentials_stat + +- name: Create credentials for registry auth + command: "docker --config={{ oreg_auth_credentials_path }} login -u {{ oreg_auth_user }} -p {{ oreg_auth_password }} {{ oreg_host }}" + when: + - oreg_auth_user is defined + - (not node_oreg_auth_credentials_stat.stat.exists or oreg_auth_credentials_replace) | bool + register: node_oreg_auth_credentials_create + notify: + - restart node + +# Container images may need the registry credentials +- name: Setup ro mount of /root/.docker for containerized hosts + set_fact: + l_bind_docker_reg_auth: True + when: + - openshift.common.is_containerized | bool + - oreg_auth_user is defined + - (node_oreg_auth_credentials_stat.stat.exists or oreg_auth_credentials_replace or node_oreg_auth_credentials_create.changed) | bool diff --git a/roles/openshift_node_upgrade/tasks/restart.yml b/roles/openshift_node_upgrade/tasks/restart.yml new file mode 100644 index 000000000..a4fa51172 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/restart.yml @@ -0,0 +1,46 @@ +--- +# input variables: +# - openshift.common.service_type +# - openshift.common.is_containerized +# - openshift.common.hostname +# - openshift.master.api_port + +# NOTE: This is needed to make sure we are using the correct set +# of systemd unit files. The RPMs lay down defaults but +# the install/upgrade may override them in /etc/systemd/system/. +# NOTE: We don't use the systemd module as some versions of the module +# require a service to be part of the call. +- name: Reload systemd to ensure latest unit files + command: systemctl daemon-reload + +- name: Restart docker + service: + name: "{{ openshift.docker.service_name }}" + state: started + register: docker_start_result + until: not docker_start_result | failed + retries: 3 + delay: 30 + +- name: Update docker facts + openshift_facts: + role: docker + +- name: Start services + service: name={{ item }} state=started + with_items: + - etcd_container + - openvswitch + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + failed_when: false + +- name: Wait for master API to come back online + wait_for: + host: "{{ openshift.common.hostname }}" + state: started + delay: 10 + port: "{{ openshift.master.api_port }}" + timeout: 600 + when: inventory_hostname in groups.oo_masters_to_config diff --git a/roles/openshift_node_upgrade/tasks/rpm_upgrade.yml b/roles/openshift_node_upgrade/tasks/rpm_upgrade.yml new file mode 100644 index 000000000..a998acf21 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/rpm_upgrade.yml @@ -0,0 +1,29 @@ +--- +# input variables: +# - openshift.common.service_type +# - component +# - openshift_pkg_version +# - openshift.common.is_atomic + +# We verified latest rpm available is suitable, so just yum update. +- name: Upgrade packages + package: "name={{ openshift.common.service_type }}-{{ component }}{{ openshift_pkg_version }} state=present" + +- name: Ensure python-yaml present for config upgrade + package: name=PyYAML state=present + when: not openshift.common.is_atomic | bool + +- name: Install Node service file + template: + dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node.service" + src: "node.service.j2" + register: l_node_unit + +# NOTE: This is needed to make sure we are using the correct set +# of systemd unit files. The RPMs lay down defaults but +# the install/upgrade may override them in /etc/systemd/system/. +# NOTE: We don't use the systemd module as some versions of the module +# require a service to be part of the call. +- name: Reload systemd units + command: systemctl daemon-reload + when: l_node_unit | changed diff --git a/roles/openshift_node_upgrade/tasks/systemd_units.yml b/roles/openshift_node_upgrade/tasks/systemd_units.yml new file mode 100644 index 000000000..226f5290c --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/systemd_units.yml @@ -0,0 +1,37 @@ +--- +# input variables +# - openshift.node.node_image +# - openshift_image_tag +# - openshift.common.is_containerized +# - openshift.node.ovs_image +# - openshift_use_openshift_sdn +# - openshift.common.service_type +# - openshift_node_debug_level +# - openshift.common.config_base +# - openshift.common.http_proxy +# - openshift.common.portal_net +# - openshift.common +# - openshift.common.http_proxy +# notify: +# - restart openvswitch +# - restart node + +# This file is included both in the openshift_master role and in the upgrade +# playbooks. +- include: config/install-node-deps-docker-service-file.yml + when: openshift.common.is_containerized | bool + +- include: config/install-node-docker-service-file.yml + when: openshift.common.is_containerized | bool + +- include: config/install-ovs-service-env-file.yml + when: openshift.common.is_containerized | bool + +- include: config/workaround-bz1331590-ovs-oom-fix.yml + when: openshift_use_openshift_sdn | bool + +- include: config/install-ovs-docker-service-file.yml + when: openshift.common.is_containerized | bool and openshift_use_openshift_sdn | bool + +- include: config/configure-node-settings.yml +- include: config/configure-proxy-settings.yml |