diff options
Diffstat (limited to 'roles/openshift_node_upgrade')
-rw-r--r-- | roles/openshift_node_upgrade/README.md | 6 | ||||
-rw-r--r-- | roles/openshift_node_upgrade/defaults/main.yml | 1 | ||||
-rw-r--r-- | roles/openshift_node_upgrade/handlers/main.yml | 25 | ||||
-rw-r--r-- | roles/openshift_node_upgrade/meta/main.yml | 1 | ||||
-rw-r--r-- | roles/openshift_node_upgrade/tasks/docker/upgrade.yml | 25 | ||||
-rw-r--r-- | roles/openshift_node_upgrade/tasks/main.yml | 114 | ||||
-rw-r--r-- | roles/openshift_node_upgrade/tasks/restart.yml (renamed from roles/openshift_node_upgrade/tasks/docker/restart.yml) | 20 | ||||
-rw-r--r-- | roles/openshift_node_upgrade/tasks/rpm_upgrade.yml | 15 | ||||
-rw-r--r-- | roles/openshift_node_upgrade/tasks/systemd_units.yml | 15 | ||||
-rw-r--r-- | roles/openshift_node_upgrade/templates/node.service.j2 | 31 | ||||
-rw-r--r-- | roles/openshift_node_upgrade/templates/openshift.docker.node.dep.service | 4 | ||||
-rw-r--r-- | roles/openshift_node_upgrade/templates/openshift.docker.node.service | 20 | ||||
-rw-r--r-- | roles/openshift_node_upgrade/templates/openvswitch.docker.service | 8 |
13 files changed, 215 insertions, 70 deletions
diff --git a/roles/openshift_node_upgrade/README.md b/roles/openshift_node_upgrade/README.md index 66bce38ec..4e6229bfb 100644 --- a/roles/openshift_node_upgrade/README.md +++ b/roles/openshift_node_upgrade/README.md @@ -84,10 +84,16 @@ Including an example of how to use your role (for instance, with variables passe command: > {{ hostvars[groups.oo_first_master.0].openshift.common.admin_binary }} drain {{ openshift.node.nodename | lower }} --force --delete-local-data --ignore-daemonsets delegate_to: "{{ groups.oo_first_master.0 }}" + register: l_docker_upgrade_drain_result + until: not l_docker_upgrade_drain_result | failed + retries: 60 + delay: 60 + roles: - openshift_facts - docker + - openshift_node_dnsmasq - openshift_node_upgrade post_tasks: diff --git a/roles/openshift_node_upgrade/defaults/main.yml b/roles/openshift_node_upgrade/defaults/main.yml new file mode 100644 index 000000000..ed97d539c --- /dev/null +++ b/roles/openshift_node_upgrade/defaults/main.yml @@ -0,0 +1 @@ +--- diff --git a/roles/openshift_node_upgrade/handlers/main.yml b/roles/openshift_node_upgrade/handlers/main.yml index cb51416d4..f26f5d573 100644 --- a/roles/openshift_node_upgrade/handlers/main.yml +++ b/roles/openshift_node_upgrade/handlers/main.yml @@ -1,7 +1,16 @@ --- - name: restart openvswitch - systemd: name=openvswitch state=restarted - when: (not skip_node_svc_handlers | default(False) | bool) and not (ovs_service_status_changed | default(false) | bool) and openshift.common.use_openshift_sdn | bool + systemd: + name: openvswitch + state: restarted + when: + - not skip_node_svc_handlers | default(False) | bool + - not (ovs_service_status_changed | default(false) | bool) + - openshift.common.use_openshift_sdn | default(true) | bool + register: l_openshift_node_upgrade_stop_openvswitch_result + until: not l_openshift_node_upgrade_stop_openvswitch_result | failed + retries: 3 + delay: 30 notify: - restart openvswitch pause @@ -10,5 +19,13 @@ when: (not skip_node_svc_handlers | default(False) | bool) and openshift.common.is_containerized | bool - name: restart node - systemd: name={{ openshift.common.service_type }}-node state=restarted - when: (not skip_node_svc_handlers | default(False) | bool) and not (node_service_status_changed | default(false) | bool) + systemd: + name: "{{ openshift.common.service_type }}-node" + state: restarted + register: l_openshift_node_upgrade_restart_node_result + until: not l_openshift_node_upgrade_restart_node_result | failed + retries: 3 + delay: 30 + when: + - (not skip_node_svc_handlers | default(False) | bool) + - not (node_service_status_changed | default(false) | bool) diff --git a/roles/openshift_node_upgrade/meta/main.yml b/roles/openshift_node_upgrade/meta/main.yml index cd2f362aa..2a36d8945 100644 --- a/roles/openshift_node_upgrade/meta/main.yml +++ b/roles/openshift_node_upgrade/meta/main.yml @@ -10,4 +10,5 @@ galaxy_info: versions: - 7 dependencies: +- role: lib_utils - role: openshift_common diff --git a/roles/openshift_node_upgrade/tasks/docker/upgrade.yml b/roles/openshift_node_upgrade/tasks/docker/upgrade.yml index e91891ca9..ebe87d6fd 100644 --- a/roles/openshift_node_upgrade/tasks/docker/upgrade.yml +++ b/roles/openshift_node_upgrade/tasks/docker/upgrade.yml @@ -6,20 +6,6 @@ # - docker_version # - skip_docker_restart -# We need docker service up to remove all the images, but these services will keep -# trying to re-start and thus re-pull the images we're trying to delete. -- name: Stop containerized services - service: name={{ item }} state=stopped - with_items: - - "{{ openshift.common.service_type }}-master" - - "{{ openshift.common.service_type }}-master-api" - - "{{ openshift.common.service_type }}-master-controllers" - - "{{ openshift.common.service_type }}-node" - - etcd_container - - openvswitch - failed_when: false - when: openshift.common.is_containerized | bool - - name: Check Docker image count shell: "docker images -aq | wc -l" register: docker_image_count @@ -40,10 +26,15 @@ - debug: var=docker_image_count.stdout when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool -- service: name=docker state=stopped +- service: + name: docker + state: stopped + register: l_openshift_node_upgrade_docker_stop_result + until: not l_openshift_node_upgrade_docker_stop_result | failed + retries: 3 + delay: 30 - name: Upgrade Docker package: name=docker{{ '-' + docker_version }} state=present -- include: restart.yml - when: not skip_docker_restart | default(False) | bool +# starting docker happens back in ../main.yml where it calls ../restart.yml diff --git a/roles/openshift_node_upgrade/tasks/main.yml b/roles/openshift_node_upgrade/tasks/main.yml index 6ae8dbc12..f984a04b2 100644 --- a/roles/openshift_node_upgrade/tasks/main.yml +++ b/roles/openshift_node_upgrade/tasks/main.yml @@ -9,6 +9,44 @@ # - openshift_release # tasks file for openshift_node_upgrade + +- name: Stop node and openvswitch services + service: + name: "{{ item }}" + state: stopped + with_items: + - "{{ openshift.common.service_type }}-node" + - openvswitch + failed_when: false + +- name: Stop additional containerized services + service: + name: "{{ item }}" + state: stopped + with_items: + - "{{ openshift.common.service_type }}-master" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-master-api" + - etcd_container + failed_when: false + when: openshift.common.is_containerized | bool + +- name: Pre-pull node image + command: > + docker pull {{ openshift.node.node_image }}:{{ openshift_image_tag }} + register: pull_result + changed_when: "'Downloaded newer image' in pull_result.stdout" + when: openshift.common.is_containerized | bool + +- name: Pre-pull openvswitch image + command: > + docker pull {{ openshift.node.ovs_image }}:{{ openshift_image_tag }} + register: pull_result + changed_when: "'Downloaded newer image' in pull_result.stdout" + when: + - openshift.common.is_containerized | bool + - openshift.common.use_openshift_sdn | default(true) | bool + - include: docker/upgrade.yml vars: # We will restart Docker ourselves after everything is ready: @@ -16,7 +54,6 @@ when: - l_docker_upgrade is defined - l_docker_upgrade | bool - - not openshift.common.is_containerized | bool - include: "{{ node_config_hook }}" when: node_config_hook is defined @@ -67,28 +104,64 @@ state: latest when: not openshift.common.is_containerized | bool -- name: Restart openvswitch - systemd: - name: openvswitch - state: started - when: - - not openshift.common.is_containerized | bool - -# Mandatory Docker restart, ensure all containerized services are running: -- include: docker/restart.yml - - name: Update oreg value yedit: src: "{{ openshift.common.config_base }}/node/node-config.yaml" key: 'imageConfig.format' - value: "{{ oreg_url }}" - when: oreg_url is defined + value: "{{ oreg_url | default(oreg_url_node) }}" + when: oreg_url is defined or oreg_url_node is defined -- name: Restart rpm node service - service: - name: "{{ openshift.common.service_type }}-node" - state: restarted - when: not openshift.common.is_containerized | bool +# https://docs.openshift.com/container-platform/3.4/admin_guide/overcommit.html#disabling-swap-memory +- name: Check for swap usage + command: grep "^[^#].*swap" /etc/fstab + # grep: match any lines which don't begin with '#' and contain 'swap' + changed_when: false + failed_when: false + register: swap_result + + # Disable Swap Block +- block: + + - name: Disable swap + command: swapoff --all + + - name: Remove swap entries from /etc/fstab + replace: + dest: /etc/fstab + regexp: '(^[^#].*swap.*)' + replace: '# \1' + backup: yes + + - name: Add notice about disabling swap + lineinfile: + dest: /etc/fstab + line: '# OpenShift-Ansible Installer disabled swap per overcommit guidelines' + state: present + + when: + - swap_result.stdout_lines | length > 0 + - openshift_disable_swap | default(true) | bool + # End Disable Swap Block + +- name: Reset selinux context + command: restorecon -RF {{ openshift.common.data_dir }}/openshift.local.volumes + when: + - ansible_selinux is defined + - ansible_selinux.status == 'enabled' + +- name: Apply 3.6 dns config changes + yedit: + src: /etc/origin/node/node-config.yaml + key: "{{ item.key }}" + value: "{{ item.value }}" + with_items: + - key: "dnsBindAddress" + value: "127.0.0.1:53" + - key: "dnsRecursiveResolvConf" + value: "/etc/origin/node/resolv.conf" + +# Restart all services +- include: restart.yml - name: Wait for node to be ready oc_obj: @@ -97,7 +170,10 @@ name: "{{ openshift.common.hostname | lower }}" register: node_output delegate_to: "{{ groups.oo_first_master.0 }}" - until: node_output.results.results[0].status.conditions | selectattr('type', 'match', '^Ready$') | map(attribute='status') | join | bool == True + until: node_output.results.returncode == 0 and node_output.results.results[0].status.conditions | selectattr('type', 'match', '^Ready$') | map(attribute='status') | join | bool == True # Give the node two minutes to come back online. retries: 24 delay: 5 + +- include_role: + name: openshift_node_dnsmasq diff --git a/roles/openshift_node_upgrade/tasks/docker/restart.yml b/roles/openshift_node_upgrade/tasks/restart.yml index 176fc3c0b..f228b6e08 100644 --- a/roles/openshift_node_upgrade/tasks/docker/restart.yml +++ b/roles/openshift_node_upgrade/tasks/restart.yml @@ -5,14 +5,28 @@ # - openshift.common.hostname # - openshift.master.api_port +# NOTE: This is needed to make sure we are using the correct set +# of systemd unit files. The RPMs lay down defaults but +# the install/upgrade may override them in /etc/systemd/system/. +# NOTE: We don't use the systemd module as some versions of the module +# require a service to be part of the call. +- name: Reload systemd to ensure latest unit files + command: systemctl daemon-reload + - name: Restart docker - service: name=docker state=restarted + service: + name: "{{ openshift.docker.service_name }}" + state: started + register: docker_start_result + until: not docker_start_result | failed + retries: 3 + delay: 30 - name: Update docker facts openshift_facts: role: docker -- name: Restart containerized services +- name: Start services service: name={{ item }} state=started with_items: - etcd_container @@ -22,7 +36,6 @@ - "{{ openshift.common.service_type }}-master-controllers" - "{{ openshift.common.service_type }}-node" failed_when: false - when: openshift.common.is_containerized | bool - name: Wait for master API to come back online wait_for: @@ -30,4 +43,5 @@ state: started delay: 10 port: "{{ openshift.master.api_port }}" + timeout: 600 when: inventory_hostname in groups.oo_masters_to_config diff --git a/roles/openshift_node_upgrade/tasks/rpm_upgrade.yml b/roles/openshift_node_upgrade/tasks/rpm_upgrade.yml index 480e87d58..a998acf21 100644 --- a/roles/openshift_node_upgrade/tasks/rpm_upgrade.yml +++ b/roles/openshift_node_upgrade/tasks/rpm_upgrade.yml @@ -12,3 +12,18 @@ - name: Ensure python-yaml present for config upgrade package: name=PyYAML state=present when: not openshift.common.is_atomic | bool + +- name: Install Node service file + template: + dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node.service" + src: "node.service.j2" + register: l_node_unit + +# NOTE: This is needed to make sure we are using the correct set +# of systemd unit files. The RPMs lay down defaults but +# the install/upgrade may override them in /etc/systemd/system/. +# NOTE: We don't use the systemd module as some versions of the module +# require a service to be part of the call. +- name: Reload systemd units + command: systemctl daemon-reload + when: l_node_unit | changed diff --git a/roles/openshift_node_upgrade/tasks/systemd_units.yml b/roles/openshift_node_upgrade/tasks/systemd_units.yml index 862cd19c4..e8f017445 100644 --- a/roles/openshift_node_upgrade/tasks/systemd_units.yml +++ b/roles/openshift_node_upgrade/tasks/systemd_units.yml @@ -18,21 +18,6 @@ # This file is included both in the openshift_master role and in the upgrade # playbooks. - -- name: Pre-pull node image - command: > - docker pull {{ openshift.node.node_image }}:{{ openshift_image_tag }} - register: pull_result - changed_when: "'Downloaded newer image' in pull_result.stdout" - when: openshift.common.is_containerized | bool - -- name: Pre-pull openvswitch image - command: > - docker pull {{ openshift.node.ovs_image }}:{{ openshift_image_tag }} - register: pull_result - changed_when: "'Downloaded newer image' in pull_result.stdout" - when: openshift.common.is_containerized | bool and openshift.common.use_openshift_sdn | bool - - name: Install Node dependencies docker service file template: dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node-dep.service" diff --git a/roles/openshift_node_upgrade/templates/node.service.j2 b/roles/openshift_node_upgrade/templates/node.service.j2 new file mode 100644 index 000000000..e12a52c15 --- /dev/null +++ b/roles/openshift_node_upgrade/templates/node.service.j2 @@ -0,0 +1,31 @@ +[Unit] +Description=OpenShift Node +After={{ openshift.docker.service_name }}.service +Wants=openvswitch.service +After=ovsdb-server.service +After=ovs-vswitchd.service +Wants={{ openshift.docker.service_name }}.service +Documentation=https://github.com/openshift/origin +Requires=dnsmasq.service +After=dnsmasq.service + +[Service] +Type=notify +EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node +Environment=GOTRACEBACK=crash +ExecStartPre=/usr/bin/cp /etc/origin/node/node-dnsmasq.conf /etc/dnsmasq.d/ +ExecStartPre=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:/in-addr.arpa/127.0.0.1,/{{ openshift.common.dns_domain }}/127.0.0.1 +ExecStopPost=/usr/bin/rm /etc/dnsmasq.d/node-dnsmasq.conf +ExecStopPost=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string: +ExecStart=/usr/bin/openshift start node --config=${CONFIG_FILE} $OPTIONS +LimitNOFILE=65536 +LimitCORE=infinity +WorkingDirectory=/var/lib/origin/ +SyslogIdentifier={{ openshift.common.service_type }}-node +Restart=always +RestartSec=5s +TimeoutStartSec=300 +OOMScoreAdjust=-999 + +[Install] +WantedBy=multi-user.target diff --git a/roles/openshift_node_upgrade/templates/openshift.docker.node.dep.service b/roles/openshift_node_upgrade/templates/openshift.docker.node.dep.service index 0fb34cffd..4c47f8c0d 100644 --- a/roles/openshift_node_upgrade/templates/openshift.docker.node.dep.service +++ b/roles/openshift_node_upgrade/templates/openshift.docker.node.dep.service @@ -1,6 +1,6 @@ [Unit] -Requires=docker.service -After=docker.service +Requires={{ openshift.docker.service_name }}.service +After={{ openshift.docker.service_name }}.service PartOf={{ openshift.common.service_type }}-node.service Before={{ openshift.common.service_type }}-node.service diff --git a/roles/openshift_node_upgrade/templates/openshift.docker.node.service b/roles/openshift_node_upgrade/templates/openshift.docker.node.service index 0ff398152..639b6f6c8 100644 --- a/roles/openshift_node_upgrade/templates/openshift.docker.node.service +++ b/roles/openshift_node_upgrade/templates/openshift.docker.node.service @@ -1,26 +1,34 @@ [Unit] After={{ openshift.common.service_type }}-master.service -After=docker.service +After={{ openshift.docker.service_name }}.service After=openvswitch.service -PartOf=docker.service -Requires=docker.service +PartOf={{ openshift.docker.service_name }}.service +Requires={{ openshift.docker.service_name }}.service {% if openshift.common.use_openshift_sdn %} -Requires=openvswitch.service +Wants=openvswitch.service +After=ovsdb-server.service +After=ovs-vswitchd.service {% endif %} Wants={{ openshift.common.service_type }}-master.service Requires={{ openshift.common.service_type }}-node-dep.service After={{ openshift.common.service_type }}-node-dep.service +Requires=dnsmasq.service +After=dnsmasq.service [Service] EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node-dep ExecStartPre=-/usr/bin/docker rm -f {{ openshift.common.service_type }}-node -ExecStart=/usr/bin/docker run --name {{ openshift.common.service_type }}-node --rm --privileged --net=host --pid=host --env-file=/etc/sysconfig/{{ openshift.common.service_type }}-node -v /:/rootfs:ro,rslave -e CONFIG_FILE=${CONFIG_FILE} -e OPTIONS=${OPTIONS} -e HOST=/rootfs -e HOST_ETC=/host-etc -v {{ openshift.common.data_dir }}:{{ openshift.common.data_dir }}{{ ':rslave' if openshift.docker.gte_1_10 | default(False) | bool else '' }} -v {{ openshift.common.config_base }}/node:{{ openshift.common.config_base }}/node {% if openshift_cloudprovider_kind | default('') != '' -%} -v {{ openshift.common.config_base }}/cloudprovider:{{ openshift.common.config_base}}/cloudprovider {% endif -%} -v /etc/localtime:/etc/localtime:ro -v /etc/machine-id:/etc/machine-id:ro -v /run:/run -v /sys:/sys:rw -v /sys/fs/cgroup:/sys/fs/cgroup:rw -v /usr/bin/docker:/usr/bin/docker:ro -v /var/lib/docker:/var/lib/docker -v /lib/modules:/lib/modules -v /etc/origin/openvswitch:/etc/openvswitch -v /etc/origin/sdn:/etc/openshift-sdn -v /var/lib/cni:/var/lib/cni -v /etc/systemd/system:/host-etc/systemd/system -v /var/log:/var/log -v /dev:/dev $DOCKER_ADDTL_BIND_MOUNTS {{ openshift.node.node_image }}:${IMAGE_VERSION} +ExecStartPre=/usr/bin/cp /etc/origin/node/node-dnsmasq.conf /etc/dnsmasq.d/ +ExecStartPre=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:/in-addr.arpa/127.0.0.1,/{{ openshift.common.dns_domain }}/127.0.0.1 +ExecStart=/usr/bin/docker run --name {{ openshift.common.service_type }}-node --rm --privileged --net=host --pid=host --env-file=/etc/sysconfig/{{ openshift.common.service_type }}-node -v /:/rootfs:ro,rslave -e CONFIG_FILE=${CONFIG_FILE} -e OPTIONS=${OPTIONS} -e HOST=/rootfs -e HOST_ETC=/host-etc -v {{ openshift.common.data_dir }}:{{ openshift.common.data_dir }}{{ ':rslave' if openshift.docker.gte_1_10 | default(False) | bool else '' }} -v {{ openshift.common.config_base }}/node:{{ openshift.common.config_base }}/node {% if openshift_cloudprovider_kind | default('') != '' -%} -v {{ openshift.common.config_base }}/cloudprovider:{{ openshift.common.config_base}}/cloudprovider {% endif -%} -v /etc/localtime:/etc/localtime:ro -v /etc/machine-id:/etc/machine-id:ro -v /run:/run -v /sys:/sys:rw -v /sys/fs/cgroup:/sys/fs/cgroup:rw -v /usr/bin/docker:/usr/bin/docker:ro -v /var/lib/docker:/var/lib/docker -v /lib/modules:/lib/modules -v /etc/origin/openvswitch:/etc/openvswitch -v /etc/origin/sdn:/etc/openshift-sdn -v /var/lib/cni:/var/lib/cni -v /etc/systemd/system:/host-etc/systemd/system -v /var/log:/var/log -v /dev:/dev $DOCKER_ADDTL_BIND_MOUNTS -v /etc/pki:/etc/pki:ro {{ openshift.node.node_image }}:${IMAGE_VERSION} ExecStartPost=/usr/bin/sleep 10 ExecStop=/usr/bin/docker stop {{ openshift.common.service_type }}-node +ExecStopPost=/usr/bin/rm /etc/dnsmasq.d/node-dnsmasq.conf +ExecStopPost=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string: SyslogIdentifier={{ openshift.common.service_type }}-node Restart=always RestartSec=5s [Install] -WantedBy=docker.service +WantedBy={{ openshift.docker.service_name }}.service diff --git a/roles/openshift_node_upgrade/templates/openvswitch.docker.service b/roles/openshift_node_upgrade/templates/openvswitch.docker.service index 1e1f8967d..34aaaabd6 100644 --- a/roles/openshift_node_upgrade/templates/openvswitch.docker.service +++ b/roles/openshift_node_upgrade/templates/openvswitch.docker.service @@ -1,7 +1,7 @@ [Unit] -After=docker.service -Requires=docker.service -PartOf=docker.service +After={{ openshift.docker.service_name }}.service +Requires={{ openshift.docker.service_name }}.service +PartOf={{ openshift.docker.service_name }}.service [Service] EnvironmentFile=/etc/sysconfig/openvswitch @@ -14,4 +14,4 @@ Restart=always RestartSec=5s [Install] -WantedBy=docker.service +WantedBy={{ openshift.docker.service_name }}.service |