diff options
Diffstat (limited to 'roles/openshift_node')
43 files changed, 911 insertions, 169 deletions
diff --git a/roles/openshift_node/README.md b/roles/openshift_node/README.md index 67f697924..87ceb8103 100644 --- a/roles/openshift_node/README.md +++ b/roles/openshift_node/README.md @@ -33,9 +33,9 @@ Notes Currently we support re-labeling nodes but we don't re-schedule running pods nor remove existing labels. That means you will have to trigger the re-schedulling manually. To re-schedule your pods, just follow the steps below: ``` -oadm manage-node --schedulable=false ${NODE} -oadm manage-node --drain ${NODE} -oadm manage-node --schedulable=true ${NODE} +oc adm manage-node --schedulable=false ${NODE} +oc adm manage-node --drain ${NODE} +oc adm manage-node --schedulable=true ${NODE} ```` > If you are using version less than 1.5/3.5 you must replace `--drain` with `--evacuate`. diff --git a/roles/openshift_node/defaults/main.yml b/roles/openshift_node/defaults/main.yml index b9f16dfd4..c1fab4382 100644 --- a/roles/openshift_node/defaults/main.yml +++ b/roles/openshift_node/defaults/main.yml @@ -1,15 +1,98 @@ --- openshift_node_debug_level: "{{ debug_level | default(2) }}" +openshift_node_iptables_sync_period: '30s' +osn_storage_plugin_deps: +- ceph +- glusterfs +- iscsi +openshift_node_local_quota_per_fsgroup: "" +openshift_node_proxy_mode: iptables +openshift_set_node_ip: False +openshift_config_base: '/etc/origin' + +openshift_oreg_url_default_dict: + origin: "openshift/origin-${component}:${version}" + openshift-enterprise: "openshift3/ose-${component}:${version}" +openshift_oreg_url_default: "{{ openshift_oreg_url_default_dict[openshift_deployment_type] }}" +oreg_url_node: "{{ oreg_url | default(openshift_oreg_url_default) }}" + +osn_ovs_image_default_dict: + origin: "openshift/openvswitch" + openshift-enterprise: "openshift3/openvswitch" +osn_ovs_image_default: "{{ osn_ovs_image_default_dict[openshift_deployment_type] }}" +osn_ovs_image: "{{ osn_ovs_image_default }}" + +openshift_dns_ip: "{{ ansible_default_ipv4['address'] }}" + +openshift_node_env_vars: {} + +# Create list of 'k=v' pairs. +l_node_kubelet_node_labels: "{{ openshift_node_labels | default({}) | lib_utils_oo_dict_to_keqv_list }}" + +openshift_node_kubelet_args_dict: + aws: + cloud-provider: + - aws + cloud-config: + - "{{ openshift_config_base ~ '/cloudprovider/aws.conf' }}" + node-labels: "{{ l_node_kubelet_node_labels }}" + openstack: + cloud-provider: + - openstack + cloud-config: + - "{{ openshift_config_base ~ '/cloudprovider/openstack.conf' }}" + node-labels: "{{ l_node_kubelet_node_labels }}" + gce: + cloud-provider: + - gce + cloud-config: + - "{{ openshift_config_base ~ '/cloudprovider/gce.conf' }}" + node-labels: "{{ l_node_kubelet_node_labels }}" + undefined: + node-labels: "{{ l_node_kubelet_node_labels }}" + +l_node_kubelet_args_default: "{{ openshift_node_kubelet_args_dict[openshift_cloudprovider_kind | default('undefined')] }}" + +l_openshift_node_kubelet_args: "{{ openshift_node_kubelet_args | default({}) }}" +# Combine the default kubelet_args dictionary (based on cloud provider, if provided) +# with user-supplied openshift_node_kubelet_args. +# openshift_node_kubelet_args will override the defaults, if keys and/or subkeys +# are present in both. +l2_openshift_node_kubelet_args: "{{ l_node_kubelet_args_default | combine(l_openshift_node_kubelet_args, recursive=True) }}" +openshift_node_dnsmasq_install_network_manager_hook: true + +# lo must always be present in this list or dnsmasq will conflict with +# the node's dns service. +openshift_node_dnsmasq_except_interfaces: +- lo r_openshift_node_firewall_enabled: "{{ os_firewall_enabled | default(True) }}" r_openshift_node_use_firewalld: "{{ os_firewall_use_firewalld | default(False) }}" +l_is_node_system_container: "{{ (openshift_use_node_system_container | default(openshift_use_system_containers | default(false)) | bool) }}" + openshift_deployment_type: "{{ openshift_deployment_type | default('origin') }}" -openshift_service_type: "{{ 'origin' if openshift_deployment_type == 'origin' else 'atomic-openshift' }}" + +openshift_node_image_dict: + origin: 'openshift/node' + openshift-enterprise: 'openshift3/node' +osn_image: "{{ openshift_node_image_dict[openshift_deployment_type] }}" + +openshift_service_type_dict: + origin: origin + openshift-enterprise: atomic-openshift +openshift_service_type: "{{ openshift_service_type_dict[openshift_deployment_type] }}" + +system_images_registry_dict: + openshift-enterprise: "registry.access.redhat.com" + origin: "docker.io" + +system_images_registry: "{{ system_images_registry_dict[openshift_deployment_type | default('origin')] }}" +l_is_openvswitch_system_container: "{{ (openshift_use_openvswitch_system_container | default(openshift_use_system_containers | default(false)) | bool) }}" openshift_image_tag: '' -openshift_node_ami_prep_packages: +default_r_openshift_node_image_prep_packages: - "{{ openshift_service_type }}-master" - "{{ openshift_service_type }}-node" - "{{ openshift_service_type }}-docker-excluder" @@ -33,7 +116,6 @@ openshift_node_ami_prep_packages: - python-dbus - PyYAML - yum-utils -- cloud-utils-growpart # gluster - glusterfs-fuse # nfs @@ -54,6 +136,7 @@ openshift_node_ami_prep_packages: # - container-selinux # - atomic # +r_openshift_node_image_prep_packages: "{{ default_r_openshift_node_image_prep_packages | union(openshift_node_image_prep_packages | default([])) }}" openshift_node_bootstrap: False @@ -85,6 +168,10 @@ oreg_host: "{{ oreg_url.split('/')[0] if (oreg_url is defined and '.' in oreg_ur oreg_auth_credentials_path: "{{ openshift_node_data_dir }}/.docker" oreg_auth_credentials_replace: False l_bind_docker_reg_auth: False +openshift_use_crio: False +openshift_docker_alternative_creds: "{{ (openshift_docker_use_system_container | default(False) | bool) or (openshift_use_crio_only | default(False) | bool) }}" + +openshift_docker_service_name: "{{ 'container-engine' if (openshift_docker_use_system_container | default(False) | bool) else 'docker' }}" # NOTE # r_openshift_node_*_default may be defined external to this role. @@ -110,5 +197,10 @@ openshift_node_use_kuryr: "{{ openshift_node_use_kuryr_default }}" openshift_node_data_dir_default: "{{ openshift_data_dir | default('/var/lib/origin') }}" openshift_node_data_dir: "{{ openshift_node_data_dir_default }}" +openshift_node_config_dir_default: "/etc/origin/node" +openshift_node_config_dir: "{{ openshift_node_config_dir_default }}" + openshift_node_image_config_latest_default: "{{ openshift_image_config_latest | default(False) }}" openshift_node_image_config_latest: "{{ openshift_node_image_config_latest_default }}" + +openshift_node_use_instance_profiles: False diff --git a/roles/openshift_node/files/bootstrap.yml b/roles/openshift_node/files/bootstrap.yml index ea280640f..a5545c81b 100644 --- a/roles/openshift_node/files/bootstrap.yml +++ b/roles/openshift_node/files/bootstrap.yml @@ -61,3 +61,11 @@ with_items: - line: "BOOTSTRAP_CONFIG_NAME=node-config-{{ openshift_group_type }}" regexp: "^BOOTSTRAP_CONFIG_NAME=.*" + + - name: "Start the {{ openshift_service_type }}-node service" + systemd: + daemon_reload: yes + state: restarted + enabled: True + name: "{{ openshift_service_type }}-node" + no_block: true diff --git a/roles/openshift_node/files/networkmanager/99-origin-dns.sh b/roles/openshift_node/files/networkmanager/99-origin-dns.sh new file mode 100755 index 000000000..f4e48b5b7 --- /dev/null +++ b/roles/openshift_node/files/networkmanager/99-origin-dns.sh @@ -0,0 +1,128 @@ +#!/bin/bash -x +# -*- mode: sh; sh-indentation: 2 -*- + +# This NetworkManager dispatcher script replicates the functionality of +# NetworkManager's dns=dnsmasq however, rather than hardcoding the listening +# address and /etc/resolv.conf to 127.0.0.1 it pulls the IP address from the +# interface that owns the default route. This enables us to then configure pods +# to use this IP address as their only resolver, where as using 127.0.0.1 inside +# a pod would fail. +# +# To use this, +# - If this host is also a master, reconfigure master dnsConfig to listen on +# 8053 to avoid conflicts on port 53 and open port 8053 in the firewall +# - Drop this script in /etc/NetworkManager/dispatcher.d/ +# - systemctl restart NetworkManager +# - Configure node-config.yaml to set dnsIP: to the ip address of this +# node +# +# Test it: +# host kubernetes.default.svc.cluster.local +# host google.com +# +# TODO: I think this would be easy to add as a config option in NetworkManager +# natively, look at hacking that up + +cd /etc/sysconfig/network-scripts +. ./network-functions + +[ -f ../network ] && . ../network + +if [[ $2 =~ ^(up|dhcp4-change|dhcp6-change)$ ]]; then + # If the origin-upstream-dns config file changed we need to restart + NEEDS_RESTART=0 + UPSTREAM_DNS='/etc/dnsmasq.d/origin-upstream-dns.conf' + # We'll regenerate the dnsmasq origin config in a temp file first + UPSTREAM_DNS_TMP=`mktemp` + UPSTREAM_DNS_TMP_SORTED=`mktemp` + CURRENT_UPSTREAM_DNS_SORTED=`mktemp` + NEW_RESOLV_CONF=`mktemp` + NEW_NODE_RESOLV_CONF=`mktemp` + + + ###################################################################### + # couldn't find an existing method to determine if the interface owns the + # default route + def_route=$(/sbin/ip route list match 0.0.0.0/0 | awk '{print $3 }') + def_route_int=$(/sbin/ip route get to ${def_route} | awk '{print $3}') + def_route_ip=$(/sbin/ip route get to ${def_route} | awk '{print $5}') + if [[ ${DEVICE_IFACE} == ${def_route_int} ]]; then + if [ ! -f /etc/dnsmasq.d/origin-dns.conf ]; then + cat << EOF > /etc/dnsmasq.d/origin-dns.conf +no-resolv +domain-needed +server=/cluster.local/172.30.0.1 +server=/30.172.in-addr.arpa/172.30.0.1 +enable-dbus +dns-forward-max=5000 +cache-size=5000 +EOF + # New config file, must restart + NEEDS_RESTART=1 + fi + + # If network manager doesn't know about the nameservers then the best + # we can do is grab them from /etc/resolv.conf but only if we've got no + # watermark + if ! grep -q '99-origin-dns.sh' /etc/resolv.conf; then + if [[ -z "${IP4_NAMESERVERS}" || "${IP4_NAMESERVERS}" == "${def_route_ip}" ]]; then + IP4_NAMESERVERS=`grep '^nameserver ' /etc/resolv.conf | awk '{ print $2 }'` + fi + ###################################################################### + # Write out default nameservers for /etc/dnsmasq.d/origin-upstream-dns.conf + # and /etc/origin/node/resolv.conf in their respective formats + for ns in ${IP4_NAMESERVERS}; do + if [[ ! -z $ns ]]; then + echo "server=${ns}" >> $UPSTREAM_DNS_TMP + echo "nameserver ${ns}" >> $NEW_NODE_RESOLV_CONF + fi + done + # Sort it in case DNS servers arrived in a different order + sort $UPSTREAM_DNS_TMP > $UPSTREAM_DNS_TMP_SORTED + sort $UPSTREAM_DNS > $CURRENT_UPSTREAM_DNS_SORTED + # Compare to the current config file (sorted) + NEW_DNS_SUM=`md5sum ${UPSTREAM_DNS_TMP_SORTED} | awk '{print $1}'` + CURRENT_DNS_SUM=`md5sum ${CURRENT_UPSTREAM_DNS_SORTED} | awk '{print $1}'` + if [ "${NEW_DNS_SUM}" != "${CURRENT_DNS_SUM}" ]; then + # DNS has changed, copy the temp file to the proper location (-Z + # sets default selinux context) and set the restart flag + cp -Z $UPSTREAM_DNS_TMP $UPSTREAM_DNS + NEEDS_RESTART=1 + fi + # compare /etc/origin/node/resolv.conf checksum and replace it if different + NEW_NODE_RESOLV_CONF_MD5=`md5sum ${NEW_NODE_RESOLV_CONF}` + OLD_NODE_RESOLV_CONF_MD5=`md5sum /etc/origin/node/resolv.conf` + if [ "${NEW_NODE_RESOLV_CONF_MD5}" != "${OLD_NODE_RESOLV_CONF_MD5}" ]; then + cp -Z $NEW_NODE_RESOLV_CONF /etc/origin/node/resolv.conf + fi + fi + + if ! `systemctl -q is-active dnsmasq.service`; then + NEEDS_RESTART=1 + fi + + ###################################################################### + if [ "${NEEDS_RESTART}" -eq "1" ]; then + systemctl restart dnsmasq + fi + + # Only if dnsmasq is running properly make it our only nameserver and place + # a watermark on /etc/resolv.conf + if `systemctl -q is-active dnsmasq.service`; then + if ! grep -q '99-origin-dns.sh' /etc/resolv.conf; then + echo "# nameserver updated by /etc/NetworkManager/dispatcher.d/99-origin-dns.sh" >> ${NEW_RESOLV_CONF} + fi + sed -e '/^nameserver.*$/d' /etc/resolv.conf >> ${NEW_RESOLV_CONF} + echo "nameserver "${def_route_ip}"" >> ${NEW_RESOLV_CONF} + if ! grep -qw search ${NEW_RESOLV_CONF}; then + echo 'search cluster.local' >> ${NEW_RESOLV_CONF} + elif ! grep -q 'search.*cluster.local' ${NEW_RESOLV_CONF}; then + sed -i '/^search/ s/$/ cluster.local/' ${NEW_RESOLV_CONF} + fi + cp -Z ${NEW_RESOLV_CONF} /etc/resolv.conf + fi + fi + + # Clean up after yourself + rm -f $UPSTREAM_DNS_TMP $UPSTREAM_DNS_TMP_SORTED $CURRENT_UPSTREAM_DNS_SORTED $NEW_RESOLV_CONF +fi diff --git a/roles/openshift_node/handlers/main.yml b/roles/openshift_node/handlers/main.yml index b102c1b18..779916335 100644 --- a/roles/openshift_node/handlers/main.yml +++ b/roles/openshift_node/handlers/main.yml @@ -1,4 +1,19 @@ --- +- name: restart NetworkManager + systemd: + name: NetworkManager + state: restarted + enabled: True + when: + - (not skip_node_svc_handlers | default(False) | bool) + +- name: restart dnsmasq + systemd: + name: dnsmasq + state: restarted + when: + - (not skip_node_svc_handlers | default(False) | bool) + - name: restart openvswitch systemd: name: openvswitch @@ -9,7 +24,7 @@ - openshift_node_use_openshift_sdn | bool - not openshift_node_bootstrap register: l_openshift_node_stop_openvswitch_result - until: not l_openshift_node_stop_openvswitch_result | failed + until: not (l_openshift_node_stop_openvswitch_result is failed) retries: 3 delay: 30 notify: @@ -19,14 +34,14 @@ pause: seconds=15 when: - (not skip_node_svc_handlers | default(False) | bool) - - openshift.common.is_containerized | bool + - openshift_is_containerized | bool - name: restart node systemd: - name: "{{ openshift.common.service_type }}-node" + name: "{{ openshift_service_type }}-node" state: restarted register: l_openshift_node_restart_node_result - until: not l_openshift_node_restart_node_result | failed + until: not (l_openshift_node_restart_node_result is failed) retries: 3 delay: 30 when: @@ -36,3 +51,5 @@ - name: reload systemd units command: systemctl daemon-reload + when: + - (not skip_node_svc_handlers | default(False) | bool) diff --git a/roles/openshift_node/meta/main.yml b/roles/openshift_node/meta/main.yml index 5bc7b9869..59e743dce 100644 --- a/roles/openshift_node/meta/main.yml +++ b/roles/openshift_node/meta/main.yml @@ -12,10 +12,5 @@ galaxy_info: categories: - cloud dependencies: -- role: openshift_node_facts - role: lib_openshift -- role: lib_os_firewall -- role: openshift_clock -- role: openshift_docker -- role: openshift_cloud_provider -- role: openshift_node_dnsmasq +- role: lib_utils diff --git a/roles/openshift_node/tasks/aws.yml b/roles/openshift_node/tasks/aws.yml index 38c2b794d..a7f1fc116 100644 --- a/roles/openshift_node/tasks/aws.yml +++ b/roles/openshift_node/tasks/aws.yml @@ -1,7 +1,7 @@ --- - name: Configure AWS Cloud Provider Settings lineinfile: - dest: /etc/sysconfig/{{ openshift.common.service_type }}-node + dest: /etc/sysconfig/{{ openshift_service_type }}-node regexp: "{{ item.regex }}" line: "{{ item.line }}" create: true diff --git a/roles/openshift_node/tasks/bootstrap.yml b/roles/openshift_node/tasks/bootstrap.yml index 8c03f6c41..1a6f209e0 100644 --- a/roles/openshift_node/tasks/bootstrap.yml +++ b/roles/openshift_node/tasks/bootstrap.yml @@ -3,7 +3,9 @@ package: name: "{{ item }}" state: present - with_items: "{{ openshift_node_ami_prep_packages }}" + with_items: "{{ r_openshift_node_image_prep_packages }}" + register: result + until: result is succeeded - name: create the directory for node file: @@ -25,15 +27,15 @@ state: "{{ item.state | default('present') }}" with_items: # add the kubeconfig - - line: "KUBECONFIG=/etc/origin/node/csr_kubeconfig" + - line: "KUBECONFIG={{ openshift_node_config_dir }}/bootstrap.kubeconfig" regexp: "^KUBECONFIG=.*" # remove the config file. This comes from openshift_facts - - regexp: "^CONFIG_FILE=.*" - state: absent + - line: "CONFIG_FILE={{ openshift_node_config_dir }}/node-config.yaml" + regexp: "^CONFIG_FILE=.*" - name: include aws sysconfig credentials - include: aws.yml - static: yes + import_tasks: aws.yml + when: not (openshift_node_use_instance_profiles | default(False)) #- name: update the ExecStart to have bootstrap # lineinfile: @@ -76,7 +78,7 @@ state: link force: yes with_items: - - /var/lib/origin/openshift.local.config/node/node-client-ca.crt + - "{{ openshift_node_config_dir }}/node-client-ca.crt" - when: rpmgenerated_config.stat.exists block: diff --git a/roles/openshift_node/tasks/config.yml b/roles/openshift_node/tasks/config.yml index c08f43118..1103fe4c9 100644 --- a/roles/openshift_node/tasks/config.yml +++ b/roles/openshift_node/tasks/config.yml @@ -1,6 +1,10 @@ --- - name: Install the systemd units - include: systemd_units.yml + include_tasks: systemd_units.yml + +- name: Pull container images + include_tasks: container_images.yml + when: openshift_is_containerized | bool - name: Start and enable openvswitch service systemd: @@ -9,20 +13,20 @@ state: started daemon_reload: yes when: - - openshift.common.is_containerized | bool + - openshift_is_containerized | bool - openshift_node_use_openshift_sdn | default(true) | bool register: ovs_start_result - until: not ovs_start_result | failed + until: not (ovs_start_result is failed) retries: 3 delay: 30 - set_fact: - ovs_service_status_changed: "{{ ovs_start_result | changed }}" + ovs_service_status_changed: "{{ ovs_start_result is changed }}" - file: - dest: "{{ (openshift_node_kubelet_args|default({'config':None})).config}}" + dest: "{{ l2_openshift_node_kubelet_args['config'] }}" state: directory - when: openshift_node_kubelet_args is defined and 'config' in openshift_node_kubelet_args + when: ('config' in l2_openshift_node_kubelet_args) | bool # TODO: add the validate parameter when there is a validation command to run - name: Create the Node config @@ -38,23 +42,23 @@ - name: Configure Node Environment Variables lineinfile: - dest: /etc/sysconfig/{{ openshift.common.service_type }}-node + dest: /etc/sysconfig/{{ openshift_service_type }}-node regexp: "^{{ item.key }}=" line: "{{ item.key }}={{ item.value }}" create: true - with_dict: "{{ openshift.node.env_vars | default({}) }}" + with_dict: "{{ openshift_node_env_vars }}" notify: - restart node - name: include aws provider credentials - include: aws.yml - static: yes + import_tasks: aws.yml + when: not (openshift_node_use_instance_profiles | default(False)) # Necessary because when you're on a node that's also a master the master will be # restarted after the node restarts docker and it will take up to 60 seconds for # systemd to start the master again - when: - - openshift.common.is_containerized | bool + - openshift_is_containerized | bool - not openshift_node_bootstrap block: - name: Wait for master API to become available before proceeding @@ -76,7 +80,7 @@ - name: Start and enable node dep systemd: daemon_reload: yes - name: "{{ openshift.common.service_type }}-node-dep" + name: "{{ openshift_service_type }}-node-dep" enabled: yes state: started @@ -84,24 +88,24 @@ block: - name: Start and enable node systemd: - name: "{{ openshift.common.service_type }}-node" + name: "{{ openshift_service_type }}-node" enabled: yes state: started daemon_reload: yes register: node_start_result - until: not node_start_result | failed + until: not node_start_result is failed retries: 1 delay: 30 ignore_errors: true - name: Dump logs from node service if it failed - command: journalctl --no-pager -n 100 -u {{ openshift.common.service_type }}-node - when: node_start_result | failed + command: journalctl --no-pager -n 100 -u {{ openshift_service_type }}-node + when: node_start_result is failed - name: Abort if node failed to start fail: msg: Node failed to start please inspect the logs and try again - when: node_start_result | failed + when: node_start_result is failed - set_fact: - node_service_status_changed: "{{ node_start_result | changed }}" + node_service_status_changed: "{{ node_start_result is changed }}" diff --git a/roles/openshift_node/tasks/config/configure-node-settings.yml b/roles/openshift_node/tasks/config/configure-node-settings.yml index 527580481..ebc1426d3 100644 --- a/roles/openshift_node/tasks/config/configure-node-settings.yml +++ b/roles/openshift_node/tasks/config/configure-node-settings.yml @@ -1,7 +1,7 @@ --- - name: Configure Node settings lineinfile: - dest: /etc/sysconfig/{{ openshift.common.service_type }}-node + dest: /etc/sysconfig/{{ openshift_service_type }}-node regexp: "{{ item.regex }}" line: "{{ item.line }}" create: true diff --git a/roles/openshift_node/tasks/config/configure-proxy-settings.yml b/roles/openshift_node/tasks/config/configure-proxy-settings.yml index d60794305..7ddd319d2 100644 --- a/roles/openshift_node/tasks/config/configure-proxy-settings.yml +++ b/roles/openshift_node/tasks/config/configure-proxy-settings.yml @@ -1,7 +1,7 @@ --- - name: Configure Proxy Settings lineinfile: - dest: /etc/sysconfig/{{ openshift.common.service_type }}-node + dest: /etc/sysconfig/{{ openshift_service_type }}-node regexp: "{{ item.regex }}" line: "{{ item.line }}" create: true diff --git a/roles/openshift_node/tasks/config/install-node-deps-docker-service-file.yml b/roles/openshift_node/tasks/config/install-node-deps-docker-service-file.yml index ee91a88ab..9f1145d12 100644 --- a/roles/openshift_node/tasks/config/install-node-deps-docker-service-file.yml +++ b/roles/openshift_node/tasks/config/install-node-deps-docker-service-file.yml @@ -1,7 +1,7 @@ --- - name: Install Node dependencies docker service file template: - dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node-dep.service" + dest: "/etc/systemd/system/{{ openshift_service_type }}-node-dep.service" src: openshift.docker.node.dep.service notify: - reload systemd units diff --git a/roles/openshift_node/tasks/config/install-node-docker-service-file.yml b/roles/openshift_node/tasks/config/install-node-docker-service-file.yml new file mode 100644 index 000000000..649fc5f6b --- /dev/null +++ b/roles/openshift_node/tasks/config/install-node-docker-service-file.yml @@ -0,0 +1,8 @@ +--- +- name: Install Node docker service file + template: + dest: "/etc/systemd/system/{{ openshift_service_type }}-node.service" + src: openshift.docker.node.service + notify: + - reload systemd units + - restart node diff --git a/roles/openshift_node/tasks/container_images.yml b/roles/openshift_node/tasks/container_images.yml new file mode 100644 index 000000000..bb788e2f1 --- /dev/null +++ b/roles/openshift_node/tasks/container_images.yml @@ -0,0 +1,20 @@ +--- +- name: Install Node system container + include_tasks: node_system_container.yml + when: + - l_is_node_system_container | bool + +- name: Install OpenvSwitch system containers + include_tasks: openvswitch_system_container.yml + when: + - openshift_node_use_openshift_sdn | bool + - l_is_openvswitch_system_container | bool + +- name: Pre-pull openvswitch image + command: > + docker pull {{ osn_ovs_image }}:{{ openshift_image_tag }} + register: pull_result + changed_when: "'Downloaded newer image' in pull_result.stdout" + when: + - openshift_node_use_openshift_sdn | bool + - not l_is_openvswitch_system_container | bool diff --git a/roles/openshift_node/tasks/dnsmasq.yml b/roles/openshift_node/tasks/dnsmasq.yml new file mode 100644 index 000000000..31ca46ec0 --- /dev/null +++ b/roles/openshift_node/tasks/dnsmasq.yml @@ -0,0 +1,26 @@ +--- +- name: Install dnsmasq configuration + template: + src: origin-dns.conf.j2 + dest: /etc/dnsmasq.d/origin-dns.conf + notify: restart dnsmasq + +- name: Deploy additional dnsmasq.conf + template: + src: "{{ openshift_node_dnsmasq_additional_config_file }}" + dest: /etc/dnsmasq.d/openshift-ansible.conf + owner: root + group: root + mode: 0644 + when: openshift_node_dnsmasq_additional_config_file is defined + notify: restart dnsmasq + +- name: Enable dnsmasq + systemd: + name: dnsmasq + enabled: yes + state: started + +# Dynamic NetworkManager based dispatcher +- include_tasks: dnsmasq/network-manager.yml + when: network_manager_active | bool diff --git a/roles/openshift_node/tasks/dnsmasq/network-manager.yml b/roles/openshift_node/tasks/dnsmasq/network-manager.yml new file mode 100644 index 000000000..e5a92a630 --- /dev/null +++ b/roles/openshift_node/tasks/dnsmasq/network-manager.yml @@ -0,0 +1,10 @@ +--- +- name: Install network manager dispatch script + copy: + src: networkmanager/99-origin-dns.sh + dest: /etc/NetworkManager/dispatcher.d/ + mode: 0755 + notify: restart NetworkManager + when: openshift_node_dnsmasq_install_network_manager_hook | default(true) | bool + +- meta: flush_handlers diff --git a/roles/openshift_node/tasks/dnsmasq/no-network-manager.yml b/roles/openshift_node/tasks/dnsmasq/no-network-manager.yml new file mode 100644 index 000000000..5d2c67b86 --- /dev/null +++ b/roles/openshift_node/tasks/dnsmasq/no-network-manager.yml @@ -0,0 +1,13 @@ +--- +- fail: msg="Currently, NetworkManager must be installed and enabled prior to installation." + when: not openshift_node_bootstrap | bool + +- name: Install NetworkManager during node_bootstrap provisioning + package: + name: NetworkManager + state: present + notify: restart NetworkManager + register: result + until: result is succeeded + +- include_tasks: network-manager.yml diff --git a/roles/openshift_node/tasks/dnsmasq_install.yml b/roles/openshift_node/tasks/dnsmasq_install.yml new file mode 100644 index 000000000..5e06ba032 --- /dev/null +++ b/roles/openshift_node/tasks/dnsmasq_install.yml @@ -0,0 +1,43 @@ +--- +- name: Check for NetworkManager service + command: > + systemctl show NetworkManager + register: nm_show + changed_when: false + ignore_errors: True + +- name: Set fact using_network_manager + set_fact: + network_manager_active: "{{ True if 'ActiveState=active' in nm_show.stdout else False }}" + +- name: Install dnsmasq + package: name=dnsmasq state=installed + when: not openshift_is_atomic | bool + register: result + until: result is succeeded + +- name: ensure origin/node directory exists + file: + state: directory + path: "{{ item }}" + owner: root + group: root + mode: '0700' + with_items: + - /etc/origin + - /etc/origin/node + +# this file is copied to /etc/dnsmasq.d/ when the node starts and is removed +# when the node stops. A dbus-message is sent to dnsmasq to add the same entries +# so that dnsmasq doesn't need to be restarted. Once we can use dnsmasq 2.77 or +# newer we can use --server-file option to update the servers dynamically and +# reload them by sending dnsmasq a SIGHUP. We write the file in case someone else +# triggers a restart of dnsmasq but not a node restart. +- name: Install node-dnsmasq.conf + template: + src: node-dnsmasq.conf.j2 + dest: /etc/origin/node/node-dnsmasq.conf + +# Relies on ansible in order to configure static config +- include_tasks: dnsmasq/no-network-manager.yml + when: not network_manager_active | bool diff --git a/roles/openshift_node/tasks/install.yml b/roles/openshift_node/tasks/install.yml index 6b7e40491..55738d759 100644 --- a/roles/openshift_node/tasks/install.yml +++ b/roles/openshift_node/tasks/install.yml @@ -1,29 +1,35 @@ --- -- when: not openshift.common.is_containerized | bool +- when: not openshift_is_containerized | bool block: - name: Install Node package package: - name: "{{ openshift.common.service_type }}-node{{ (openshift_pkg_version | default('')) | oo_image_tag_to_rpm_version(include_dash=True) }}" + name: "{{ openshift_service_type }}-node{{ (openshift_pkg_version | default('')) | lib_utils_oo_image_tag_to_rpm_version(include_dash=True) }}" state: present + register: result + until: result is succeeded - name: Install sdn-ovs package package: - name: "{{ openshift.common.service_type }}-sdn-ovs{{ (openshift_pkg_version | default('')) | oo_image_tag_to_rpm_version(include_dash=True) }}" + name: "{{ openshift_service_type }}-sdn-ovs{{ (openshift_pkg_version | default('')) | lib_utils_oo_image_tag_to_rpm_version(include_dash=True) }}" state: present when: - openshift_node_use_openshift_sdn | bool + register: result + until: result is succeeded - name: Install conntrack-tools package package: name: "conntrack-tools" state: present + register: result + until: result is succeeded - when: - - openshift.common.is_containerized | bool - - not openshift.common.is_node_system_container | bool + - openshift_is_containerized | bool + - not l_is_node_system_container | bool block: - name: Pre-pull node image when containerized command: > - docker pull {{ openshift.node.node_image }}:{{ openshift_image_tag }} + docker pull {{ osn_image }}:{{ openshift_image_tag }} register: pull_result changed_when: "'Downloaded newer image' in pull_result.stdout" diff --git a/roles/openshift_node/tasks/main.yml b/roles/openshift_node/tasks/main.yml index eae9ca7bc..754ecacaf 100644 --- a/roles/openshift_node/tasks/main.yml +++ b/roles/openshift_node/tasks/main.yml @@ -3,12 +3,14 @@ msg: "SELinux is disabled, This deployment type requires that SELinux is enabled." when: - (not ansible_selinux or ansible_selinux.status != 'enabled') - - deployment_type == 'openshift-enterprise' - - not openshift_use_crio | default(false) + - openshift_deployment_type == 'openshift-enterprise' + - not openshift_use_crio | bool + +- include_tasks: dnsmasq_install.yml +- include_tasks: dnsmasq.yml - name: setup firewall - include: firewall.yml - static: yes + import_tasks: firewall.yml #### Disable SWAP ##### # https://docs.openshift.com/container-platform/3.4/admin_guide/overcommit.html#disabling-swap-memory @@ -41,14 +43,18 @@ #### End Disable Swap Block #### - name: include node installer - include: install.yml + include_tasks: install.yml - name: Restart cri-o systemd: name: cri-o enabled: yes state: restarted - when: openshift_use_crio | default(false) + when: openshift_use_crio | bool + register: task_result + failed_when: + - task_result is failed + - ('could not find the requested service' not in task_result.msg|lower) - name: restart NetworkManager to ensure resolv.conf is present systemd: @@ -66,34 +72,30 @@ sysctl_file: "/etc/sysctl.d/99-openshift.conf" reload: yes -- include: registry_auth.yml +- include_tasks: registry_auth.yml - name: include standard node config - include: config.yml + include_tasks: config.yml #### Storage class plugins here #### - name: NFS storage plugin configuration - include: storage_plugins/nfs.yml + include_tasks: storage_plugins/nfs.yml tags: - nfs - name: GlusterFS storage plugin configuration - include: storage_plugins/glusterfs.yml - when: "'glusterfs' in openshift.node.storage_plugin_deps" + include_tasks: storage_plugins/glusterfs.yml + when: "'glusterfs' in osn_storage_plugin_deps" - name: Ceph storage plugin configuration - include: storage_plugins/ceph.yml - when: "'ceph' in openshift.node.storage_plugin_deps" + include_tasks: storage_plugins/ceph.yml + when: "'ceph' in osn_storage_plugin_deps" - name: iSCSI storage plugin configuration - include: storage_plugins/iscsi.yml - when: "'iscsi' in openshift.node.storage_plugin_deps" + include_tasks: storage_plugins/iscsi.yml + when: "'iscsi' in osn_storage_plugin_deps" ##### END Storage ##### -- include: config/workaround-bz1331590-ovs-oom-fix.yml +- include_tasks: config/workaround-bz1331590-ovs-oom-fix.yml when: openshift_node_use_openshift_sdn | default(true) | bool - -- name: include bootstrap node config - include: bootstrap.yml - when: openshift_node_bootstrap diff --git a/roles/openshift_node/tasks/node_system_container.yml b/roles/openshift_node/tasks/node_system_container.yml index 20d7a9539..06b879050 100644 --- a/roles/openshift_node/tasks/node_system_container.yml +++ b/roles/openshift_node/tasks/node_system_container.yml @@ -1,16 +1,17 @@ --- + - name: Pre-pull node system container image command: > - atomic pull --storage=ostree {{ 'docker:' if openshift.common.system_images_registry == 'docker' else openshift.common.system_images_registry + '/' }}{{ openshift.node.node_system_image }}:{{ openshift_image_tag }} + atomic pull --storage=ostree {{ 'docker:' if system_images_registry == 'docker' else system_images_registry + '/' }}{{ osn_image }}:{{ openshift_image_tag }} register: pull_result changed_when: "'Pulling layer' in pull_result.stdout" - name: Install or Update node system container oc_atomic_container: - name: "{{ openshift.common.service_type }}-node" - image: "{{ 'docker:' if openshift.common.system_images_registry == 'docker' else openshift.common.system_images_registry + '/' }}{{ openshift.node.node_system_image }}:{{ openshift_image_tag }}" + name: "{{ openshift_service_type }}-node" + image: "{{ 'docker:' if system_images_registry == 'docker' else system_images_registry + '/' }}{{ osn_image }}:{{ openshift_image_tag }}" values: - "DNS_DOMAIN={{ openshift.common.dns_domain }}" - - "DOCKER_SERVICE={{ openshift.docker.service_name }}.service" - - "MASTER_SERVICE={{ openshift.common.service_type }}.service" + - "DOCKER_SERVICE={{ openshift_docker_service_name }}.service" + - "MASTER_SERVICE={{ openshift_service_type }}.service" state: latest diff --git a/roles/openshift_node/tasks/openvswitch_system_container.yml b/roles/openshift_node/tasks/openvswitch_system_container.yml index e09063aa5..d7dce6969 100644 --- a/roles/openshift_node/tasks/openvswitch_system_container.yml +++ b/roles/openshift_node/tasks/openvswitch_system_container.yml @@ -1,25 +1,22 @@ --- - set_fact: - l_use_crio: "{{ openshift_use_crio | default(false) }}" - -- set_fact: l_service_name: "cri-o" - when: l_use_crio + when: openshift_use_crio | bool - set_fact: - l_service_name: "{{ openshift.docker.service_name }}" - when: not l_use_crio + l_service_name: "{{ openshift_docker_service_name }}" + when: not openshift_use_crio | bool - name: Pre-pull OpenVSwitch system container image command: > - atomic pull --storage=ostree {{ 'docker:' if openshift.common.system_images_registry == 'docker' else openshift.common.system_images_registry + '/' }}{{ openshift.node.ovs_system_image }}:{{ openshift_image_tag }} + atomic pull --storage=ostree {{ 'docker:' if system_images_registry == 'docker' else system_images_registry + '/' }}{{ osn_ovs_image }}:{{ openshift_image_tag }} register: pull_result changed_when: "'Pulling layer' in pull_result.stdout" - name: Install or Update OpenVSwitch system container oc_atomic_container: name: openvswitch - image: "{{ 'docker:' if openshift.common.system_images_registry == 'docker' else openshift.common.system_images_registry + '/' }}{{ openshift.node.ovs_system_image }}:{{ openshift_image_tag }}" + image: "{{ 'docker:' if system_images_registry == 'docker' else system_images_registry + '/' }}{{ osn_ovs_image }}:{{ openshift_image_tag }}" state: latest values: - "DOCKER_SERVICE={{ l_service_name }}" diff --git a/roles/openshift_node/tasks/registry_auth.yml b/roles/openshift_node/tasks/registry_auth.yml index de396fb4b..92650e6b7 100644 --- a/roles/openshift_node/tasks/registry_auth.yml +++ b/roles/openshift_node/tasks/registry_auth.yml @@ -8,9 +8,31 @@ - name: Create credentials for registry auth command: "docker --config={{ oreg_auth_credentials_path }} login -u {{ oreg_auth_user }} -p {{ oreg_auth_password }} {{ oreg_host }}" when: + - not (openshift_docker_alternative_creds | default(False)) - oreg_auth_user is defined - (not node_oreg_auth_credentials_stat.stat.exists or oreg_auth_credentials_replace) | bool register: node_oreg_auth_credentials_create + retries: 3 + delay: 5 + until: node_oreg_auth_credentials_create.rc == 0 + notify: + - restart node + +# docker_creds is a custom module from lib_utils +# 'docker login' requires a docker.service running on the local host, this is an +# alternative implementation for non-docker hosts. This implementation does not +# check the registry to determine whether or not the credentials will work. +- name: Create credentials for registry auth (alternative) + docker_creds: + path: "{{ oreg_auth_credentials_path }}" + registry: "{{ oreg_host }}" + username: "{{ oreg_auth_user }}" + password: "{{ oreg_auth_password }}" + when: + - openshift_docker_alternative_creds | bool + - oreg_auth_user is defined + - (not node_oreg_auth_credentials_stat.stat.exists or oreg_auth_credentials_replace) | bool + register: node_oreg_auth_credentials_create_alt notify: - restart node @@ -19,6 +41,10 @@ set_fact: l_bind_docker_reg_auth: True when: - - openshift.common.is_containerized | bool + - openshift_is_containerized | bool - oreg_auth_user is defined - - (node_oreg_auth_credentials_stat.stat.exists or oreg_auth_credentials_replace or node_oreg_auth_credentials_create.changed) | bool + - > + (node_oreg_auth_credentials_stat.stat.exists + or oreg_auth_credentials_replace + or node_oreg_auth_credentials_create.changed + or node_oreg_auth_credentials_create_alt.changed) | bool diff --git a/roles/openshift_node/tasks/storage_plugins/ceph.yml b/roles/openshift_node/tasks/storage_plugins/ceph.yml index 037efe81a..e30f58a9a 100644 --- a/roles/openshift_node/tasks/storage_plugins/ceph.yml +++ b/roles/openshift_node/tasks/storage_plugins/ceph.yml @@ -1,4 +1,6 @@ --- - name: Install Ceph storage plugin dependencies package: name=ceph-common state=present - when: not openshift.common.is_atomic | bool + when: not openshift_is_atomic | bool + register: result + until: result is succeeded diff --git a/roles/openshift_node/tasks/storage_plugins/glusterfs.yml b/roles/openshift_node/tasks/storage_plugins/glusterfs.yml index 1b8a7ad50..c04a6922a 100644 --- a/roles/openshift_node/tasks/storage_plugins/glusterfs.yml +++ b/roles/openshift_node/tasks/storage_plugins/glusterfs.yml @@ -1,7 +1,9 @@ --- - name: Install GlusterFS storage plugin dependencies package: name=glusterfs-fuse state=present - when: not openshift.common.is_atomic | bool + when: not openshift_is_atomic | bool + register: result + until: result is succeeded - name: Check for existence of fusefs sebooleans command: getsebool {{ item }} @@ -29,7 +31,7 @@ # since getsebool prints the resolved name. (At some point Ansible's seboolean module # should learn to deal with aliases) - item.item in item.stdout # Boolean does not have an alias. - - ansible_python_version | version_compare('3', '<') + - ansible_python_version is version_compare('3', '<') with_items: "{{ fusefs_getsebool_status.results }}" # Workaround for https://github.com/openshift/openshift-ansible/issues/4438 @@ -50,5 +52,5 @@ # should learn to deal with aliases) - item.item in item.stdout # Boolean does not have an alias. - ('--> off' in item.stdout) # Boolean is currently off. - - ansible_python_version | version_compare('3', '>=') + - ansible_python_version is version_compare('3', '>=') with_items: "{{ fusefs_getsebool_status.results }}" diff --git a/roles/openshift_node/tasks/storage_plugins/iscsi.yml b/roles/openshift_node/tasks/storage_plugins/iscsi.yml index 1c5478c55..a8048c42f 100644 --- a/roles/openshift_node/tasks/storage_plugins/iscsi.yml +++ b/roles/openshift_node/tasks/storage_plugins/iscsi.yml @@ -1,4 +1,6 @@ --- - name: Install iSCSI storage plugin dependencies package: name=iscsi-initiator-utils state=present - when: not openshift.common.is_atomic | bool + when: not openshift_is_atomic | bool + register: result + until: result is succeeded diff --git a/roles/openshift_node/tasks/storage_plugins/nfs.yml b/roles/openshift_node/tasks/storage_plugins/nfs.yml index 7e1035893..c2922644f 100644 --- a/roles/openshift_node/tasks/storage_plugins/nfs.yml +++ b/roles/openshift_node/tasks/storage_plugins/nfs.yml @@ -1,7 +1,9 @@ --- - name: Install NFS storage plugin dependencies package: name=nfs-utils state=present - when: not openshift.common.is_atomic | bool + when: not openshift_is_atomic | bool + register: result + until: result is succeeded - name: Check for existence of nfs sebooleans command: getsebool {{ item }} @@ -29,7 +31,7 @@ # since getsebool prints the resolved name. (At some point Ansible's seboolean module # should learn to deal with aliases) - item.item in item.stdout # Boolean does not have an alias. - - ansible_python_version | version_compare('3', '<') + - ansible_python_version is version_compare('3', '<') with_items: "{{ nfs_getsebool_status.results }}" # Workaround for https://github.com/openshift/openshift-ansible/issues/4438 @@ -50,5 +52,5 @@ # should learn to deal with aliases) - item.item in item.stdout # Boolean does not have an alias. - ('--> off' in item.stdout) # Boolean is currently off. - - ansible_python_version | version_compare('3', '>=') + - ansible_python_version is version_compare('3', '>=') with_items: "{{ nfs_getsebool_status.results }}" diff --git a/roles/openshift_node/tasks/systemd_units.yml b/roles/openshift_node/tasks/systemd_units.yml index 9c182ade6..e33a4999f 100644 --- a/roles/openshift_node/tasks/systemd_units.yml +++ b/roles/openshift_node/tasks/systemd_units.yml @@ -1,44 +1,25 @@ --- - name: Install Node service file template: - dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node.service" - src: "{{ openshift.common.is_containerized | bool | ternary('openshift.docker.node.service', 'node.service.j2') }}" - when: not openshift.common.is_node_system_container | bool + dest: "/etc/systemd/system/{{ openshift_service_type }}-node.service" + src: "{{ openshift_is_containerized | bool | ternary('openshift.docker.node.service', 'node.service.j2') }}" + when: not l_is_node_system_container | bool notify: - reload systemd units - restart node -- when: openshift.common.is_containerized | bool +- when: openshift_is_containerized | bool block: - name: include node deps docker service file - include: config/install-node-deps-docker-service-file.yml + include_tasks: config/install-node-deps-docker-service-file.yml - name: include ovs service environment file - include: config/install-ovs-service-env-file.yml + include_tasks: config/install-ovs-service-env-file.yml - - name: Install Node system container - include: node_system_container.yml - when: - - openshift.common.is_node_system_container | bool - - - name: Install OpenvSwitch system containers - include: openvswitch_system_container.yml + - include_tasks: config/install-ovs-docker-service-file.yml when: - openshift_node_use_openshift_sdn | bool - - openshift.common.is_openvswitch_system_container | bool - -- block: - - name: Pre-pull openvswitch image - command: > - docker pull {{ openshift.node.ovs_image }}:{{ openshift_image_tag }} - register: pull_result - changed_when: "'Downloaded newer image' in pull_result.stdout" - - - include: config/install-ovs-docker-service-file.yml - when: - - openshift.common.is_containerized | bool - - openshift_node_use_openshift_sdn | bool - - not openshift.common.is_openvswitch_system_container | bool + - not l_is_openvswitch_system_container | bool -- include: config/configure-node-settings.yml -- include: config/configure-proxy-settings.yml +- include_tasks: config/configure-node-settings.yml +- include_tasks: config/configure-proxy-settings.yml diff --git a/roles/openshift_node/tasks/upgrade.yml b/roles/openshift_node/tasks/upgrade.yml new file mode 100644 index 000000000..02e417937 --- /dev/null +++ b/roles/openshift_node/tasks/upgrade.yml @@ -0,0 +1,54 @@ +--- +# input variables: +# - l_docker_upgrade +# - openshift_is_atomic +# - node_config_hook +# - openshift_pkg_version +# - openshift_is_containerized +# - openshift_release + +# tasks file for openshift_node_upgrade + +- name: stop services for upgrade + include_tasks: upgrade/stop_services.yml + +# Ensure actually install latest package. +- name: download docker upgrade rpm + command: "{{ ansible_pkg_mgr }} install -C -y docker{{ '-' + docker_version }}" + register: result + until: result is succeeded + when: + - l_docker_upgrade is defined + - l_docker_upgrade | bool + +- name: install pre-pulled rpms. + include_tasks: upgrade/rpm_upgrade_install.yml + vars: + openshift_version: "{{ openshift_pkg_version | default('') }}" + when: not openshift_is_containerized | bool + + +- include_tasks: "{{ node_config_hook }}" + when: node_config_hook is defined + +- include_tasks: upgrade/config_changes.yml + +# Restart all services +- include_tasks: upgrade/restart.yml + +- name: Wait for node to be ready + oc_obj: + state: list + kind: node + name: "{{ openshift.common.hostname | lower }}" + register: node_output + delegate_to: "{{ groups.oo_first_master.0 }}" + until: node_output.results.returncode == 0 and node_output.results.results[0].status.conditions | selectattr('type', 'match', '^Ready$') | map(attribute='status') | join | bool == True + # Give the node two minutes to come back online. + retries: 24 + delay: 5 + +- include_tasks: dnsmasq_install.yml +- include_tasks: dnsmasq.yml + +- meta: flush_handlers diff --git a/roles/openshift_node/tasks/upgrade/config_changes.yml b/roles/openshift_node/tasks/upgrade/config_changes.yml new file mode 100644 index 000000000..721656117 --- /dev/null +++ b/roles/openshift_node/tasks/upgrade/config_changes.yml @@ -0,0 +1,77 @@ +--- +- name: Update systemd units + include_tasks: ../systemd_units.yml + when: openshift_is_containerized | bool + +- name: Update oreg value + yedit: + src: "{{ openshift.common.config_base }}/node/node-config.yaml" + key: 'imageConfig.format' + value: "{{ oreg_url | default(oreg_url_node) }}" + when: oreg_url is defined or oreg_url_node is defined + +- name: Remove obsolete docker-sdn-ovs.conf + file: + path: "/etc/systemd/system/docker.service.d/docker-sdn-ovs.conf" + state: absent + +# https://bugzilla.redhat.com/show_bug.cgi?id=1513054 +- name: Clean up dockershim data + file: + path: "/var/lib/dockershim/sandbox/" + state: absent + +# Disable Swap Block (pre) +- block: + - name: Remove swap entries from /etc/fstab + replace: + dest: /etc/fstab + regexp: '(^[^#].*swap.*)' + replace: '# \1' + backup: yes + + - name: Add notice about disabling swap + lineinfile: + dest: /etc/fstab + line: '# OpenShift-Ansible Installer disabled swap per overcommit guidelines' + state: present + + - name: Disable swap + command: swapoff --all + + when: + - openshift_node_upgrade_swap_result | default(False) | bool + - openshift_disable_swap | default(true) | bool +# End Disable Swap Block + +- name: Apply 3.6 dns config changes + yedit: + src: /etc/origin/node/node-config.yaml + key: "{{ item.key }}" + value: "{{ item.value }}" + with_items: + - key: "dnsBindAddress" + value: "127.0.0.1:53" + - key: "dnsRecursiveResolvConf" + value: "/etc/origin/node/resolv.conf" + +- name: Install Node service file + template: + dest: "/etc/systemd/system/{{ openshift_service_type }}-node.service" + src: "node.service.j2" + register: l_node_unit + when: not openshift_is_containerized | bool + +- name: Reset selinux context + command: restorecon -RF {{ openshift_node_data_dir }}/openshift.local.volumes + when: + - ansible_selinux is defined + - ansible_selinux.status == 'enabled' + +# NOTE: This is needed to make sure we are using the correct set +# of systemd unit files. The RPMs lay down defaults but +# the install/upgrade may override them in /etc/systemd/system/. +# NOTE: We don't use the systemd module as some versions of the module +# require a service to be part of the call. +- name: Reload systemd units + command: systemctl daemon-reload diff --git a/roles/openshift_node/tasks/upgrade/containerized_upgrade_pull.yml b/roles/openshift_node/tasks/upgrade/containerized_upgrade_pull.yml new file mode 100644 index 000000000..e5477f389 --- /dev/null +++ b/roles/openshift_node/tasks/upgrade/containerized_upgrade_pull.yml @@ -0,0 +1,15 @@ +--- +- name: Pre-pull node image + command: > + docker pull {{ osn_image }}:{{ openshift_image_tag }} + register: pull_result + changed_when: "'Downloaded newer image' in pull_result.stdout" + +- name: Pre-pull openvswitch image + command: > + docker pull {{ osn_ovs_image }}:{{ openshift_image_tag }} + register: pull_result + changed_when: "'Downloaded newer image' in pull_result.stdout" + when: openshift_node_use_openshift_sdn | bool + +- include_tasks: ../container_images.yml diff --git a/roles/openshift_node/tasks/upgrade/restart.yml b/roles/openshift_node/tasks/upgrade/restart.yml new file mode 100644 index 000000000..bd6f42182 --- /dev/null +++ b/roles/openshift_node/tasks/upgrade/restart.yml @@ -0,0 +1,51 @@ +--- +# input variables: +# - openshift_service_type +# - openshift_is_containerized +# - openshift.common.hostname +# - openshift.master.api_port + +# NOTE: This is needed to make sure we are using the correct set +# of systemd unit files. The RPMs lay down defaults but +# the install/upgrade may override them in /etc/systemd/system/. +# NOTE: We don't use the systemd module as some versions of the module +# require a service to be part of the call. +- name: Reload systemd to ensure latest unit files + command: systemctl daemon-reload + +- name: Restart support services + service: + name: "{{ item }}" + state: restarted + enabled: True + with_items: + - NetworkManager + - dnsmasq + +- name: Restart container runtime + service: + name: "{{ openshift_docker_service_name }}" + state: started + register: docker_start_result + until: not (docker_start_result is failed) + retries: 3 + delay: 30 + +- name: Start services + service: name={{ item }} state=started + with_items: + - etcd_container + - openvswitch + - "{{ openshift_service_type }}-master-api" + - "{{ openshift_service_type }}-master-controllers" + - "{{ openshift_service_type }}-node" + failed_when: false + +- name: Wait for master API to come back online + wait_for: + host: "{{ openshift.common.hostname }}" + state: started + delay: 10 + port: "{{ openshift.master.api_port }}" + timeout: 600 + when: inventory_hostname in groups.oo_masters_to_config diff --git a/roles/openshift_node/tasks/upgrade/rpm_upgrade.yml b/roles/openshift_node/tasks/upgrade/rpm_upgrade.yml new file mode 100644 index 000000000..91a358095 --- /dev/null +++ b/roles/openshift_node/tasks/upgrade/rpm_upgrade.yml @@ -0,0 +1,24 @@ +--- +# input variables: +# - openshift_service_type +# - component +# - openshift_pkg_version +# - openshift_is_atomic + +# Pre-pull new node rpm, but don't install +- name: download new node packages + command: "{{ ansible_pkg_mgr }} install -y --downloadonly {{ openshift_node_upgrade_rpm_list | join(' ')}}" + register: result + until: result is succeeded + vars: + openshift_node_upgrade_rpm_list: + - "{{ openshift_service_type }}-node{{ openshift_pkg_version }}" + - "PyYAML" + - "dnsmasq" + +# Pre-pull the rpms for openvswitch, but don't install +# openvswitch requires the latest version to be installed. +- name: download openvswitch upgrade rpm + command: "{{ ansible_pkg_mgr }} update -y --downloadonly openvswitch" + register: result + until: result is succeeded diff --git a/roles/openshift_node/tasks/upgrade/rpm_upgrade_install.yml b/roles/openshift_node/tasks/upgrade/rpm_upgrade_install.yml new file mode 100644 index 000000000..c9094e05a --- /dev/null +++ b/roles/openshift_node/tasks/upgrade/rpm_upgrade_install.yml @@ -0,0 +1,19 @@ +--- +# input variables: +# - openshift_service_type +# - component +# - openshift_pkg_version +# - openshift_is_atomic + +# Install the pre-pulled RPM +# Note: dnsmasq is covered in it's own play. openvswitch is included here +# because once we have the latest rpm downloaded, it will happily be installed. +- name: download new node packages + command: "{{ ansible_pkg_mgr }} install -C -y {{ openshift_node_upgrade_rpm_list | join(' ')}}" + register: result + until: result is succeeded + vars: + openshift_node_upgrade_rpm_list: + - "{{ openshift_service_type }}-node{{ openshift_pkg_version }}" + - "PyYAML" + - "openvswitch" diff --git a/roles/openshift_node/tasks/upgrade/stop_services.yml b/roles/openshift_node/tasks/upgrade/stop_services.yml new file mode 100644 index 000000000..6d92516c3 --- /dev/null +++ b/roles/openshift_node/tasks/upgrade/stop_services.yml @@ -0,0 +1,43 @@ +--- +- name: Stop node and openvswitch services + service: + name: "{{ item }}" + state: stopped + with_items: + - "{{ openshift_service_type }}-node" + - openvswitch + failed_when: false + +- name: Ensure containerized services stopped before Docker restart + service: + name: "{{ item }}" + state: stopped + with_items: + - etcd_container + - openvswitch + - "{{ openshift_service_type }}-master-api" + - "{{ openshift_service_type }}-master-controllers" + - "{{ openshift_service_type }}-node" + failed_when: false + when: openshift_is_containerized | bool + +- service: + name: docker + state: stopped + register: l_openshift_node_upgrade_docker_stop_result + until: not (l_openshift_node_upgrade_docker_stop_result is failed) + retries: 3 + delay: 30 + when: + - l_docker_upgrade is defined + - l_docker_upgrade | bool + +- name: Stop rpm based services + service: + name: "{{ item }}" + state: stopped + with_items: + - "{{ openshift_service_type }}-node" + - openvswitch + failed_when: false + when: not openshift_is_containerized | bool diff --git a/roles/openshift_node/tasks/upgrade_pre.yml b/roles/openshift_node/tasks/upgrade_pre.yml new file mode 100644 index 000000000..3ae7dc6b6 --- /dev/null +++ b/roles/openshift_node/tasks/upgrade_pre.yml @@ -0,0 +1,56 @@ +--- +# This is a hack to allow us to update various components without restarting +# services. This will persist into the upgrade play as well, so everything +# needs to be restarted by hand. +- set_fact: + skip_node_svc_handlers: True + +- include_tasks: registry_auth.yml + +- name: update package meta data to speed install later. + command: "{{ ansible_pkg_mgr }} makecache" + register: result + until: result is succeeded + when: not openshift_is_containerized | bool + +- name: Check Docker image count + shell: "docker images -aq | wc -l" + register: docker_image_count + when: + - l_docker_upgrade is defined + - l_docker_upgrade | bool + +- debug: var=docker_image_count.stdout + when: + - l_docker_upgrade is defined + - l_docker_upgrade | bool + +- include_tasks: upgrade/containerized_upgrade_pull.yml + when: openshift_is_containerized | bool + +# Prepull the rpms for docker upgrade, but don't install +- name: download docker upgrade rpm + command: "{{ ansible_pkg_mgr }} install -y --downloadonly docker{{ '-' + docker_version }}" + register: result + until: result is succeeded + when: + - l_docker_upgrade is defined + - l_docker_upgrade | bool + +- include_tasks: upgrade/rpm_upgrade.yml + vars: + openshift_version: "{{ openshift_pkg_version | default('') }}" + when: not openshift_is_containerized | bool + +# https://docs.openshift.com/container-platform/3.4/admin_guide/overcommit.html#disabling-swap-memory +- name: Check for swap usage + command: grep "^[^#].*swap" /etc/fstab + # grep: match any lines which don't begin with '#' and contain 'swap' + changed_when: false + failed_when: false + register: swap_result + +# Set this fact here so we can use it during the next play, which is serial. +- name: set_fact swap_result + set_fact: + openshift_node_upgrade_swap_result: "{{ swap_result.stdout_lines | length > 0 | bool }}" diff --git a/roles/openshift_node/templates/node-dnsmasq.conf.j2 b/roles/openshift_node/templates/node-dnsmasq.conf.j2 new file mode 100644 index 000000000..3caa3bd4a --- /dev/null +++ b/roles/openshift_node/templates/node-dnsmasq.conf.j2 @@ -0,0 +1,2 @@ +server=/in-addr.arpa/127.0.0.1 +server=/{{ openshift.common.dns_domain }}/127.0.0.1 diff --git a/roles/openshift_node/templates/node.service.j2 b/roles/openshift_node/templates/node.service.j2 index 7602d8ee6..777f4a449 100644 --- a/roles/openshift_node/templates/node.service.j2 +++ b/roles/openshift_node/templates/node.service.j2 @@ -1,14 +1,14 @@ [Unit] Description=OpenShift Node -After={{ openshift.docker.service_name }}.service +After={{ openshift_docker_service_name }}.service Wants=openvswitch.service After=ovsdb-server.service After=ovs-vswitchd.service -Wants={{ openshift.docker.service_name }}.service +Wants={{ openshift_docker_service_name }}.service Documentation=https://github.com/openshift/origin Requires=dnsmasq.service After=dnsmasq.service -{% if openshift_use_crio|default(false) %}Wants=cri-o.service{% endif %} +{% if openshift_use_crio | bool %}Wants=cri-o.service{% endif %} [Service] Type=notify diff --git a/roles/openshift_node/templates/node.yaml.v1.j2 b/roles/openshift_node/templates/node.yaml.v1.j2 index 718d35dca..5f2a94ea2 100644 --- a/roles/openshift_node/templates/node.yaml.v1.j2 +++ b/roles/openshift_node/templates/node.yaml.v1.j2 @@ -5,37 +5,33 @@ dnsBindAddress: 127.0.0.1:53 dnsRecursiveResolvConf: /etc/origin/node/resolv.conf {% endif %} dnsDomain: {{ openshift.common.dns_domain }} -{% if 'dns_ip' in openshift.node %} -dnsIP: {{ openshift.node.dns_ip }} -{% endif %} +dnsIP: {{ openshift_dns_ip }} dockerConfig: execHandlerName: "" -iptablesSyncPeriod: "{{ openshift.node.iptables_sync_period }}" +iptablesSyncPeriod: "{{ openshift_node_iptables_sync_period }}" imageConfig: - format: {{ openshift.node.registry_url }} + format: {{ oreg_url_node }} latest: {{ openshift_node_image_config_latest }} kind: NodeConfig -kubeletArguments: {{ openshift.node.kubelet_args | default(None) | to_padded_yaml(level=1) }} -{% if openshift_use_crio | default(False) %} +kubeletArguments: {{ l2_openshift_node_kubelet_args | default(None) | lib_utils_to_padded_yaml(level=1) }} +{% if openshift_use_crio | bool %} container-runtime: - remote container-runtime-endpoint: - - /var/run/crio.sock + - /var/run/crio/crio.sock image-service-endpoint: - - /var/run/crio.sock + - /var/run/crio/crio.sock node-labels: - router=true - registry=true runtime-request-timeout: - 10m {% endif %} -{% if openshift.common.version_gte_3_3_or_1_3 | bool %} masterClientConnectionOverrides: acceptContentTypes: application/vnd.kubernetes.protobuf,application/json contentType: application/vnd.kubernetes.protobuf burst: 200 qps: 100 -{% endif %} masterKubeConfig: system:node:{{ openshift.common.hostname }}.kubeconfig {% if openshift_node_use_openshift_sdn | bool %} networkPluginName: {{ openshift_node_sdn_network_plugin_name }} @@ -47,7 +43,7 @@ networkConfig: {% if openshift_node_use_openshift_sdn | bool or openshift_node_use_nuage | bool or openshift_node_use_contiv | bool or openshift_node_use_kuryr | bool or openshift_node_sdn_network_plugin_name == 'cni' %} networkPluginName: {{ openshift_node_sdn_network_plugin_name }} {% endif %} -{% if openshift.node.set_node_ip | bool %} +{% if openshift_set_node_ip | bool %} nodeIP: {{ openshift.common.ip }} {% endif %} nodeName: {{ openshift.node.nodename }} @@ -70,8 +66,8 @@ volumeDirectory: {{ openshift_node_data_dir }}/openshift.local.volumes {% if not (openshift_node_use_kuryr | default(False)) | bool %} proxyArguments: proxy-mode: - - {{ openshift.node.proxy_mode }} + - {{ openshift_node_proxy_mode }} {% endif %} volumeConfig: localQuota: - perFSGroup: {{ openshift.node.local_quota_per_fsgroup }} + perFSGroup: {{ openshift_node_local_quota_per_fsgroup }} diff --git a/roles/openshift_node/templates/openshift.docker.node.dep.service b/roles/openshift_node/templates/openshift.docker.node.dep.service index fa7238849..9fe779057 100644 --- a/roles/openshift_node/templates/openshift.docker.node.dep.service +++ b/roles/openshift_node/templates/openshift.docker.node.dep.service @@ -1,11 +1,17 @@ [Unit] -Requires={{ openshift.docker.service_name }}.service -After={{ openshift.docker.service_name }}.service -PartOf={{ openshift.common.service_type }}-node.service -Before={{ openshift.common.service_type }}-node.service -{% if openshift_use_crio|default(false) %}Wants=cri-o.service{% endif %} +Requires={{ openshift_docker_service_name }}.service +After={{ openshift_docker_service_name }}.service +PartOf={{ openshift_service_type }}-node.service +Before={{ openshift_service_type }}-node.service +{% if openshift_use_crio | bool %}Wants=cri-o.service{% endif %} [Service] -ExecStart=/bin/bash -c "if [[ -f /usr/bin/docker-current ]]; then echo \"DOCKER_ADDTL_BIND_MOUNTS=--volume=/usr/bin/docker-current:/usr/bin/docker-current:ro --volume=/etc/sysconfig/docker:/etc/sysconfig/docker:ro --volume=/etc/containers/registries:/etc/containers/registries:ro\" > /etc/sysconfig/{{ openshift.common.service_type }}-node-dep; else echo \"#DOCKER_ADDTL_BIND_MOUNTS=\" > /etc/sysconfig/{{ openshift.common.service_type }}-node-dep; fi" +ExecStart=/bin/bash -c 'if [[ -f /usr/bin/docker-current ]]; \ + then echo DOCKER_ADDTL_BIND_MOUNTS=\"--volume=/usr/bin/docker-current:/usr/bin/docker-current:ro \ + --volume=/etc/sysconfig/docker:/etc/sysconfig/docker:ro \ + --volume=/etc/containers/registries:/etc/containers/registries:ro \ + {% if l_bind_docker_reg_auth %} --volume={{ oreg_auth_credentials_path }}:/root/.docker:ro{% endif %}\" > \ + /etc/sysconfig/{{ openshift_service_type }}-node-dep; \ + else echo "#DOCKER_ADDTL_BIND_MOUNTS=" > /etc/sysconfig/{{ openshift_service_type }}-node-dep; fi' ExecStop= -SyslogIdentifier={{ openshift.common.service_type }}-node-dep +SyslogIdentifier={{ openshift_service_type }}-node-dep diff --git a/roles/openshift_node/templates/openshift.docker.node.service b/roles/openshift_node/templates/openshift.docker.node.service index 561aa01f4..ae7b147a6 100644 --- a/roles/openshift_node/templates/openshift.docker.node.service +++ b/roles/openshift_node/templates/openshift.docker.node.service @@ -1,32 +1,32 @@ [Unit] -After={{ openshift.common.service_type }}-master.service -After={{ openshift.docker.service_name }}.service +After={{ openshift_service_type }}-master.service +After={{ openshift_docker_service_name }}.service After=openvswitch.service -PartOf={{ openshift.docker.service_name }}.service -Requires={{ openshift.docker.service_name }}.service +PartOf={{ openshift_docker_service_name }}.service +Requires={{ openshift_docker_service_name }}.service {% if openshift_node_use_openshift_sdn %} Wants=openvswitch.service PartOf=openvswitch.service After=ovsdb-server.service After=ovs-vswitchd.service {% endif %} -Wants={{ openshift.common.service_type }}-master.service -Requires={{ openshift.common.service_type }}-node-dep.service -After={{ openshift.common.service_type }}-node-dep.service +Wants={{ openshift_service_type }}-master.service +Requires={{ openshift_service_type }}-node-dep.service +After={{ openshift_service_type }}-node-dep.service Requires=dnsmasq.service After=dnsmasq.service [Service] -EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node -EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node-dep -ExecStartPre=-/usr/bin/docker rm -f {{ openshift.common.service_type }}-node +EnvironmentFile=/etc/sysconfig/{{ openshift_service_type }}-node +EnvironmentFile=/etc/sysconfig/{{ openshift_service_type }}-node-dep +ExecStartPre=-/usr/bin/docker rm -f {{ openshift_service_type }}-node ExecStartPre=/usr/bin/cp /etc/origin/node/node-dnsmasq.conf /etc/dnsmasq.d/ ExecStartPre=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:/in-addr.arpa/127.0.0.1,/{{ openshift.common.dns_domain }}/127.0.0.1 -ExecStart=/usr/bin/docker run --name {{ openshift.common.service_type }}-node \ - --rm --privileged --net=host --pid=host --env-file=/etc/sysconfig/{{ openshift.common.service_type }}-node \ +ExecStart=/usr/bin/docker run --name {{ openshift_service_type }}-node \ + --rm --privileged --net=host --pid=host --env-file=/etc/sysconfig/{{ openshift_service_type }}-node \ -v /:/rootfs:ro,rslave -e CONFIG_FILE=${CONFIG_FILE} -e OPTIONS=${OPTIONS} \ -e HOST=/rootfs -e HOST_ETC=/host-etc \ - -v {{ openshift_node_data_dir }}:{{ openshift_node_data_dir }}{{ ':rslave' if openshift.docker.gte_1_10 | default(False) | bool else '' }} \ + -v {{ openshift_node_data_dir }}:{{ openshift_node_data_dir }}:rslave \ -v {{ openshift.common.config_base }}/node:{{ openshift.common.config_base }}/node \ {% if openshift_cloudprovider_kind | default('') != '' -%} -v {{ openshift.common.config_base }}/cloudprovider:{{ openshift.common.config_base}}/cloudprovider {% endif -%} \ -v /etc/localtime:/etc/localtime:ro -v /etc/machine-id:/etc/machine-id:ro \ @@ -38,14 +38,14 @@ ExecStart=/usr/bin/docker run --name {{ openshift.common.service_type }}-node \ {% if openshift_use_nuage | default(false) -%} $NUAGE_ADDTL_BIND_MOUNTS {% endif -%} \ -v /dev:/dev $DOCKER_ADDTL_BIND_MOUNTS -v /etc/pki:/etc/pki:ro \ {% if l_bind_docker_reg_auth %} -v {{ oreg_auth_credentials_path }}:/root/.docker:ro{% endif %}\ - {{ openshift.node.node_image }}:${IMAGE_VERSION} + {{ osn_image }}:${IMAGE_VERSION} ExecStartPost=/usr/bin/sleep 10 -ExecStop=/usr/bin/docker stop {{ openshift.common.service_type }}-node +ExecStop=/usr/bin/docker stop {{ openshift_service_type }}-node ExecStopPost=/usr/bin/rm /etc/dnsmasq.d/node-dnsmasq.conf ExecStopPost=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string: -SyslogIdentifier={{ openshift.common.service_type }}-node +SyslogIdentifier={{ openshift_service_type }}-node Restart=always RestartSec=5s [Install] -WantedBy={{ openshift.docker.service_name }}.service +WantedBy={{ openshift_docker_service_name }}.service diff --git a/roles/openshift_node/templates/openvswitch.docker.service b/roles/openshift_node/templates/openvswitch.docker.service index 34aaaabd6..1fc9b6e72 100644 --- a/roles/openshift_node/templates/openvswitch.docker.service +++ b/roles/openshift_node/templates/openvswitch.docker.service @@ -1,12 +1,12 @@ [Unit] -After={{ openshift.docker.service_name }}.service -Requires={{ openshift.docker.service_name }}.service -PartOf={{ openshift.docker.service_name }}.service +After={{ openshift_docker_service_name }}.service +Requires={{ openshift_docker_service_name }}.service +PartOf={{ openshift_docker_service_name }}.service [Service] EnvironmentFile=/etc/sysconfig/openvswitch ExecStartPre=-/usr/bin/docker rm -f openvswitch -ExecStart=/usr/bin/docker run --name openvswitch --rm --privileged --net=host --pid=host -v /lib/modules:/lib/modules -v /run:/run -v /sys:/sys:ro -v /etc/origin/openvswitch:/etc/openvswitch {{ openshift.node.ovs_image }}:${IMAGE_VERSION} +ExecStart=/usr/bin/docker run --name openvswitch --rm --privileged --net=host --pid=host -v /lib/modules:/lib/modules -v /run:/run -v /sys:/sys:ro -v /etc/origin/openvswitch:/etc/openvswitch {{ osn_ovs_image }}:${IMAGE_VERSION} ExecStartPost=/usr/bin/sleep 5 ExecStop=/usr/bin/docker stop openvswitch SyslogIdentifier=openvswitch @@ -14,4 +14,4 @@ Restart=always RestartSec=5s [Install] -WantedBy={{ openshift.docker.service_name }}.service +WantedBy={{ openshift_docker_service_name }}.service diff --git a/roles/openshift_node/templates/origin-dns.conf.j2 b/roles/openshift_node/templates/origin-dns.conf.j2 new file mode 100644 index 000000000..6543c7c3e --- /dev/null +++ b/roles/openshift_node/templates/origin-dns.conf.j2 @@ -0,0 +1,12 @@ +no-resolv +domain-needed +no-negcache +max-cache-ttl=1 +enable-dbus +dns-forward-max=5000 +cache-size=5000 +bind-dynamic +{% for interface in openshift_node_dnsmasq_except_interfaces %} +except-interface={{ interface }} +{% endfor %} +# End of config |