11 files changed, 149 insertions, 72 deletions
diff --git a/roles/openshift_node/handlers/main.yml b/roles/openshift_node/handlers/main.yml
index a6bd12d4e..6b38da7f8 100644
--- a/roles/openshift_node/handlers/main.yml
+++ b/roles/openshift_node/handlers/main.yml
@@ -4,9 +4,14 @@
     name: openvswitch
     state: restarted
   when: (not skip_node_svc_handlers | default(False) | bool) and not (ovs_service_status_changed | default(false) | bool) and openshift.common.use_openshift_sdn | bool
+  register: l_openshift_node_stop_openvswitch_result
+  until: not l_openshift_node_stop_openvswitch_result | failed
+  retries: 3
+  delay: 30
   notify:
   - restart openvswitch pause
 
+
 - name: restart openvswitch pause
   pause: seconds=15
   when: (not skip_node_svc_handlers | default(False) | bool) and openshift.common.is_containerized | bool
@@ -15,7 +20,13 @@
   systemd:
     name: "{{ openshift.common.service_type }}-node"
     state: restarted
-  when: (not skip_node_svc_handlers | default(False) | bool) and not (node_service_status_changed | default(false) | bool)
+  register: l_openshift_node_restart_node_result
+  until: not l_openshift_node_restart_node_result | failed
+  retries: 3
+  delay: 30
+  when:
+  - (not skip_node_svc_handlers | default(False) | bool)
+  - not (node_service_status_changed | default(false) | bool)
 
 - name: reload sysctl.conf
   command: /sbin/sysctl -p
diff --git a/roles/openshift_node/tasks/main.yml b/roles/openshift_node/tasks/main.yml
index 573051504..0133533fc 100644
--- a/roles/openshift_node/tasks/main.yml
+++ b/roles/openshift_node/tasks/main.yml
@@ -118,8 +118,12 @@
     name: openvswitch.service
     enabled: yes
     state: started
+    daemon_reload: yes
   when: openshift.common.is_containerized | bool and openshift.common.use_openshift_sdn | bool
   register: ovs_start_result
+  until: not ovs_start_result | failed
+  retries: 3
+  delay: 30
 
 - set_fact:
     ovs_service_status_changed: "{{ ovs_start_result | changed }}"
@@ -212,15 +216,27 @@
     state: started
   when: openshift.common.is_containerized | bool
 
+
 - name: Start and enable node
   systemd:
     name: "{{ openshift.common.service_type }}-node"
     enabled: yes
     state: started
+    daemon_reload: yes
   register: node_start_result
   until: not node_start_result | failed
   retries: 1
   delay: 30
+  ignore_errors: true
+
+- name: Dump logs from node service if it failed
+  command: journalctl --no-pager -n 100 -u {{ openshift.common.service_type }}-node
+  when: node_start_result | failed
+
+- name: Abort if node failed to start
+  fail:
+    msg: Node failed to start please inspect the logs and try again
+  when: node_start_result | failed
 
 - set_fact:
     node_service_status_changed: "{{ node_start_result | changed }}"
diff --git a/roles/openshift_node/tasks/openvswitch_system_container.yml b/roles/openshift_node/tasks/openvswitch_system_container.yml
index 8cfa5a026..c8d653880 100644
--- a/roles/openshift_node/tasks/openvswitch_system_container.yml
+++ b/roles/openshift_node/tasks/openvswitch_system_container.yml
@@ -10,3 +10,5 @@
     name: openvswitch
     image: "{{ openshift.common.system_images_registry }}/{{ openshift.node.ovs_system_image }}:{{ openshift_image_tag }}"
     state: latest
+    values:
+      - "DOCKER_SERVICE={{ openshift.docker.service_name }}.service"
diff --git a/roles/openshift_node/tasks/storage_plugins/glusterfs.yml b/roles/openshift_node/tasks/storage_plugins/glusterfs.yml
index 7d8c42ee2..1b8a7ad50 100644
--- a/roles/openshift_node/tasks/storage_plugins/glusterfs.yml
+++ b/roles/openshift_node/tasks/storage_plugins/glusterfs.yml
@@ -3,30 +3,52 @@
   package: name=glusterfs-fuse state=present
   when: not openshift.common.is_atomic | bool
 
-- name: Check for existence of virt_use_fusefs seboolean
-  command: getsebool virt_use_fusefs
-  register: virt_use_fusefs_output
-  when: ansible_selinux and ansible_selinux.status == "enabled"
+- name: Check for existence of fusefs sebooleans
+  command: getsebool {{ item }}
+  register: fusefs_getsebool_status
+  when:
+  - ansible_selinux
+  - ansible_selinux.status == "enabled"
   failed_when: false
   changed_when: false
+  with_items:
+  - virt_use_fusefs
+  - virt_sandbox_use_fusefs
 
 - name: Set seboolean to allow gluster storage plugin access from containers
   seboolean:
-    name: virt_use_fusefs
+    name: "{{ item.item }}"
     state: yes
     persistent: yes
-  when: ansible_selinux and ansible_selinux.status == "enabled" and virt_use_fusefs_output.rc == 0
+  when:
+  - ansible_selinux
+  - ansible_selinux.status == "enabled"
+  - item.rc == 0
+  # We need to detect whether or not the boolean is an alias, since `seboolean`
+  # will error if it is an alias.  We do this by inspecting stdout for the boolean name,
+  # since getsebool prints the resolved name.  (At some point Ansible's seboolean module
+  # should learn to deal with aliases)
+  - item.item in item.stdout  # Boolean does not have an alias.
+  - ansible_python_version | version_compare('3', '<')
+  with_items: "{{ fusefs_getsebool_status.results }}"
 
-- name: Check for existence of virt_sandbox_use_fusefs seboolean
-  command: getsebool virt_sandbox_use_fusefs
-  register: virt_sandbox_use_fusefs_output
-  when: ansible_selinux and ansible_selinux.status == "enabled"
-  failed_when: false
-  changed_when: false
-
-- name: Set seboolean to allow gluster storage plugin access from containers(sandbox)
-  seboolean:
-    name: virt_sandbox_use_fusefs
-    state: yes
-    persistent: yes
-  when: ansible_selinux and ansible_selinux.status == "enabled" and virt_sandbox_use_fusefs_output.rc == 0
+# Workaround for https://github.com/openshift/openshift-ansible/issues/4438
+# Use command module rather than seboolean module to set GlusterFS booleans.
+# TODO: Remove this task and the ansible_python_version comparison in
+# the previous task when the problem has been addressed in current
+# ansible release.
+- name: Set seboolean to allow gluster storage plugin access from containers (python 3)
+  command: >
+    setsebool -P {{ item.item }} on
+  when:
+  - ansible_selinux
+  - ansible_selinux.status == "enabled"
+  - item.rc == 0
+  # We need to detect whether or not the boolean is an alias, since `seboolean`
+  # will error if it is an alias.  We do this by inspecting stdout for the boolean name,
+  # since getsebool prints the resolved name.  (At some point Ansible's seboolean module
+  # should learn to deal with aliases)
+  - item.item in item.stdout  # Boolean does not have an alias.
+  - ('--> off' in item.stdout)  # Boolean is currently off.
+  - ansible_python_version | version_compare('3', '>=')
+  with_items: "{{ fusefs_getsebool_status.results }}"
diff --git a/roles/openshift_node/tasks/storage_plugins/nfs.yml b/roles/openshift_node/tasks/storage_plugins/nfs.yml
index d40ae66cb..7e1035893 100644
--- a/roles/openshift_node/tasks/storage_plugins/nfs.yml
+++ b/roles/openshift_node/tasks/storage_plugins/nfs.yml
@@ -3,24 +3,52 @@
   package: name=nfs-utils state=present
   when: not openshift.common.is_atomic | bool
 
-- name: Check for existence of seboolean
+- name: Check for existence of nfs sebooleans
   command: getsebool {{ item }}
-  register: getsebool_status
-  when: ansible_selinux and ansible_selinux.status == "enabled"
+  register: nfs_getsebool_status
+  when:
+  - ansible_selinux
+  - ansible_selinux.status == "enabled"
   failed_when: false
   changed_when: false
   with_items:
-    - virt_use_nfs
-    - virt_sandbox_use_nfs
+  - virt_use_nfs
+  - virt_sandbox_use_nfs
 
 - name: Set seboolean to allow nfs storage plugin access from containers
   seboolean:
     name: "{{ item.item }}"
     state: yes
     persistent: yes
+  when:
+  - ansible_selinux
+  - ansible_selinux.status == "enabled"
+  - item.rc == 0
   # We need to detect whether or not the boolean is an alias, since `seboolean`
   # will error if it is an alias.  We do this by inspecting stdout for the boolean name,
   # since getsebool prints the resolved name.  (At some point Ansible's seboolean module
   # should learn to deal with aliases)
-  when: ansible_selinux and ansible_selinux.status == "enabled" and item.rc == 0 and item.stdout.find(item.item) != -1
-  with_items: "{{ getsebool_status.results }}"
+  - item.item in item.stdout  # Boolean does not have an alias.
+  - ansible_python_version | version_compare('3', '<')
+  with_items: "{{ nfs_getsebool_status.results }}"
+
+# Workaround for https://github.com/openshift/openshift-ansible/issues/4438
+# Use command module rather than seboolean module to set NFS booleans.
+# TODO: Remove this task and the ansible_python_version comparison in
+# the previous task when the problem has been addressed in current
+# ansible release.
+- name: Set seboolean to allow nfs storage plugin access from containers (python 3)
+  command: >
+    setsebool -P {{ item.item }} on
+  when:
+  - ansible_selinux
+  - ansible_selinux.status == "enabled"
+  - item.rc == 0
+  # We need to detect whether or not the boolean is an alias, since `seboolean`
+  # will error if it is an alias.  We do this by inspecting stdout for the boolean name,
+  # since getsebool prints the resolved name.  (At some point Ansible's seboolean module
+  # should learn to deal with aliases)
+  - item.item in item.stdout  # Boolean does not have an alias.
+  - ('--> off' in item.stdout)  # Boolean is currently off.
+  - ansible_python_version | version_compare('3', '>=')
+  with_items: "{{ nfs_getsebool_status.results }}"
diff --git a/roles/openshift_node/tasks/systemd_units.yml b/roles/openshift_node/tasks/systemd_units.yml
index e3ce5df3d..2ccc28461 100644
--- a/roles/openshift_node/tasks/systemd_units.yml
+++ b/roles/openshift_node/tasks/systemd_units.yml
@@ -34,7 +34,7 @@
 - name: Install Node service file
   template:
     dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node.service"
-    src: "{{ openshift.common.service_type }}-node.service.j2"
+    src: "node.service.j2"
   register: install_node_result
   when: not openshift.common.is_containerized | bool
   notify:
diff --git a/roles/openshift_node/templates/atomic-openshift-node.service.j2 b/roles/openshift_node/templates/atomic-openshift-node.service.j2
deleted file mode 100644
index 80232094a..000000000
--- a/roles/openshift_node/templates/atomic-openshift-node.service.j2
+++ /dev/null
@@ -1,22 +0,0 @@
-[Unit]
-Description=Atomic OpenShift Node
-After={{ openshift.docker.service_name }}.service
-After=openvswitch.service
-Wants={{ openshift.docker.service_name }}.service
-Documentation=https://github.com/openshift/origin
-
-[Service]
-Type=notify
-EnvironmentFile=/etc/sysconfig/atomic-openshift-node
-Environment=GOTRACEBACK=crash
-ExecStart=/usr/bin/openshift start node --config=${CONFIG_FILE} $OPTIONS
-LimitNOFILE=65536
-LimitCORE=infinity
-WorkingDirectory=/var/lib/origin/
-SyslogIdentifier=atomic-openshift-node
-Restart=always
-RestartSec=5s
-OOMScoreAdjust=-999
-
-[Install]
-WantedBy=multi-user.target
diff --git a/roles/openshift_node/templates/node.service.j2 b/roles/openshift_node/templates/node.service.j2
new file mode 100644
index 000000000..e12a52c15
--- /dev/null
+++ b/roles/openshift_node/templates/node.service.j2
@@ -0,0 +1,31 @@
+[Unit]
+Description=OpenShift Node
+After={{ openshift.docker.service_name }}.service
+Wants=openvswitch.service
+After=ovsdb-server.service
+After=ovs-vswitchd.service
+Wants={{ openshift.docker.service_name }}.service
+Documentation=https://github.com/openshift/origin
+Requires=dnsmasq.service
+After=dnsmasq.service
+
+[Service]
+Type=notify
+EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node
+Environment=GOTRACEBACK=crash
+ExecStartPre=/usr/bin/cp /etc/origin/node/node-dnsmasq.conf /etc/dnsmasq.d/
+ExecStartPre=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:/in-addr.arpa/127.0.0.1,/{{ openshift.common.dns_domain }}/127.0.0.1
+ExecStopPost=/usr/bin/rm /etc/dnsmasq.d/node-dnsmasq.conf
+ExecStopPost=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:
+ExecStart=/usr/bin/openshift start node --config=${CONFIG_FILE} $OPTIONS
+LimitNOFILE=65536
+LimitCORE=infinity
+WorkingDirectory=/var/lib/origin/
+SyslogIdentifier={{ openshift.common.service_type }}-node
+Restart=always
+RestartSec=5s
+TimeoutStartSec=300
+OOMScoreAdjust=-999
+
+[Install]
+WantedBy=multi-user.target
diff --git a/roles/openshift_node/templates/node.yaml.v1.j2 b/roles/openshift_node/templates/node.yaml.v1.j2
index f2f929232..351c8c9f6 100644
--- a/roles/openshift_node/templates/node.yaml.v1.j2
+++ b/roles/openshift_node/templates/node.yaml.v1.j2
@@ -1,5 +1,9 @@
 allowDisabledDocker: false
 apiVersion: v1
+{% if openshift.common.version_gte_3_6 %}
+dnsBindAddress: 127.0.0.1:53
+dnsRecursiveResolvConf: /etc/origin/node/resolv.conf
+{% endif %}
 dnsDomain: {{ openshift.common.dns_domain }}
 {% if 'dns_ip' in openshift.node %}
 dnsIP: {{ openshift.node.dns_ip }}
diff --git a/roles/openshift_node/templates/openshift.docker.node.service b/roles/openshift_node/templates/openshift.docker.node.service
index d89b64b06..639b6f6c8 100644
--- a/roles/openshift_node/templates/openshift.docker.node.service
+++ b/roles/openshift_node/templates/openshift.docker.node.service
@@ -12,14 +12,20 @@ After=ovs-vswitchd.service
 Wants={{ openshift.common.service_type }}-master.service
 Requires={{ openshift.common.service_type }}-node-dep.service
 After={{ openshift.common.service_type }}-node-dep.service
+Requires=dnsmasq.service
+After=dnsmasq.service
 
 [Service]
 EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node
 EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node-dep
 ExecStartPre=-/usr/bin/docker rm -f {{ openshift.common.service_type }}-node
-ExecStart=/usr/bin/docker run --name {{ openshift.common.service_type }}-node --rm --privileged --net=host --pid=host --env-file=/etc/sysconfig/{{ openshift.common.service_type }}-node -v /:/rootfs:ro,rslave -e CONFIG_FILE=${CONFIG_FILE} -e OPTIONS=${OPTIONS} -e HOST=/rootfs -e HOST_ETC=/host-etc -v {{ openshift.common.data_dir }}:{{ openshift.common.data_dir }}{{ ':rslave' if openshift.docker.gte_1_10 | default(False) | bool else '' }} -v {{ openshift.common.config_base }}/node:{{ openshift.common.config_base }}/node {% if openshift_cloudprovider_kind | default('') != '' -%} -v {{ openshift.common.config_base }}/cloudprovider:{{ openshift.common.config_base}}/cloudprovider {% endif -%} -v /etc/localtime:/etc/localtime:ro -v /etc/machine-id:/etc/machine-id:ro -v /run:/run -v /sys:/sys:rw -v /sys/fs/cgroup:/sys/fs/cgroup:rw -v /usr/bin/docker:/usr/bin/docker:ro -v /var/lib/docker:/var/lib/docker -v /lib/modules:/lib/modules -v /etc/origin/openvswitch:/etc/openvswitch -v /etc/origin/sdn:/etc/openshift-sdn -v /var/lib/cni:/var/lib/cni -v /etc/systemd/system:/host-etc/systemd/system -v /var/log:/var/log -v /dev:/dev $DOCKER_ADDTL_BIND_MOUNTS {{ openshift.node.node_image }}:${IMAGE_VERSION}
+ExecStartPre=/usr/bin/cp /etc/origin/node/node-dnsmasq.conf /etc/dnsmasq.d/
+ExecStartPre=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:/in-addr.arpa/127.0.0.1,/{{ openshift.common.dns_domain }}/127.0.0.1
+ExecStart=/usr/bin/docker run --name {{ openshift.common.service_type }}-node --rm --privileged --net=host --pid=host --env-file=/etc/sysconfig/{{ openshift.common.service_type }}-node -v /:/rootfs:ro,rslave -e CONFIG_FILE=${CONFIG_FILE} -e OPTIONS=${OPTIONS} -e HOST=/rootfs -e HOST_ETC=/host-etc -v {{ openshift.common.data_dir }}:{{ openshift.common.data_dir }}{{ ':rslave' if openshift.docker.gte_1_10 | default(False) | bool else '' }} -v {{ openshift.common.config_base }}/node:{{ openshift.common.config_base }}/node {% if openshift_cloudprovider_kind | default('') != '' -%} -v {{ openshift.common.config_base }}/cloudprovider:{{ openshift.common.config_base}}/cloudprovider {% endif -%} -v /etc/localtime:/etc/localtime:ro -v /etc/machine-id:/etc/machine-id:ro -v /run:/run -v /sys:/sys:rw -v /sys/fs/cgroup:/sys/fs/cgroup:rw -v /usr/bin/docker:/usr/bin/docker:ro -v /var/lib/docker:/var/lib/docker -v /lib/modules:/lib/modules -v /etc/origin/openvswitch:/etc/openvswitch -v /etc/origin/sdn:/etc/openshift-sdn -v /var/lib/cni:/var/lib/cni -v /etc/systemd/system:/host-etc/systemd/system -v /var/log:/var/log -v /dev:/dev $DOCKER_ADDTL_BIND_MOUNTS -v /etc/pki:/etc/pki:ro {{ openshift.node.node_image }}:${IMAGE_VERSION}
 ExecStartPost=/usr/bin/sleep 10
 ExecStop=/usr/bin/docker stop {{ openshift.common.service_type }}-node
+ExecStopPost=/usr/bin/rm /etc/dnsmasq.d/node-dnsmasq.conf
+ExecStopPost=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:
 SyslogIdentifier={{ openshift.common.service_type }}-node
 Restart=always
 RestartSec=5s
diff --git a/roles/openshift_node/templates/origin-node.service.j2 b/roles/openshift_node/templates/origin-node.service.j2
deleted file mode 100644
index 8047301e6..000000000
--- a/roles/openshift_node/templates/origin-node.service.j2
+++ /dev/null
@@ -1,21 +0,0 @@
-[Unit]
-Description=Origin Node
-After={{ openshift.docker.service_name }}.service
-Wants={{ openshift.docker.service_name }}.service
-Documentation=https://github.com/openshift/origin
-
-[Service]
-Type=notify
-EnvironmentFile=/etc/sysconfig/origin-node
-Environment=GOTRACEBACK=crash
-ExecStart=/usr/bin/openshift start node --config=${CONFIG_FILE} $OPTIONS
-LimitNOFILE=65536
-LimitCORE=infinity
-WorkingDirectory=/var/lib/origin/
-SyslogIdentifier=origin-node
-Restart=always
-RestartSec=5s
-OOMScoreAdjust=-999
-
-[Install]
-WantedBy=multi-user.target