Merge pull request #5230 from kwoodson/openshift_node_bootstrap

Merged by openshift-bot

8 files changed, 292 insertions, 187 deletions

diff --git a/roles/openshift_node/defaults/main.yml b/roles/openshift_node/defaults/main.yml
index cc000496a..a7dad5b1f 100644
--- a/roles/openshift_node/defaults/main.yml
+++ b/roles/openshift_node/defaults/main.yml
@@ -1,6 +1,64 @@
 ---
 r_openshift_node_firewall_enabled: "{{ os_firewall_enabled | default(True) }}"
 r_openshift_node_use_firewalld: "{{ os_firewall_use_firewalld | default(False) }}"
+
+openshift_service_type: "{{ openshift.common.service_type }}"
+
+openshift_image_tag: ''
+
+openshift_node_ami_prep_packages:
+- "{{ openshift_service_type }}-master"
+- "{{ openshift_service_type }}-node"
+- "{{ openshift_service_type }}-docker-excluder"
+- "{{ openshift_service_type }}-sdn-ovs"
+- ansible
+- openvswitch
+- docker
+- etcd
+#- pcs
+- haproxy
+- dnsmasq
+- ntp
+- logrotate
+- httpd-tools
+- bind
+- firewalld
+- libselinux-python
+- conntrack-tools
+- openssl
+- cloud-init
+- iproute
+- python-dbus
+- PyYAML
+- yum-utils
+- python2-boto
+- python2-boto3
+- cloud-utils-growpart
+# gluster
+- glusterfs-fuse
+- heketi-client
+# nfs
+- nfs-utils
+- flannel
+- bash-completion
+# cockpit
+- cockpit-ws
+- cockpit-system
+- cockpit-bridge
+- cockpit-docker
+# iscsi
+- iscsi-initiator-utils
+# ceph
+- ceph-common
+# systemcontainer
+# - runc
+# - container-selinux
+# - atomic
+#
+openshift_deployment_type: origin
+
+openshift_node_bootstrap: False
+
 r_openshift_node_os_firewall_deny: []
 r_openshift_node_os_firewall_allow:
 - service: Kubernetes kubelet
diff --git a/roles/openshift_node/handlers/main.yml b/roles/openshift_node/handlers/main.yml
index f2c45a4bd..14ba48aba 100644
--- a/roles/openshift_node/handlers/main.yml
+++ b/roles/openshift_node/handlers/main.yml
@@ -27,6 +27,7 @@
   when:
   - (not skip_node_svc_handlers | default(False) | bool)
   - not (node_service_status_changed | default(false) | bool)
+  - not openshift_node_bootstrap
 
 - name: reload sysctl.conf
   command: /sbin/sysctl -p
diff --git a/roles/openshift_node/meta/main.yml b/roles/openshift_node/meta/main.yml
index 06373de04..3db980514 100644
--- a/roles/openshift_node/meta/main.yml
+++ b/roles/openshift_node/meta/main.yml
@@ -19,6 +19,7 @@ dependencies:
 - role: openshift_clock
 - role: openshift_docker
 - role: openshift_node_certificates
+  when: not openshift_node_bootstrap
 - role: openshift_cloud_provider
 - role: openshift_node_dnsmasq
   when: openshift.common.use_dnsmasq | bool
diff --git a/roles/openshift_node/tasks/bootstrap.yml b/roles/openshift_node/tasks/bootstrap.yml
new file mode 100644
index 000000000..cb1440283
--- /dev/null
+++ b/roles/openshift_node/tasks/bootstrap.yml
@@ -0,0 +1,55 @@
+---
+- name: install needed rpm(s)
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items: "{{ openshift_node_ami_prep_packages }}"
+
+- name: create the directory for node
+  file:
+    state: directory
+    path: "/etc/systemd/system/{{ openshift_service_type }}-node.service.d"
+
+- name: laydown systemd override
+  copy:
+    dest: "/etc/systemd/system/{{ openshift_service_type }}-node.service.d/override.conf"
+    content: |
+      [Unit]
+      After=cloud-init.service
+
+- name: update the sysconfig to have KUBECONFIG
+  lineinfile:
+    dest: "/etc/sysconfig/{{ openshift_service_type }}-node"
+    line: "KUBECONFIG=/root/csr_kubeconfig"
+    regexp: "^KUBECONFIG=.*"
+
+- name: update the ExecStart to have bootstrap
+  lineinfile:
+    dest: "/usr/lib/systemd/system/{{ openshift_service_type }}-node.service"
+    line: "{% raw %}ExecStart=/usr/bin/openshift start node --bootstrap --kubeconfig=${KUBECONFIG} $OPTIONS{% endraw %}"
+    regexp: "^ExecStart=.*"
+
+- name: "systemctl enable {{ openshift_service_type }}-node"
+  systemd:
+    name: "{{ item }}"
+    enabled: no
+  with_items:
+  - "{{ openshift_service_type }}-node.service"
+  - "{{ openshift_service_type }}-master.service"
+
+- name: Check for RPM generated config marker file .config_managed
+  stat:
+    path: /etc/origin/.config_managed
+  register: rpmgenerated_config
+
+- name: Remove RPM generated config files if present
+  file:
+    path: "/etc/origin/{{ item }}"
+    state: absent
+  when:
+  - rpmgenerated_config.stat.exists
+  - openshift_deployment_type in ['openshift-enterprise', 'atomic-enterprise']
+  with_items:
+  - master
+  - node
+  - .config_managed
diff --git a/roles/openshift_node/tasks/config.yml b/roles/openshift_node/tasks/config.yml
new file mode 100644
index 000000000..8210fd881
--- /dev/null
+++ b/roles/openshift_node/tasks/config.yml
@@ -0,0 +1,111 @@
+---
+- name: Install the systemd units
+  include: systemd_units.yml
+
+- name: Check for tuned package
+  command: rpm -q tuned
+  args:
+    warn: no
+  register: tuned_installed
+  changed_when: false
+  failed_when: false
+
+- name: Set atomic-guest tuned profile
+  command: "tuned-adm profile atomic-guest"
+  when: tuned_installed.rc == 0 and openshift.common.is_atomic | bool
+
+- name: Start and enable openvswitch service
+  systemd:
+    name: openvswitch.service
+    enabled: yes
+    state: started
+    daemon_reload: yes
+  when:
+    - openshift.common.is_containerized | bool
+    - openshift.common.use_openshift_sdn | default(true) | bool
+  register: ovs_start_result
+  until: not ovs_start_result | failed
+  retries: 3
+  delay: 30
+
+- set_fact:
+    ovs_service_status_changed: "{{ ovs_start_result | changed }}"
+
+- file:
+    dest: "{{ (openshift_node_kubelet_args|default({'config':None})).config}}"
+    state: directory
+  when: openshift_node_kubelet_args is defined and 'config' in openshift_node_kubelet_args
+
+# TODO: add the validate parameter when there is a validation command to run
+- name: Create the Node config
+  template:
+    dest: "{{ openshift.common.config_base }}/node/node-config.yaml"
+    src: node.yaml.v1.j2
+    backup: true
+    owner: root
+    group: root
+    mode: 0600
+  notify:
+    - restart node
+
+- name: Configure Node Environment Variables
+  lineinfile:
+    dest: /etc/sysconfig/{{ openshift.common.service_type }}-node
+    regexp: "^{{ item.key }}="
+    line: "{{ item.key }}={{ item.value }}"
+    create: true
+  with_dict: "{{ openshift.node.env_vars | default({}) }}"
+  notify:
+    - restart node
+
+# Necessary because when you're on a node that's also a master the master will be
+# restarted after the node restarts docker and it will take up to 60 seconds for
+# systemd to start the master again
+- when: openshift.common.is_containerized | bool
+  block:
+    - name: Wait for master API to become available before proceeding
+      # Using curl here since the uri module requires python-httplib2 and
+      # wait_for port doesn't provide health information.
+      command: >
+        curl --silent --tlsv1.2 --cacert {{ openshift.common.config_base }}/node/ca.crt
+        {{ openshift_node_master_api_url }}/healthz/ready
+      args:
+        # Disables the following warning:
+        # Consider using get_url or uri module rather than running curl
+        warn: no
+      register: api_available_output
+      until: api_available_output.stdout == 'ok'
+      retries: 120
+      delay: 1
+      changed_when: false
+
+    - name: Start and enable node dep
+      systemd:
+        daemon_reload: yes
+        name: "{{ openshift.common.service_type }}-node-dep"
+        enabled: yes
+        state: started
+
+- name: Start and enable node
+  systemd:
+    name: "{{ openshift.common.service_type }}-node"
+    enabled: yes
+    state: started
+    daemon_reload: yes
+  register: node_start_result
+  until: not node_start_result | failed
+  retries: 1
+  delay: 30
+  ignore_errors: true
+
+- name: Dump logs from node service if it failed
+  command: journalctl --no-pager -n 100 -u {{ openshift.common.service_type }}-node
+  when: node_start_result | failed
+
+- name: Abort if node failed to start
+  fail:
+    msg: Node failed to start please inspect the logs and try again
+  when: node_start_result | failed
+
+- set_fact:
+    node_service_status_changed: "{{ node_start_result | changed }}"
diff --git a/roles/openshift_node/tasks/install.yml b/roles/openshift_node/tasks/install.yml
new file mode 100644
index 000000000..9bf4ed879
--- /dev/null
+++ b/roles/openshift_node/tasks/install.yml
@@ -0,0 +1,33 @@
+---
+# We have to add tuned-profiles in the same transaction otherwise we run into depsolving
+# problems because the rpms don't pin the version properly. This was fixed in 3.1 packaging.
+- when: not openshift.common.is_containerized | bool
+  block:
+  - name: Install Node package
+    package:
+      name: "{{ openshift.common.service_type }}-node{{ openshift_pkg_version | default('') | oo_image_tag_to_rpm_version(include_dash=True) }},tuned-profiles-{{ openshift.common.service_type }}-node{{ openshift_pkg_version | default('') | oo_image_tag_to_rpm_version(include_dash=True) }}"
+      state: present
+
+  - name: Install sdn-ovs package
+    package:
+      name: "{{ openshift.common.service_type }}-sdn-ovs{{ openshift_pkg_version | oo_image_tag_to_rpm_version(include_dash=True) }}"
+      state: present
+    when:
+    - openshift.common.use_openshift_sdn | default(true) | bool
+
+  - name: Install conntrack-tools package
+    package:
+      name: "conntrack-tools"
+      state: present
+
+- when:
+  - openshift.common.is_containerized | bool
+  - not openshift.common.is_node_system_container | bool
+  block:
+  - name: Pre-pull node image when containerized
+    command: >
+      docker pull {{ openshift.node.node_image }}:{{ openshift_image_tag }}
+    register: pull_result
+    changed_when: "'Downloaded newer image' in pull_result.stdout"
+
+  - include: config/install-node-docker-service-file.yml
diff --git a/roles/openshift_node/tasks/main.yml b/roles/openshift_node/tasks/main.yml
index 525dd1d1a..60a25dcc6 100644
--- a/roles/openshift_node/tasks/main.yml
+++ b/roles/openshift_node/tasks/main.yml
@@ -1,5 +1,4 @@
 ---
-# TODO: allow for overriding default ports where possible
 - fail:
     msg: "SELinux is disabled, This deployment type requires that SELinux is enabled."
   when:
@@ -10,34 +9,7 @@
   include: firewall.yml
   static: yes
 
-- name: Set node facts
-  openshift_facts:
-    role: "{{ item.role }}"
-    local_facts: "{{ item.local_facts }}"
-  with_items:
-    # Reset node labels to an empty dictionary.
-    - role: node
-      local_facts:
-        labels: {}
-    - role: node
-      local_facts:
-        annotations: "{{ openshift_node_annotations | default(none) }}"
-        debug_level: "{{ openshift_node_debug_level | default(openshift.common.debug_level) }}"
-        iptables_sync_period: "{{ openshift_node_iptables_sync_period | default(None) }}"
-        kubelet_args: "{{ openshift_node_kubelet_args | default(None) }}"
-        labels: "{{ lookup('oo_option', 'openshift_node_labels') | default( openshift_node_labels | default(none), true) }}"
-        registry_url: "{{ oreg_url_node | default(oreg_url) | default(None) }}"
-        schedulable: "{{ openshift_schedulable | default(openshift_scheduleable) | default(None) }}"
-        sdn_mtu: "{{ openshift_node_sdn_mtu | default(None) }}"
-        storage_plugin_deps: "{{ osn_storage_plugin_deps | default(None) }}"
-        set_node_ip: "{{ openshift_set_node_ip | default(None) }}"
-        node_image: "{{ osn_image | default(None) }}"
-        ovs_image: "{{ osn_ovs_image | default(None) }}"
-        proxy_mode: "{{ openshift_node_proxy_mode | default('iptables') }}"
-        local_quota_per_fsgroup: "{{ openshift_node_local_quota_per_fsgroup | default(None) }}"
-        dns_ip: "{{ openshift_dns_ip | default(none) | get_dns_ip(hostvars[inventory_hostname])}}"
-        env_vars: "{{ openshift_node_env_vars | default(None) }}"
-
+#### Disable SWAP #####
 # https://docs.openshift.com/container-platform/3.4/admin_guide/overcommit.html#disabling-swap-memory
 - name: Check for swap usage
   command: grep "^[^#].*swap" /etc/fstab
@@ -46,9 +18,10 @@
   failed_when: false
   register: swap_result
 
-# Disable Swap Block
-- block:
-
+- when:
+    - swap_result.stdout_lines | length > 0
+    - openshift_disable_swap | default(true) | bool
+  block:
     - name: Disable swap
       command: swapoff --all
 
@@ -64,29 +37,10 @@
         dest: /etc/fstab
         line: '# OpenShift-Ansible Installer disabled swap per overcommit guidelines'
         state: present
+#### End Disable Swap Block ####
 
-  when:
-    - swap_result.stdout_lines | length > 0
-    - openshift_disable_swap | default(true) | bool
-# End Disable Swap Block
-
-- name: Install Node package
-  package:
-    name: "{{ openshift.common.service_type }}-node{{ openshift_pkg_version | default('') | oo_image_tag_to_rpm_version(include_dash=True) }}"
-    state: present
-  when: not openshift.common.is_containerized | bool
-
-- name: setup tuned
-  include: tuned.yml
-  static: yes
-
-- name: Install sdn-ovs package
-  package:
-    name: "{{ openshift.common.service_type }}-sdn-ovs{{ openshift_pkg_version | oo_image_tag_to_rpm_version(include_dash=True) }}"
-    state: present
-  when:
-    - openshift.common.use_openshift_sdn | default(true) | bool
-    - not openshift.common.is_containerized | bool
+- name: include node installer
+  include: install.yml
 
 - name: Restart cri-o
   systemd:
@@ -95,15 +49,6 @@
     state: restarted
   when: openshift_use_crio | default(false)
 
-- name: Install conntrack-tools package
-  package:
-    name: "conntrack-tools"
-    state: present
-  when: not openshift.common.is_containerized | bool
-
-- name: Install the systemd units
-  include: systemd_units.yml
-
 # The atomic-openshift-node service will set this parameter on
 # startup, but if the network service is restarted this setting is
 # lost. Reference: https://bugzilla.redhat.com/show_bug.cgi?id=1372388
@@ -116,39 +61,13 @@
   notify:
     - reload sysctl.conf
 
-- name: Start and enable openvswitch service
-  systemd:
-    name: openvswitch.service
-    enabled: yes
-    state: started
-    daemon_reload: yes
-  when:
-    - openshift.common.is_containerized | bool
-    - openshift.common.use_openshift_sdn | default(true) | bool
-  register: ovs_start_result
-  until: not ovs_start_result | failed
-  retries: 3
-  delay: 30
-
-- set_fact:
-    ovs_service_status_changed: "{{ ovs_start_result | changed }}"
+- name: include bootstrap node config
+  include: bootstrap.yml
+  when: openshift_node_bootstrap
 
-- file:
-    dest: "{{ (openshift_node_kubelet_args|default({'config':None})).config}}"
-    state: directory
-  when: openshift_node_kubelet_args is defined and 'config' in openshift_node_kubelet_args
-
-# TODO: add the validate parameter when there is a validation command to run
-- name: Create the Node config
-  template:
-    dest: "{{ openshift.common.config_base }}/node/node-config.yaml"
-    src: node.yaml.v1.j2
-    backup: true
-    owner: root
-    group: root
-    mode: 0600
-  notify:
-    - restart node
+- name: include standard node config
+  include: config.yml
+  when: not openshift_node_bootstrap
 
 - name: Check for credentials file for registry auth
   stat:
@@ -181,16 +100,7 @@
   notify:
     - restart node
 
-- name: Configure Node Environment Variables
-  lineinfile:
-    dest: /etc/sysconfig/{{ openshift.common.service_type }}-node
-    regexp: "^{{ item.key }}="
-    line: "{{ item.key }}={{ item.value }}"
-    create: true
-  with_dict: "{{ openshift.node.env_vars | default({}) }}"
-  notify:
-    - restart node
-
+#### Storage class plugins here ####
 - name: NFS storage plugin configuration
   include: storage_plugins/nfs.yml
   tags:
@@ -208,55 +118,7 @@
   include: storage_plugins/iscsi.yml
   when: "'iscsi' in openshift.node.storage_plugin_deps"
 
-# Necessary because when you're on a node that's also a master the master will be
-# restarted after the node restarts docker and it will take up to 60 seconds for
-# systemd to start the master again
-- name: Wait for master API to become available before proceeding
-  # Using curl here since the uri module requires python-httplib2 and
-  # wait_for port doesn't provide health information.
-  command: >
-    curl --silent --tlsv1.2 --cacert {{ openshift.common.config_base }}/node/ca.crt
-    {{ openshift_node_master_api_url }}/healthz/ready
-  args:
-    # Disables the following warning:
-    # Consider using get_url or uri module rather than running curl
-    warn: no
-  register: api_available_output
-  until: api_available_output.stdout == 'ok'
-  retries: 120
-  delay: 1
-  changed_when: false
-  when: openshift.common.is_containerized | bool
-
-- name: Start and enable node dep
-  systemd:
-    daemon_reload: yes
-    name: "{{ openshift.common.service_type }}-node-dep"
-    enabled: yes
-    state: started
-  when: openshift.common.is_containerized | bool
-
-
-- name: Start and enable node
-  systemd:
-    name: "{{ openshift.common.service_type }}-node"
-    enabled: yes
-    state: started
-    daemon_reload: yes
-  register: node_start_result
-  until: not node_start_result | failed
-  retries: 1
-  delay: 30
-  ignore_errors: true
-
-- name: Dump logs from node service if it failed
-  command: journalctl --no-pager -n 100 -u {{ openshift.common.service_type }}-node
-  when: node_start_result | failed
-
-- name: Abort if node failed to start
-  fail:
-    msg: Node failed to start please inspect the logs and try again
-  when: node_start_result | failed
+##### END Storage #####
 
-- set_fact:
-    node_service_status_changed: "{{ node_start_result | changed }}"
+- include: config/workaround-bz1331590-ovs-oom-fix.yml
+  when: openshift.common.use_openshift_sdn | default(true) | bool
diff --git a/roles/openshift_node/tasks/systemd_units.yml b/roles/openshift_node/tasks/systemd_units.yml
index b86bb1549..4687400cd 100644
--- a/roles/openshift_node/tasks/systemd_units.yml
+++ b/roles/openshift_node/tasks/systemd_units.yml
@@ -1,22 +1,6 @@
 ---
 # This file is included both in the openshift_master role and in the upgrade
 # playbooks.
-
-- include: config/install-node-deps-docker-service-file.yml
-  when: openshift.common.is_containerized | bool
-
-- block:
-  - name: Pre-pull node image
-    command: >
-      docker pull {{ openshift.node.node_image }}:{{ openshift_image_tag }}
-    register: pull_result
-    changed_when: "'Downloaded newer image' in pull_result.stdout"
-
-  - include: config/install-node-docker-service-file.yml
-  when:
-  - openshift.common.is_containerized | bool
-  - not openshift.common.is_node_system_container | bool
-
 - name: Install Node service file
   template:
     dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node.service"
@@ -26,24 +10,24 @@
   - reload systemd units
   - restart node
 
-- include: config/install-ovs-service-env-file.yml
-  when: openshift.common.is_containerized | bool
+- when: openshift.common.is_containerized | bool
+  block:
+  - name: include node deps docker service file
+    include: config/install-node-deps-docker-service-file.yml
 
-- name: Install Node system container
-  include: node_system_container.yml
-  when:
-  - openshift.common.is_containerized | bool
-  - openshift.common.is_node_system_container | bool
+  - name: include ovs service environment file
+    include: config/install-ovs-service-env-file.yml
 
-- name: Install OpenvSwitch system containers
-  include: openvswitch_system_container.yml
-  when:
-  - openshift.common.use_openshift_sdn | default(true) | bool
-  - openshift.common.is_containerized | bool
-  - openshift.common.is_openvswitch_system_container | bool
+  - name: Install Node system container
+    include: node_system_container.yml
+    when:
+    - openshift.common.is_node_system_container | bool
 
-- include: config/workaround-bz1331590-ovs-oom-fix.yml
-  when: openshift.common.use_openshift_sdn | default(true) | bool
+  - name: Install OpenvSwitch system containers
+    include: openvswitch_system_container.yml
+    when:
+    - openshift.common.use_openshift_sdn | default(true) | bool
+    - openshift.common.is_openvswitch_system_container | bool
 
 - block:
   - name: Pre-pull openvswitch image