Merge pull request #5361 from kwoodson/fix_bootstrap_files

Provisioning updates.

9 files changed, 178 insertions, 60 deletions

diff --git a/roles/openshift_node/defaults/main.yml b/roles/openshift_node/defaults/main.yml
index 739b0d968..b310a8f64 100644
--- a/roles/openshift_node/defaults/main.yml
+++ b/roles/openshift_node/defaults/main.yml
@@ -4,7 +4,8 @@ openshift_node_debug_level: "{{ debug_level | default(2) }}"
 r_openshift_node_firewall_enabled: "{{ os_firewall_enabled | default(True) }}"
 r_openshift_node_use_firewalld: "{{ os_firewall_use_firewalld | default(False) }}"
 
-openshift_service_type: "{{ openshift.common.service_type }}"
+openshift_deployment_type: "{{ openshift_deployment_type | default('origin') }}"
+openshift_service_type: "{{ 'origin' if openshift_deployment_type == 'origin' else 'atomic-openshift' }}"
 
 openshift_image_tag: ''
 
@@ -17,7 +18,6 @@ openshift_node_ami_prep_packages:
 - openvswitch
 - docker
 - etcd
-#- pcs
 - haproxy
 - dnsmasq
 - ntp
@@ -54,7 +54,6 @@ openshift_node_ami_prep_packages:
 # - container-selinux
 # - atomic
 #
-openshift_deployment_type: origin
 
 openshift_node_bootstrap: False
 
diff --git a/roles/openshift_node/files/bootstrap.yml b/roles/openshift_node/files/bootstrap.yml
new file mode 100644
index 000000000..ea280640f
--- /dev/null
+++ b/roles/openshift_node/files/bootstrap.yml
@@ -0,0 +1,63 @@
+#!/usr/bin/ansible-playbook
+---
+- hosts: localhost
+  gather_facts: yes
+  vars:
+    origin_dns:
+      file: /etc/dnsmasq.d/origin-dns.conf
+      lines:
+      - regex: ^listen-address
+        state: present
+        line: "listen-address={{ ansible_default_ipv4.address }}"
+    node_dns:
+      file: /etc/dnsmasq.d/node-dnsmasq.conf
+      lines:
+      - regex: "^server=/in-addr.arpa/127.0.0.1$"
+        line: server=/in-addr.arpa/127.0.0.1
+      - regex: "^server=/cluster.local/127.0.0.1$"
+        line: server=/cluster.local/127.0.0.1
+
+  tasks:
+  - include_vars: openshift_settings.yaml
+
+  - name: set the data for node_dns
+    lineinfile:
+      create: yes
+      insertafter: EOF
+      path: "{{ node_dns.file }}"
+      regexp: "{{ item.regex }}"
+      line: "{{ item.line | default(omit) }}"
+    with_items: "{{ node_dns.lines }}"
+
+  - name: set the data for origin_dns
+    lineinfile:
+      create: yes
+      state: "{{ item.state | default('present') }}"
+      insertafter: "{{ item.after | default(omit) }}"
+      path: "{{ origin_dns.file }}"
+      regexp: "{{ item.regex }}"
+      line: "{{ item.line | default(omit)}}"
+    with_items: "{{ origin_dns.lines }}"
+
+  - when:
+    - openshift_group_type is defined
+    - openshift_group_type != ''
+    - openshift_group_type != 'master'
+    block:
+    - name: determine the openshift_service_type
+      stat:
+        path: /etc/sysconfig/atomic-openshift-node
+      register: service_type_results
+
+    - name: set openshift_service_type fact based on stat results
+      set_fact:
+        openshift_service_type: "{{ service_type_results.stat.exists | ternary('atomic-openshift', 'origin') }}"
+
+    - name: update the sysconfig to have necessary variables
+      lineinfile:
+        dest: "/etc/sysconfig/{{ openshift_service_type }}-node"
+        line: "{{ item.line }}"
+        regexp: "{{ item.regexp }}"
+      with_items:
+      - line: "BOOTSTRAP_CONFIG_NAME=node-config-{{ openshift_group_type }}"
+        regexp: "^BOOTSTRAP_CONFIG_NAME=.*"
diff --git a/roles/openshift_node/handlers/main.yml b/roles/openshift_node/handlers/main.yml
index 25a6fc721..b102c1b18 100644
--- a/roles/openshift_node/handlers/main.yml
+++ b/roles/openshift_node/handlers/main.yml
@@ -3,7 +3,11 @@
   systemd:
     name: openvswitch
     state: restarted
-  when: (not skip_node_svc_handlers | default(False) | bool) and not (ovs_service_status_changed | default(false) | bool) and openshift_node_use_openshift_sdn | bool
+  when:
+  - (not skip_node_svc_handlers | default(False) | bool)
+  - not (ovs_service_status_changed | default(false) | bool)
+  - openshift_node_use_openshift_sdn | bool
+  - not openshift_node_bootstrap
   register: l_openshift_node_stop_openvswitch_result
   until: not l_openshift_node_stop_openvswitch_result | failed
   retries: 3
@@ -11,10 +15,11 @@
   notify:
   - restart openvswitch pause
 
-
 - name: restart openvswitch pause
   pause: seconds=15
-  when: (not skip_node_svc_handlers | default(False) | bool) and openshift.common.is_containerized | bool
+  when:
+  - (not skip_node_svc_handlers | default(False) | bool)
+  - openshift.common.is_containerized | bool
 
 - name: restart node
   systemd:
diff --git a/roles/openshift_node/tasks/aws.yml b/roles/openshift_node/tasks/aws.yml
new file mode 100644
index 000000000..38c2b794d
--- /dev/null
+++ b/roles/openshift_node/tasks/aws.yml
@@ -0,0 +1,21 @@
+---
+- name: Configure AWS Cloud Provider Settings
+  lineinfile:
+    dest: /etc/sysconfig/{{ openshift.common.service_type }}-node
+    regexp: "{{ item.regex }}"
+    line: "{{ item.line }}"
+    create: true
+  with_items:
+    - regex: '^AWS_ACCESS_KEY_ID='
+      line: "AWS_ACCESS_KEY_ID={{ openshift_cloudprovider_aws_access_key | default('') }}"
+    - regex: '^AWS_SECRET_ACCESS_KEY='
+      line: "AWS_SECRET_ACCESS_KEY={{ openshift_cloudprovider_aws_secret_key | default('') }}"
+  register: sys_env_update
+  no_log: True
+  when:
+    - openshift_cloudprovider_kind is defined
+    - openshift_cloudprovider_kind == 'aws'
+    - openshift_cloudprovider_aws_access_key is defined
+    - openshift_cloudprovider_aws_secret_key is defined
+  notify:
+    - restart node
diff --git a/roles/openshift_node/tasks/bootstrap.yml b/roles/openshift_node/tasks/bootstrap.yml
index 6bd2df362..8c03f6c41 100644
--- a/roles/openshift_node/tasks/bootstrap.yml
+++ b/roles/openshift_node/tasks/bootstrap.yml
@@ -17,17 +17,29 @@
       [Unit]
       After=cloud-init.service
 
-- name: update the sysconfig to have KUBECONFIG
+- name: update the sysconfig to have necessary variables
   lineinfile:
     dest: "/etc/sysconfig/{{ openshift_service_type }}-node"
-    line: "KUBECONFIG=/root/csr_kubeconfig"
+    line: "{{ item.line | default(omit) }}"
+    regexp: "{{ item.regexp }}"
+    state: "{{ item.state | default('present') }}"
+  with_items:
+  # add the kubeconfig
+  - line: "KUBECONFIG=/etc/origin/node/csr_kubeconfig"
     regexp: "^KUBECONFIG=.*"
+  # remove the config file.  This comes from openshift_facts
+  - regexp: "^CONFIG_FILE=.*"
+    state: absent
 
-- name: update the ExecStart to have bootstrap
-  lineinfile:
-    dest: "/usr/lib/systemd/system/{{ openshift_service_type }}-node.service"
-    line: "{% raw %}ExecStart=/usr/bin/openshift start node --bootstrap --kubeconfig=${KUBECONFIG} $OPTIONS{% endraw %}"
-    regexp: "^ExecStart=.*"
+- name: include aws sysconfig credentials
+  include: aws.yml
+  static: yes
+
+#- name: update the ExecStart to have bootstrap
+#  lineinfile:
+#    dest: "/usr/lib/systemd/system/{{ openshift_service_type }}-node.service"
+#    line: "{% raw %}ExecStart=/usr/bin/openshift start node --bootstrap --kubeconfig=${KUBECONFIG} $OPTIONS{% endraw %}"
+#    regexp: "^ExecStart=.*"
 
 - name: "disable {{ openshift_service_type }}-node and {{ openshift_service_type }}-master services"
   systemd:
@@ -42,6 +54,30 @@
     path: /etc/origin/.config_managed
   register: rpmgenerated_config
 
+- name: create directories for bootstrapping
+  file:
+    state: directory
+    dest: "{{ item }}"
+  with_items:
+  - /root/openshift_bootstrap
+  - /var/lib/origin/openshift.local.config
+  - /var/lib/origin/openshift.local.config/node
+  - "/etc/docker/certs.d/docker-registry.default.svc:5000"
+
+- name: laydown the bootstrap.yml file for on boot configuration
+  copy:
+    src: bootstrap.yml
+    dest: /root/openshift_bootstrap/bootstrap.yml
+
+- name: symlink master ca for docker-registry
+  file:
+    src: "{{ item }}"
+    dest: "/etc/docker/certs.d/docker-registry.default.svc:5000/{{ item | basename }}"
+    state: link
+    force: yes
+  with_items:
+  - /var/lib/origin/openshift.local.config/node/node-client-ca.crt
+
 - when: rpmgenerated_config.stat.exists
   block:
   - name: Remove RPM generated config files if present
@@ -50,6 +86,7 @@
       state: absent
     with_items:
     - master
+    - .config_managed
 
   # with_fileglob doesn't work correctly due to a few issues.
   # Could change this to fileglob when it gets fixed.
@@ -62,5 +99,7 @@
     file:
       path: "{{ item.path }}"
       state: absent
-    when: "'resolv.conf' not in item.path or 'node-dnsmasq.conf' not in item.path"
+    when:
+    - "'resolv.conf' not in item.path"
+    - "'node-dnsmasq.conf' not in item.path"
     with_items: "{{ find_results.files }}"
diff --git a/roles/openshift_node/tasks/config.yml b/roles/openshift_node/tasks/config.yml
index e5fcaf9af..c08f43118 100644
--- a/roles/openshift_node/tasks/config.yml
+++ b/roles/openshift_node/tasks/config.yml
@@ -46,26 +46,16 @@
   notify:
     - restart node
 
-- name: Configure AWS Cloud Provider Settings
-  lineinfile:
-    dest: /etc/sysconfig/{{ openshift.common.service_type }}-node
-    regexp: "{{ item.regex }}"
-    line: "{{ item.line }}"
-    create: true
-  with_items:
-    - regex: '^AWS_ACCESS_KEY_ID='
-      line: "AWS_ACCESS_KEY_ID={{ openshift_cloudprovider_aws_access_key | default('') }}"
-    - regex: '^AWS_SECRET_ACCESS_KEY='
-      line: "AWS_SECRET_ACCESS_KEY={{ openshift_cloudprovider_aws_secret_key | default('') }}"
-  no_log: True
-  when: openshift_cloudprovider_kind is defined and openshift_cloudprovider_kind == 'aws' and openshift_cloudprovider_aws_access_key is defined and openshift_cloudprovider_aws_secret_key is defined
-  notify:
-    - restart node
+- name: include aws provider credentials
+  include: aws.yml
+  static: yes
 
 # Necessary because when you're on a node that's also a master the master will be
 # restarted after the node restarts docker and it will take up to 60 seconds for
 # systemd to start the master again
-- when: openshift.common.is_containerized | bool
+- when:
+    - openshift.common.is_containerized | bool
+    - not openshift_node_bootstrap
   block:
     - name: Wait for master API to become available before proceeding
       # Using curl here since the uri module requires python-httplib2 and
@@ -90,26 +80,28 @@
         enabled: yes
         state: started
 
-- name: Start and enable node
-  systemd:
-    name: "{{ openshift.common.service_type }}-node"
-    enabled: yes
-    state: started
-    daemon_reload: yes
-  register: node_start_result
-  until: not node_start_result | failed
-  retries: 1
-  delay: 30
-  ignore_errors: true
+- when: not openshift_node_bootstrap
+  block:
+    - name: Start and enable node
+      systemd:
+        name: "{{ openshift.common.service_type }}-node"
+        enabled: yes
+        state: started
+        daemon_reload: yes
+      register: node_start_result
+      until: not node_start_result | failed
+      retries: 1
+      delay: 30
+      ignore_errors: true
 
-- name: Dump logs from node service if it failed
-  command: journalctl --no-pager -n 100 -u {{ openshift.common.service_type }}-node
-  when: node_start_result | failed
+    - name: Dump logs from node service if it failed
+      command: journalctl --no-pager -n 100 -u {{ openshift.common.service_type }}-node
+      when: node_start_result | failed
 
-- name: Abort if node failed to start
-  fail:
-    msg: Node failed to start please inspect the logs and try again
-  when: node_start_result | failed
+    - name: Abort if node failed to start
+      fail:
+        msg: Node failed to start please inspect the logs and try again
+      when: node_start_result | failed
 
-- set_fact:
-    node_service_status_changed: "{{ node_start_result | changed }}"
+    - set_fact:
+        node_service_status_changed: "{{ node_start_result | changed }}"
diff --git a/roles/openshift_node/tasks/install.yml b/roles/openshift_node/tasks/install.yml
index 1539d6e3b..6b7e40491 100644
--- a/roles/openshift_node/tasks/install.yml
+++ b/roles/openshift_node/tasks/install.yml
@@ -3,12 +3,12 @@
   block:
   - name: Install Node package
     package:
-      name: "{{ openshift.common.service_type }}-node{{ openshift_pkg_version | default('') | oo_image_tag_to_rpm_version(include_dash=True) }}"
+      name: "{{ openshift.common.service_type }}-node{{ (openshift_pkg_version | default('')) | oo_image_tag_to_rpm_version(include_dash=True) }}"
       state: present
 
   - name: Install sdn-ovs package
     package:
-      name: "{{ openshift.common.service_type }}-sdn-ovs{{ openshift_pkg_version | oo_image_tag_to_rpm_version(include_dash=True) }}"
+      name: "{{ openshift.common.service_type }}-sdn-ovs{{ (openshift_pkg_version | default('')) | oo_image_tag_to_rpm_version(include_dash=True) }}"
       state: present
     when:
     - openshift_node_use_openshift_sdn | bool
diff --git a/roles/openshift_node/tasks/main.yml b/roles/openshift_node/tasks/main.yml
index 59b8bb76e..eae9ca7bc 100644
--- a/roles/openshift_node/tasks/main.yml
+++ b/roles/openshift_node/tasks/main.yml
@@ -66,15 +66,10 @@
     sysctl_file: "/etc/sysctl.d/99-openshift.conf"
     reload: yes
 
-- name: include bootstrap node config
-  include: bootstrap.yml
-  when: openshift_node_bootstrap
-
 - include: registry_auth.yml
 
 - name: include standard node config
   include: config.yml
-  when: not openshift_node_bootstrap
 
 #### Storage class plugins here ####
 - name: NFS storage plugin configuration
@@ -98,3 +93,7 @@
 
 - include: config/workaround-bz1331590-ovs-oom-fix.yml
   when: openshift_node_use_openshift_sdn | default(true) | bool
+
+- name: include bootstrap node config
+  include: bootstrap.yml
+  when: openshift_node_bootstrap
diff --git a/roles/openshift_node/templates/node.service.j2 b/roles/openshift_node/templates/node.service.j2
index 0856737f6..7602d8ee6 100644
--- a/roles/openshift_node/templates/node.service.j2
+++ b/roles/openshift_node/templates/node.service.j2
@@ -12,17 +12,17 @@ After=dnsmasq.service
 
 [Service]
 Type=notify
-EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node
+EnvironmentFile=/etc/sysconfig/{{ openshift_service_type }}-node
 Environment=GOTRACEBACK=crash
 ExecStartPre=/usr/bin/cp /etc/origin/node/node-dnsmasq.conf /etc/dnsmasq.d/
 ExecStartPre=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:/in-addr.arpa/127.0.0.1,/{{ openshift.common.dns_domain }}/127.0.0.1
 ExecStopPost=/usr/bin/rm /etc/dnsmasq.d/node-dnsmasq.conf
 ExecStopPost=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:
-ExecStart=/usr/bin/openshift start node --config=${CONFIG_FILE} $OPTIONS
+ExecStart=/usr/bin/openshift start node {% if openshift_node_bootstrap %} --kubeconfig=${KUBECONFIG} --bootstrap-config-name=${BOOTSTRAP_CONFIG_NAME}{% endif %} --config=${CONFIG_FILE} $OPTIONS
 LimitNOFILE=65536
 LimitCORE=infinity
 WorkingDirectory=/var/lib/origin/
-SyslogIdentifier={{ openshift.common.service_type }}-node
+SyslogIdentifier={{ openshift_service_type }}-node
 Restart=always
 RestartSec=5s
 TimeoutStartSec=300