Merge pull request #97 from jwhonce/wip/cluster

Use ansible playbook to initialize openshift cluster

16 files changed, 261 insertions, 81 deletions

diff --git a/playbooks/aws/openshift-master/config.yml b/playbooks/aws/openshift-master/config.yml
index b3227afa9..bbf1f654a 100644
--- a/playbooks/aws/openshift-master/config.yml
+++ b/playbooks/aws/openshift-master/config.yml
@@ -1,10 +1,10 @@
 ---
-- name: "populate oo_hosts_to_config host group if needed"
+- name: "populate oo_masters_to_config host group if needed"
   hosts: localhost
   gather_facts: no
   tasks:
   - name: "Evaluate oo_host_group_exp if it's set"
-    add_host: "name={{ item }} groups=oo_hosts_to_config"
+    add_host: "name={{ item }} groups=oo_masters_to_config"
     with_items: "{{ oo_host_group_exp | default('') }}"
     when: oo_host_group_exp is defined
 
@@ -25,7 +25,7 @@
       when: groups['tag_env-host-type_' + oo_env + '-openshift-node'] is defined
 
 - name: "Configure instances"
-  hosts: oo_hosts_to_config
+  hosts: oo_masters_to_config
   connection: ssh
   user: root
   vars_files:
diff --git a/playbooks/aws/openshift-master/launch.yml b/playbooks/aws/openshift-master/launch.yml
index a889b93be..3d5a7f579 100644
--- a/playbooks/aws/openshift-master/launch.yml
+++ b/playbooks/aws/openshift-master/launch.yml
@@ -45,8 +45,8 @@
       args:
         tags: "{{ oo_new_inst_tags }}"
 
-    - name: Add new instances public IPs to oo_hosts_to_config
-      add_host: "hostname={{ item.0 }} ansible_ssh_host={{ item.1.dns_name }} groupname=oo_hosts_to_config"
+    - name: Add new instances public IPs to oo_masters_to_config
+      add_host: "hostname={{ item.0 }} ansible_ssh_host={{ item.1.dns_name }} groupname=oo_masters_to_config"
       with_together:
         - oo_new_inst_names
         - ec2.instances
diff --git a/playbooks/aws/openshift-node/config.yml b/playbooks/aws/openshift-node/config.yml
index 21807b1cf..3cf2c58b2 100644
--- a/playbooks/aws/openshift-node/config.yml
+++ b/playbooks/aws/openshift-node/config.yml
@@ -1,10 +1,10 @@
 ---
-- name: "populate oo_hosts_to_config host group if needed"
+- name: "populate oo_nodes_to_config host group if needed"
   hosts: localhost
   gather_facts: no
   tasks:
   - name: Evaluate oo_host_group_exp
-    add_host: "name={{ item }} groups=oo_hosts_to_config"
+    add_host: "name={{ item }} groups=oo_nodes_to_config"
     with_items: "{{ oo_host_group_exp | default('') }}"
     when: oo_host_group_exp is defined
 
@@ -31,7 +31,7 @@
       when: groups['tag_env-host-type-' + oo_env + '-openshift-master'] is defined
 
 - name: "Configure instances"
-  hosts: oo_hosts_to_config
+  hosts: oo_nodes_to_config
   connection: ssh
   user: root
   vars_files:
@@ -44,5 +44,5 @@
         openshift_env: "{{ oo_env }}",
         openshift_public_ip: "{{ ec2_ip_address }}"
       }
-    - docker
     - os_env_extras
+    - os_env_extras_node
diff --git a/playbooks/aws/openshift-node/launch.yml b/playbooks/aws/openshift-node/launch.yml
index a889b93be..4745fc658 100644
--- a/playbooks/aws/openshift-node/launch.yml
+++ b/playbooks/aws/openshift-node/launch.yml
@@ -45,8 +45,8 @@
       args:
         tags: "{{ oo_new_inst_tags }}"
 
-    - name: Add new instances public IPs to oo_hosts_to_config
-      add_host: "hostname={{ item.0 }} ansible_ssh_host={{ item.1.dns_name }} groupname=oo_hosts_to_config"
+    - name: Add new instances public IPs to oo_nodes_to_config
+      add_host: "hostname={{ item.0 }} ansible_ssh_host={{ item.1.dns_name }} groupname=oo_nodes_to_config"
       with_together:
         - oo_new_inst_names
         - ec2.instances
diff --git a/playbooks/gce/openshift-cluster/filter_plugins b/playbooks/gce/openshift-cluster/filter_plugins
new file mode 120000
index 000000000..99a95e4ca
--- /dev/null
+++ b/playbooks/gce/openshift-cluster/filter_plugins
@@ -0,0 +1 @@
+../../../filter_plugins
\ No newline at end of file
diff --git a/playbooks/gce/openshift-cluster/launch.yml b/playbooks/gce/openshift-cluster/launch.yml
new file mode 100644
index 000000000..889d92d40
--- /dev/null
+++ b/playbooks/gce/openshift-cluster/launch.yml
@@ -0,0 +1,63 @@
+---
+- name: Launch instance(s)
+  hosts: localhost
+  connection: local
+  gather_facts: no
+  vars_files:
+      - vars.yml
+  tasks:
+    - set_fact: k8s_type="master"
+
+    - name: Generate master instance names(s)
+      set_fact: scratch={{ cluster_id }}-{{ k8s_type }}-{{ '%05x' |format( 1048576 |random) }}
+      register: master_names_output
+      with_sequence: start=1 end={{ masters }}
+
+    # These set_fact's cannot be combined
+    - set_fact:
+        master_names_string: "{% for item in master_names_output.results %}{{ item.ansible_facts.scratch }} {% endfor %}"
+
+    - set_fact:
+        master_names: "{{ master_names_string.strip().split(' ') }}"
+
+    - include: launch_instances.yml
+      vars:
+        instances: "{{ master_names }}"
+        cluster: "{{ cluster_id }}"
+        type: "{{ k8s_type }}"
+        group_name: "tag_env-host-type-{{ cluster_id }}-openshift-master"
+
+    - set_fact: k8s_type="node"
+
+    - name: Generate node instance names(s)
+      set_fact: scratch={{ cluster_id }}-{{ k8s_type }}-{{ '%05x' |format( 1048576 |random) }}
+      register: node_names_output
+      with_sequence: start=1 end={{ nodes }}
+
+    # These set_fact's cannot be combined
+    - set_fact:
+        node_names_string: "{% for item in node_names_output.results %}{{ item.ansible_facts.scratch }} {% endfor %}"
+
+    - set_fact:
+        node_names: "{{ node_names_string.strip().split(' ') }}"
+
+    - include: launch_instances.yml
+      vars:
+        instances: "{{ node_names }}"
+        cluster: "{{ cluster_id }}"
+        type: "{{ k8s_type }}"
+
+- hosts: "tag_env-{{ cluster_id }}"
+  roles:
+  - openshift_repos
+  - os_update_latest
+
+- include: ../openshift-master/config.yml
+  vars:
+    oo_host_group_exp: "groups[\"tag_env-host-type-{{ cluster_id }}-openshift-master\"]"
+    oo_env: "{{ cluster_id }}"
+
+- include: ../openshift-node/config.yml
+  vars:
+    oo_host_group_exp: "groups[\"tag_env-host-type-{{ cluster_id }}-openshift-node\"]"
+    oo_env: "{{ cluster_id }}"
diff --git a/playbooks/gce/openshift-cluster/launch_instances.yml b/playbooks/gce/openshift-cluster/launch_instances.yml
new file mode 100644
index 000000000..20e31d990
--- /dev/null
+++ b/playbooks/gce/openshift-cluster/launch_instances.yml
@@ -0,0 +1,39 @@
+
+- set_fact:
+    machine_type: "{{ lookup('env', 'gce_machine_type') |default('n1-standard-1', true) }}"
+    machine_image: "{{ lookup('env', 'gce_machine_image') |default('libra-rhel7', true) }}"
+
+- name: Launch instance(s)
+  gce:
+    instance_names: "{{ instances }}"
+    machine_type: "{{ machine_type }}"
+    image: "{{ machine_image }}"
+    service_account_email: "{{ lookup('env', 'gce_service_account_email_address') }}"
+    pem_file: "{{ lookup('env', 'gce_service_account_pem_file_path') }}"
+    project_id: "{{ lookup('env', 'gce_project_id') }}"
+    tags:
+      - "created-by-{{ lookup('env', 'LOGNAME') |default(cluster, true) }}"
+      - "env-{{ cluster }}"
+      - "host-type-{{ type }}"
+      - "env-host-type-{{ cluster }}-openshift-{{ type }}"
+  register: gce
+
+- name: Add new instances public IPs
+  add_host:
+    hostname: "{{ item.name }}"
+    ansible_ssh_host: "{{ item.public_ip }}"
+    groups: "{{ item.tags | oo_prepend_strings_in_list('tag_') | join(',') }}"
+    gce_public_ip: "{{ item.public_ip }}"
+  with_items: gce.instance_data
+
+- name: Wait for ssh
+  wait_for: "port=22 host={{ item.public_ip }}"
+  with_items: gce.instance_data
+
+- name: Wait for root user setup
+  command: "ssh -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null root@{{ item.public_ip }} echo root user is setup"
+  register: result
+  until: result.rc == 0
+  retries: 20
+  delay: 10
+  with_items: gce.instance_data
diff --git a/playbooks/gce/openshift-cluster/roles b/playbooks/gce/openshift-cluster/roles
new file mode 120000
index 000000000..20c4c58cf
--- /dev/null
+++ b/playbooks/gce/openshift-cluster/roles
@@ -0,0 +1 @@
+../../../roles
\ No newline at end of file
diff --git a/playbooks/gce/openshift-cluster/terminate.yml b/playbooks/gce/openshift-cluster/terminate.yml
new file mode 100644
index 000000000..0281ae953
--- /dev/null
+++ b/playbooks/gce/openshift-cluster/terminate.yml
@@ -0,0 +1,20 @@
+---
+- name: Terminate instance(s)
+  hosts: localhost
+
+  vars_files:
+    - vars.yml
+
+- include: ../openshift-node/terminate.yml
+  vars:
+    oo_host_group_exp: 'groups["tag_env-host-type-{{ cluster_id }}-openshift-node"]'
+    gce_service_account_email: "{{ lookup('env', 'gce_service_account_email_address') }}"
+    gce_pem_file: "{{ lookup('env', 'gce_service_account_pem_file_path') }}"
+    gce_project_id: "{{ lookup('env', 'gce_project_id') }}"
+
+- include: ../openshift-master/terminate.yml
+  vars:
+    oo_host_group_exp: 'groups["tag_env-host-type-{{ cluster_id }}-openshift-master"]'
+    gce_service_account_email: "{{ lookup('env', 'gce_service_account_email_address') }}"
+    gce_pem_file: "{{ lookup('env', 'gce_service_account_pem_file_path') }}"
+    gce_project_id: "{{ lookup('env', 'gce_project_id') }}"
diff --git a/playbooks/gce/openshift-cluster/vars.yml b/playbooks/gce/openshift-cluster/vars.yml
new file mode 100644
index 000000000..ed97d539c
--- /dev/null
+++ b/playbooks/gce/openshift-cluster/vars.yml
@@ -0,0 +1 @@
+---
diff --git a/playbooks/gce/openshift-master/config.yml b/playbooks/gce/openshift-master/config.yml
index a74250d13..e405e2fb4 100644
--- a/playbooks/gce/openshift-master/config.yml
+++ b/playbooks/gce/openshift-master/config.yml
@@ -1,41 +1,20 @@
----
-- name: "populate oo_hosts_to_config host group if needed"
+- name: master/config.yml, populate oo_masters_to_config host group if needed
   hosts: localhost
   gather_facts: no
   tasks:
   - name: "Evaluate oo_host_group_exp if it's set"
-    add_host: "name={{ item }} groups=oo_hosts_to_config"
+    add_host: "name={{ item }} groups=oo_masters_to_config"
     with_items: "{{ oo_host_group_exp | default('') }}"
     when: oo_host_group_exp is defined
 
-- name: "Gather facts for nodes in {{ oo_env }}"
+- name: Gather facts for nodes in {{ oo_env }}
   hosts: "tag_env-host-type-{{ oo_env }}-openshift-node"
-  connection: ssh
-  user: root
-
-- name: "Set Origin specific facts on localhost (for later use)"
-  hosts: localhost
-  gather_facts: no
-  tasks:
-    - name: Setting openshift_node_ips fact on localhost
-      set_fact:
-        openshift_node_ips: "{{ hostvars
-            | oo_select_keys(groups['tag_env-host-type-' + oo_env + '-openshift-node'])
-            | oo_collect(attribute='ansible_default_ipv4.address') }}"
-      when: groups['tag_env-host-type-' + oo_env + '-openshift-node'] is defined
 
 - name: "Configure instances"
-  hosts: oo_hosts_to_config
-  connection: ssh
-  user: root
+  hosts: oo_masters_to_config
   vars_files:
-    - vars.yml
+  - vars.yml
   roles:
-    - {
-        role: openshift_master,
-        openshift_node_ips: "{{ hostvars['localhost'].openshift_node_ips | default(['']) }}",
-        openshift_public_ip: "{{ gce_public_ip }}",
-        openshift_env: "{{ oo_env }}",
-      }
+    - openshift_master
     - pods
     - os_env_extras
diff --git a/playbooks/gce/openshift-master/launch.yml b/playbooks/gce/openshift-master/launch.yml
index f2800b061..3512274cc 100644
--- a/playbooks/gce/openshift-master/launch.yml
+++ b/playbooks/gce/openshift-master/launch.yml
@@ -24,8 +24,8 @@
         tags: "{{ oo_new_inst_tags }}"
       register: gce
 
-    - name: Add new instances public IPs to oo_hosts_to_config
-      add_host: "hostname={{ item.name }} ansible_ssh_host={{ item.public_ip }} groupname=oo_hosts_to_config"
+    - name: Add new instances public IPs to oo_masters_to_config
+      add_host: "hostname={{ item.name }} ansible_ssh_host={{ item.public_ip }} groupname=oo_masters_to_config"
       with_items: gce.instance_data
 
     - name: Wait for ssh
diff --git a/playbooks/gce/openshift-master/terminate.yml b/playbooks/gce/openshift-master/terminate.yml
index 76e1404b5..9e027cf41 100644
--- a/playbooks/gce/openshift-master/terminate.yml
+++ b/playbooks/gce/openshift-master/terminate.yml
@@ -12,9 +12,10 @@
     - debug: msg="{{ groups['oo_hosts_to_terminate'] }}"
 
 
-- name: Terminate instances
+- name: Terminate master instances
   hosts: localhost
   connection: local
+  gather_facts: no
   tasks:
     - name: Terminate master instances
       gce:
diff --git a/playbooks/gce/openshift-node/config.yml b/playbooks/gce/openshift-node/config.yml
index 78047cf40..e0d074572 100644
--- a/playbooks/gce/openshift-node/config.yml
+++ b/playbooks/gce/openshift-node/config.yml
@@ -1,48 +1,122 @@
----
-- name: "populate oo_hosts_to_config host group if needed"
+- name: node/config.yml, populate oo_nodes_to_config host group if needed
   hosts: localhost
   gather_facts: no
   tasks:
   - name: Evaluate oo_host_group_exp
-    add_host: "name={{ item }} groups=oo_hosts_to_config"
+    add_host: "name={{ item }} groups=oo_nodes_to_config"
     with_items: "{{ oo_host_group_exp | default('') }}"
     when: oo_host_group_exp is defined
+  - name: Find masters for env
+    add_host: "name={{ item }} groups=oo_masters_for_node_config"
+    with_items: groups['tag_env-host-type-' + oo_env + '-openshift-master']
 
-- name: "Gather facts for masters in {{ oo_env }}"
-  hosts: "tag_env-host-type-{{ oo_env }}-openshift-master"
-  connection: ssh
-  user: root
+- name: Gather facts for masters in {{ oo_env }}
+  hosts: tag_env-host-type-{{ oo_env }}-openshift-master
+  tasks:
+  - set_fact:
+      openshift_master_ip: "{{ openshift_ip }}"
+      openshift_master_api_url: "{{ openshift_api_url }}"
+      openshift_master_webui_url: "{{ openshift_webui_url }}"
+      openshift_master_hostname: "{{ openshift_hostname }}"
+      openshift_master_public_ip: "{{ openshift_public_ip }}"
+      openshift_master_api_public_url: "{{ openshift_api_public_url }}"
+      openshift_master_webui_public_url: "{{ openshift_webui_public_url }}"
+      openshift_master_public_hostnames: "{{ openshift_public_hostname }}"
 
-- name: "Set OO sepcific facts on localhost (for later use)"
-  hosts: localhost
-  gather_facts: no
+- name: Gather facts for hosts to configure
+  hosts: tag_env-host-type-{{ oo_env }}-openshift-node
+  tasks:
+  - set_fact:
+      openshift_node_hostname: "{{ openshift_hostname }}"
+      openshift_node_name: "{{ openshift_hostname }}"
+      openshift_node_cpu: "{{ openshift_node_cpu if openshift_node_cpu else ansible_processor_cores }}"
+      openshift_node_memory: "{{ openshift_node_memory if openshift_node_memory else (ansible_memtotal_mb|int * 1024 * 1024 * 0.75)|int }}"
+      openshift_node_pod_cidr: "{{ openshift_node_pod_cidr if openshift_node_pod_cidr else None }}"
+      openshift_node_host_ip: "{{ openshift_ip }}"
+      openshift_node_labels: "{{ openshift_node_labels if openshift_node_labels else {} }}"
+      openshift_node_annotations: "{{ openshift_node_annotations if openshift_node_annotations else {} }}"
+
+- name: Register nodes
+  hosts: tag_env-host-type-{{ oo_env }}-openshift-master[0]
+  vars:
+    openshift_node_group: tag_env-host-type-{{ oo_env }}-openshift-node
+    openshift_nodes: "{{ hostvars
+          | oo_select_keys(groups[openshift_node_group]) }}"
+    openshift_master_group: tag_env-host-type-{{ oo_env }}-openshift-master
+    openshift_master_urls: "{{ hostvars
+          | oo_select_keys(groups[openshift_master_group])
+          | oo_collect(attribute='openshift_master_api_url') }}"
+    openshift_master_public_urls: "{{ hostvars
+          | oo_select_keys(groups[openshift_master_group])
+          | oo_collect(attribute='openshift_master_api_public_url') }}"
+  pre_tasks:
+  roles:
+  - openshift_register_nodes
   tasks:
-    - name: Setting openshift_master_ips fact on localhost
-      set_fact:
-        openshift_master_ips: "{{ hostvars
-            | oo_select_keys(groups['tag_env-host-type-' + oo_env + '-openshift-master'])
-            | oo_collect(attribute='ansible_default_ipv4.address') }}"
-      when: groups['tag_env-host-type-' + oo_env + '-openshift-master'] is defined
-    - name: Setting openshift_master_public_ips fact on localhost
-      set_fact:
-        openshift_master_public_ips: "{{ hostvars
-            | oo_select_keys(groups['tag_env-host-type-' + oo_env + '-openshift-master'])
-            | oo_collect(attribute='gce_public_ip') }}"
-      when: groups['tag_env-host-type-' + oo_env + '-openshift-master'] is defined
+  - name: Create local temp directory for syncing certs
+    local_action: command /usr/bin/mktemp -d /tmp/openshift-ansible-XXXXXXX
+    register: mktemp
 
-- name: "Configure instances"
-  hosts: oo_hosts_to_config
-  connection: ssh
-  user: root
+  - name: Sync master certs to localhost
+    synchronize:
+      mode: pull
+      checksum: yes
+      src: /var/lib/openshift/openshift.local.certificates
+      dest: "{{ mktemp.stdout }}"
+
+# TODO: sync generated certs between masters
+#
+- name: Configure instances
+  hosts: oo_nodes_to_config
   vars_files:
-    - vars.yml
+  - vars.yml
+  vars:
+    openshift_master_group: tag_env-host-type-{{ oo_env }}-openshift-master
+    openshift_master_ips: "{{ hostvars
+          | oo_select_keys(groups[openshift_master_group])
+          | oo_collect(attribute='openshift_master_ip') }}"
+    openshift_master_hostnames: "{{ hostvars
+          | oo_select_keys(groups[openshift_master_group])
+          | oo_collect(attribute='openshift_master_hostname') }}"
+    openshift_master_public_ips: "{{ hostvars
+          | oo_select_keys(groups[openshift_master_group])
+          | oo_collect(attribute='openshift_master_public_ip') }}"
+    openshift_master_public_hostnames: "{{ hostvars
+          | oo_select_keys(groups[openshift_master_group])
+          | oo_collect(attribute='openshift_master_public_hostname') }}"
+    cert_parent_rel_path: openshift.local.certificates
+    cert_rel_path: "{{ cert_parent_rel_path }}/node-{{ openshift_node_name }}"
+    cert_base_path: /var/lib/openshift
+    cert_parent_path: "{{ cert_base_path }}/{{ cert_parent_rel_path }}"
+    cert_path: "{{ cert_base_path }}/{{ cert_rel_path }}"
+  pre_tasks:
+  - name: Ensure certificate directories exists
+    file:
+      path: "{{ item }}"
+      state: directory
+    with_items:
+    - "{{ cert_path }}"
+    - "{{ cert_parent_path }}/ca"
+
+  # TODO: only sync to a node if it's certs have been updated
+  # TODO: notify restart openshift-node and/or restart openshift-sdn-node,
+  # possibly test service started time against certificate/config file
+  # timestamps in openshift-node or openshift-sdn-node to trigger notify
+  # TODO: also copy ca cert: /var/lib/openshift/openshift.local.certificates/ca/cert.crt
+  - name: Sync certs to nodes
+    synchronize:
+      checksum: yes
+      src: "{{ item.src }}"
+      dest: "{{ item.dest }}"
+      owner: no
+      group: no
+    with_items:
+    - src: "{{ hostvars[groups[openshift_master_group][0]].mktemp.stdout }}/{{ cert_rel_path }}"
+      dest: "{{ cert_parent_path }}"
+    - src: "{{ hostvars[groups[openshift_master_group][0]].mktemp.stdout }}/{{ cert_parent_rel_path }}/ca/cert.crt"
+      dest: "{{ cert_parent_path }}/ca/cert.crt"
+  - local_action: file name={{ hostvars[groups[openshift_master_group][0]].mktemp.stdout }} state=absent
+    run_once: true
   roles:
-    - {
-        role: openshift_node,
-        openshift_master_ips: "{{ hostvars['localhost'].openshift_master_ips | default(['']) }}",
-        openshift_master_public_ips: "{{ hostvars['localhost'].openshift_master_public_ips | default(['']) }}",
-        openshift_public_ip: "{{ gce_public_ip }}",
-        openshift_env: "{{ oo_env }}",
-      }
-    - docker
+    - openshift_node
     - os_env_extras
diff --git a/playbooks/gce/openshift-node/launch.yml b/playbooks/gce/openshift-node/launch.yml
index 935599efd..ca2914d8a 100644
--- a/playbooks/gce/openshift-node/launch.yml
+++ b/playbooks/gce/openshift-node/launch.yml
@@ -24,8 +24,8 @@
         tags: "{{ oo_new_inst_tags }}"
       register: gce
 
-    - name: Add new instances public IPs to oo_hosts_to_config
-      add_host: "hostname={{ item.name }} ansible_ssh_host={{ item.public_ip }} groupname=oo_hosts_to_config"
+    - name: Add new instances public IPs to oo_nodes_to_config
+      add_host: "hostname={{ item.name }} ansible_ssh_host={{ item.public_ip }} groupname=oo_nodes_to_config"
       with_items: gce.instance_data
 
     - name: Wait for ssh
@@ -48,10 +48,10 @@
 
 # Always bounce service to pick up new credentials
 #- name: "Restart instances"
-#  hosts: oo_hosts_to_config
+#  hosts: oo_nodes_to_config
 #  connection: ssh
 #  user: root
 #  tasks:
-#    - debug: var=groups.oo_hosts_to_config
+#    - debug: var=groups.oo_nodes_to_config
 #    - name: Restart OpenShift
 #      service: name=openshift-node enabled=yes state=restarted
diff --git a/playbooks/gce/openshift-node/terminate.yml b/playbooks/gce/openshift-node/terminate.yml
index 8d60f27b3..9aa8a48c1 100644
--- a/playbooks/gce/openshift-node/terminate.yml
+++ b/playbooks/gce/openshift-node/terminate.yml
@@ -12,9 +12,10 @@
     - debug: msg="{{ groups['oo_hosts_to_terminate'] }}"
 
 
-- name: Terminate instances
+- name: Terminate node instances
   hosts: localhost
   connection: local
+  gather_facts: no
   tasks:
     - name: Terminate node instances
       gce: