diff options
Diffstat (limited to 'playbooks/common')
84 files changed, 2862 insertions, 427 deletions
diff --git a/playbooks/common/openshift-cluster/additional_config.yml b/playbooks/common/openshift-cluster/additional_config.yml new file mode 100644 index 000000000..825f46415 --- /dev/null +++ b/playbooks/common/openshift-cluster/additional_config.yml @@ -0,0 +1,22 @@ +- name: Additional master configuration + hosts: oo_first_master + vars: + cockpit_plugins: "{{ osm_cockpit_plugins | default(['cockpit-kubernetes']) }}" + etcd_urls: "{{ openshift.master.etcd_urls }}" + openshift_master_ha: "{{ groups.oo_masters | length > 1 }}" + omc_cluster_hosts: "{{ groups.oo_masters | join(' ')}}" + roles: + - role: openshift_master_cluster + when: openshift_master_ha | bool and openshift.master.cluster_method == "pacemaker" + - role: openshift_examples + registry_url: "{{ openshift.master.registry_url }}" + when: openshift.common.install_examples | bool + - role: openshift_hosted_templates + registry_url: "{{ openshift.master.registry_url }}" + - role: openshift_manageiq + when: openshift.common.use_manageiq | bool + - role: cockpit + when: not openshift.common.is_atomic and ( deployment_type in ['atomic-enterprise','openshift-enterprise'] ) and + (osm_use_cockpit | bool or osm_use_cockpit is undefined ) and ( openshift.common.deployment_subtype != 'registry' ) + - role: flannel_register + when: openshift.common.use_flannel | bool diff --git a/playbooks/common/openshift-cluster/config.yml b/playbooks/common/openshift-cluster/config.yml index 0779cfe47..801c8065d 100644 --- a/playbooks/common/openshift-cluster/config.yml +++ b/playbooks/common/openshift-cluster/config.yml @@ -1,65 +1,66 @@ --- -- name: Populate config host groups - hosts: localhost - gather_facts: no - tasks: - - fail: - msg: This playbook rquires g_etcd_group to be set - when: g_etcd_group is not defined +- include: evaluate_groups.yml + tags: + - always - - fail: - msg: This playbook rquires g_masters_group to be set - when: g_masters_group is not defined +- include: initialize_facts.yml + tags: + - always - - fail: - msg: This playbook rquires g_nodes_group to be set - when: g_nodes_group is not defined +- include: validate_hostnames.yml + tags: + - node - - name: Evaluate oo_etcd_to_config - add_host: - name: "{{ item }}" - groups: oo_etcd_to_config - ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" - ansible_sudo: "{{ g_sudo | default(omit) }}" - with_items: groups[g_etcd_group] | default([]) +- include: initialize_openshift_version.yml - - name: Evaluate oo_masters_to_config - add_host: - name: "{{ item }}" - groups: oo_masters_to_config - ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" - ansible_sudo: "{{ g_sudo | default(omit) }}" - with_items: groups[g_masters_group] | default([]) +- name: Set oo_option facts + hosts: oo_all_hosts + tags: + - always + tasks: + - set_fact: + openshift_docker_additional_registries: "{{ lookup('oo_option', 'docker_additional_registries') }}" + when: openshift_docker_additional_registries is not defined + - set_fact: + openshift_docker_insecure_registries: "{{ lookup('oo_option', 'docker_insecure_registries') }}" + when: openshift_docker_insecure_registries is not defined + - set_fact: + openshift_docker_blocked_registries: "{{ lookup('oo_option', 'docker_blocked_registries') }}" + when: openshift_docker_blocked_registries is not defined + - set_fact: + openshift_docker_options: "{{ lookup('oo_option', 'docker_options') }}" + when: openshift_docker_options is not defined + - set_fact: + openshift_docker_log_driver: "{{ lookup('oo_option', 'docker_log_driver') }}" + when: openshift_docker_log_driver is not defined + - set_fact: + openshift_docker_log_options: "{{ lookup('oo_option', 'docker_log_options') }}" + when: openshift_docker_log_options is not defined - - name: Evaluate oo_nodes_to_config - add_host: - name: "{{ item }}" - groups: oo_nodes_to_config - ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" - ansible_sudo: "{{ g_sudo | default(omit) }}" - with_items: groups[g_nodes_group] | default([]) +- include: ../openshift-etcd/config.yml + tags: + - etcd - - name: Evaluate oo_first_etcd - add_host: - name: "{{ groups[g_etcd_group][0] }}" - groups: oo_first_etcd - ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" - ansible_sudo: "{{ g_sudo | default(omit) }}" - when: g_etcd_group in groups and (groups[g_etcd_group] | length) > 0 +- include: ../openshift-nfs/config.yml + tags: + - nfs - - name: Evaluate oo_first_master - add_host: - name: "{{ groups[g_masters_group][0] }}" - groups: oo_first_master - ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" - ansible_sudo: "{{ g_sudo | default(omit) }}" - when: g_masters_group in groups and (groups[g_masters_group] | length) > 0 - -- include: ../openshift-etcd/config.yml +- include: ../openshift-loadbalancer/config.yml + tags: + - loadbalancer - include: ../openshift-master/config.yml + tags: + - master + +- include: additional_config.yml + tags: + - master - include: ../openshift-node/config.yml - vars: - osn_cluster_dns_domain: "{{ hostvars[groups.oo_first_master.0].openshift.dns.domain }}" - osn_cluster_dns_ip: "{{ hostvars[groups.oo_first_master.0].openshift.dns.ip }}" + tags: + - node + +- include: openshift_hosted.yml + tags: + - hosted diff --git a/playbooks/common/openshift-cluster/create_services.yml b/playbooks/common/openshift-cluster/create_services.yml deleted file mode 100644 index e70709d19..000000000 --- a/playbooks/common/openshift-cluster/create_services.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -- name: Deploy OpenShift Services - hosts: "{{ g_svc_master }}" - connection: ssh - gather_facts: yes - roles: - - openshift_registry - - openshift_router diff --git a/playbooks/common/openshift-cluster/enable_dnsmasq.yml b/playbooks/common/openshift-cluster/enable_dnsmasq.yml new file mode 100644 index 000000000..4cfe8617e --- /dev/null +++ b/playbooks/common/openshift-cluster/enable_dnsmasq.yml @@ -0,0 +1,68 @@ +--- +- include: evaluate_groups.yml + +- name: Load openshift_facts + hosts: oo_masters_to_config:oo_nodes_to_config + roles: + - openshift_facts + post_tasks: + - fail: msg="This playbook requires a master version of at least Origin 1.1 or OSE 3.1" + when: not openshift.common.version_gte_3_1_1_or_1_1_1 | bool + +- name: Reconfigure masters to listen on our new dns_port + hosts: oo_masters_to_config + handlers: + - include: ../../../roles/openshift_master/handlers/main.yml + static: yes + vars: + os_firewall_allow: + - service: skydns tcp + port: "{{ openshift.master.dns_port }}/tcp" + - service: skydns udp + port: "{{ openshift.master.dns_port }}/udp" + roles: + - os_firewall + tasks: + - openshift_facts: + role: "{{ item.role }}" + local_facts: "{{ item.local_facts }}" + with_items: + - role: common + local_facts: + use_dnsmasq: True + - role: master + local_facts: + dns_port: '8053' + - modify_yaml: + dest: "{{ openshift.common.config_base }}/master/master-config.yaml" + yaml_key: dnsConfig.bindAddress + yaml_value: "{{ openshift.master.bind_addr }}:{{ openshift.master.dns_port }}" + notify: restart master + - meta: flush_handlers + +- name: Configure nodes for dnsmasq + hosts: oo_nodes_to_config + handlers: + - include: ../../../roles/openshift_node/handlers/main.yml + static: yes + pre_tasks: + - openshift_facts: + role: "{{ item.role }}" + local_facts: "{{ item.local_facts }}" + with_items: + - role: common + local_facts: + use_dnsmasq: True + - role: node + local_facts: + dns_ip: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}" + vars: + openshift_deployment_type: "{{ deployment_type }}" + roles: + - openshift_node_dnsmasq + post_tasks: + - modify_yaml: + dest: "{{ openshift.common.config_base }}/node/node-config.yaml" + yaml_key: dnsIP + yaml_value: "{{ openshift.node.dns_ip }}" + notify: restart node diff --git a/playbooks/common/openshift-cluster/evaluate_groups.yml b/playbooks/common/openshift-cluster/evaluate_groups.yml new file mode 100644 index 000000000..b3e02fb97 --- /dev/null +++ b/playbooks/common/openshift-cluster/evaluate_groups.yml @@ -0,0 +1,111 @@ +--- +- name: Populate config host groups + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - fail: + msg: This playbook requires g_etcd_hosts to be set + when: g_etcd_hosts is not defined + + - fail: + msg: This playbook requires g_master_hosts or g_new_master_hosts to be set + when: g_master_hosts is not defined and g_new_master_hosts is not defined + + - fail: + msg: This playbook requires g_node_hosts or g_new_node_hosts to be set + when: g_node_hosts is not defined and g_new_node_hosts is not defined + + - fail: + msg: This playbook requires g_lb_hosts to be set + when: g_lb_hosts is not defined + + - fail: + msg: This playbook requires g_nfs_hosts to be set + when: g_nfs_hosts is not defined + + - fail: + msg: The nfs group must be limited to one host + when: (groups[g_nfs_hosts] | default([])) | length > 1 + + - name: Evaluate oo_all_hosts + add_host: + name: "{{ item }}" + groups: oo_all_hosts + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_become: "{{ g_sudo | default(omit) }}" + with_items: "{{ g_all_hosts | default([]) }}" + + - name: Evaluate oo_masters + add_host: + name: "{{ item }}" + groups: oo_masters + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_become: "{{ g_sudo | default(omit) }}" + with_items: "{{ g_master_hosts | union(g_new_master_hosts) | default([]) }}" + + - name: Evaluate oo_etcd_to_config + add_host: + name: "{{ item }}" + groups: oo_etcd_to_config + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_become: "{{ g_sudo | default(omit) }}" + with_items: "{{ g_etcd_hosts | default([]) }}" + + - name: Evaluate oo_masters_to_config + add_host: + name: "{{ item }}" + groups: oo_masters_to_config + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_become: "{{ g_sudo | default(omit) }}" + with_items: "{{ g_new_master_hosts | default(g_master_hosts | default([], true), true) }}" + + - name: Evaluate oo_nodes_to_config + add_host: + name: "{{ item }}" + groups: oo_nodes_to_config + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_become: "{{ g_sudo | default(omit) }}" + with_items: "{{ g_new_node_hosts | default(g_node_hosts | default([], true), true) }}" + + # Skip adding the master to oo_nodes_to_config when g_new_node_hosts is + - name: Evaluate oo_nodes_to_config + add_host: + name: "{{ item }}" + groups: oo_nodes_to_config + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_become: "{{ g_sudo | default(omit) }}" + with_items: "{{ g_master_hosts | default([]) }}" + when: g_nodeonmaster | default(false) | bool and not g_new_node_hosts | default(false) | bool + + - name: Evaluate oo_first_etcd + add_host: + name: "{{ g_etcd_hosts[0] }}" + groups: oo_first_etcd + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + when: g_etcd_hosts|length > 0 + + - name: Evaluate oo_first_master + add_host: + name: "{{ g_master_hosts[0] }}" + groups: oo_first_master + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_become: "{{ g_sudo | default(omit) }}" + when: g_master_hosts|length > 0 + + - name: Evaluate oo_lb_to_config + add_host: + name: "{{ item }}" + groups: oo_lb_to_config + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_become: "{{ g_sudo | default(omit) }}" + with_items: "{{ g_lb_hosts | default([]) }}" + + - name: Evaluate oo_nfs_to_config + add_host: + name: "{{ item }}" + groups: oo_nfs_to_config + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_become: "{{ g_sudo | default(omit) }}" + with_items: "{{ g_nfs_hosts | default([]) }}" diff --git a/playbooks/common/openshift-cluster/initialize_facts.yml b/playbooks/common/openshift-cluster/initialize_facts.yml new file mode 100644 index 000000000..6d83d2527 --- /dev/null +++ b/playbooks/common/openshift-cluster/initialize_facts.yml @@ -0,0 +1,15 @@ +--- +- name: Initialize host facts + hosts: oo_all_hosts + any_errors_fatal: true + roles: + - openshift_facts + tasks: + - openshift_facts: + role: common + local_facts: + hostname: "{{ openshift_hostname | default(None) }}" + - set_fact: + openshift_docker_hosted_registry_network: "{{ hostvars[groups.oo_first_master.0].openshift.common.portal_net }}" + - set_fact: + openshift_deployment_type: "{{ deployment_type }}" diff --git a/playbooks/common/openshift-cluster/initialize_openshift_version.yml b/playbooks/common/openshift-cluster/initialize_openshift_version.yml new file mode 100644 index 000000000..2f384ddea --- /dev/null +++ b/playbooks/common/openshift-cluster/initialize_openshift_version.yml @@ -0,0 +1,32 @@ +--- +# NOTE: requires openshift_facts be run +- hosts: l_oo_all_hosts + gather_facts: no + tasks: + # See: + # https://bugzilla.redhat.com/show_bug.cgi?id=1395047 + # https://bugzilla.redhat.com/show_bug.cgi?id=1282961 + # https://github.com/openshift/openshift-ansible/issues/1138 + - name: Check for bad combinations of yum and subscription-manager + command: > + {{ repoquery_cmd }} --installed --qf '%{version}' "yum" + register: yum_ver_test + changed_when: false + - fail: + msg: Incompatible versions of yum and subscription-manager found. You may need to update yum and yum-utils. + when: "'Plugin \"search-disabled-repos\" requires API 2.7. Supported API is 2.6.' in yum_ver_test.stdout" + +- name: Determine openshift_version to configure on first master + hosts: oo_first_master + roles: + - openshift_version + +# NOTE: We set this even on etcd hosts as they may also later run as masters, +# and we don't want to install wrong version of docker and have to downgrade +# later. +- name: Set openshift_version for all hosts + hosts: oo_all_hosts:!oo_first_master + vars: + openshift_version: "{{ hostvars[groups.oo_first_master.0].openshift_version }}" + roles: + - openshift_version diff --git a/playbooks/common/openshift-cluster/library b/playbooks/common/openshift-cluster/library new file mode 120000 index 000000000..d0b7393d3 --- /dev/null +++ b/playbooks/common/openshift-cluster/library @@ -0,0 +1 @@ +../../../library/
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/openshift_hosted.yml b/playbooks/common/openshift-cluster/openshift_hosted.yml new file mode 100644 index 000000000..ccbba54b4 --- /dev/null +++ b/playbooks/common/openshift-cluster/openshift_hosted.yml @@ -0,0 +1,68 @@ +--- +- name: Create persistent volumes + hosts: oo_first_master + tags: + - hosted + vars: + persistent_volumes: "{{ hostvars[groups.oo_first_master.0] | oo_persistent_volumes(groups) }}" + persistent_volume_claims: "{{ hostvars[groups.oo_first_master.0] | oo_persistent_volume_claims }}" + roles: + - role: openshift_persistent_volumes + when: persistent_volumes | length > 0 or persistent_volume_claims | length > 0 + +- name: Create Hosted Resources + hosts: oo_first_master + tags: + - hosted + pre_tasks: + - set_fact: + openshift_hosted_router_registryurl: "{{ hostvars[groups.oo_first_master.0].openshift.master.registry_url }}" + openshift_hosted_registry_registryurl: "{{ hostvars[groups.oo_first_master.0].openshift.master.registry_url }}" + when: "'master' in hostvars[groups.oo_first_master.0].openshift and 'registry_url' in hostvars[groups.oo_first_master.0].openshift.master" + - set_fact: + logging_hostname: "{{ openshift_hosted_logging_hostname | default('kibana.' ~ (openshift.master.default_subdomain | default('router.default.svc.cluster.local', true))) }}" + logging_ops_hostname: "{{ openshift_hosted_logging_ops_hostname | default('kibana-ops.' ~ (openshift.master.default_subdomain | default('router.default.svc.cluster.local', true))) }}" + logging_master_public_url: "{{ openshift_hosted_logging_master_public_url | default(openshift.master.public_api_url) }}" + logging_elasticsearch_cluster_size: "{{ openshift_hosted_logging_elasticsearch_cluster_size | default(1) }}" + logging_elasticsearch_ops_cluster_size: "{{ openshift_hosted_logging_elasticsearch_ops_cluster_size | default(1) }}" + roles: + - role: openshift_cli + - role: openshift_hosted_facts + - role: openshift_projects + # TODO: Move standard project definitions to openshift_hosted/vars/main.yml + # Vars are not accessible in meta/main.yml in ansible-1.9.x + openshift_projects: "{{ openshift_additional_projects | default({}) | oo_merge_dicts({'default':{'default_node_selector':''},'openshift-infra':{'default_node_selector':''},'logging':{'default_node_selector':''}}) }}" + - role: openshift_serviceaccounts + openshift_serviceaccounts_names: + - router + openshift_serviceaccounts_namespace: default + openshift_serviceaccounts_sccs: + - hostnetwork + when: openshift.common.version_gte_3_2_or_1_2 + - role: openshift_serviceaccounts + openshift_serviceaccounts_names: + - router + - registry + openshift_serviceaccounts_namespace: default + openshift_serviceaccounts_sccs: + - privileged + when: not openshift.common.version_gte_3_2_or_1_2 + - role: openshift_hosted + - role: openshift_metrics + when: openshift_hosted_metrics_deploy | default(false) | bool + - role: openshift_hosted_logging + when: openshift_hosted_logging_deploy | default(false) | bool + openshift_hosted_logging_hostname: "{{ logging_hostname }}" + openshift_hosted_logging_ops_hostname: "{{ logging_ops_hostname }}" + openshift_hosted_logging_master_public_url: "{{ logging_master_public_url }}" + openshift_hosted_logging_elasticsearch_cluster_size: "{{ logging_elasticsearch_cluster_size }}" + openshift_hosted_logging_elasticsearch_pvc_dynamic: "{{ 'true' if openshift_hosted_logging_storage_kind | default(none) == 'dynamic' else '' }}" + openshift_hosted_logging_elasticsearch_pvc_size: "{{ openshift.hosted.logging.storage.volume.size if openshift_hosted_logging_storage_kind | default(none) in ['dynamic','nfs'] else '' }}" + openshift_hosted_logging_elasticsearch_pvc_prefix: "{{ 'logging-es' if openshift_hosted_logging_storage_kind | default(none) == 'dynamic' else '' }}" + openshift_hosted_logging_elasticsearch_ops_cluster_size: "{{ logging_elasticsearch_ops_cluster_size }}" + openshift_hosted_logging_elasticsearch_ops_pvc_dynamic: "{{ 'true' if openshift_hosted_logging_storage_kind | default(none) == 'dynamic' else '' }}" + openshift_hosted_logging_elasticsearch_ops_pvc_size: "{{ openshift.hosted.logging.storage.volume.size if openshift_hosted_logging_storage_kind | default(none) in ['dynamic','nfs' ] else '' }}" + openshift_hosted_logging_elasticsearch_ops_pvc_prefix: "{{ 'logging-es' if openshift_hosted_logging_storage_kind | default(none) =='dynamic' else '' }}" + + - role: cockpit-ui + when: ( openshift.common.version_gte_3_3_or_1_3 | bool ) and ( openshift_hosted_manage_registry | default(true) | bool ) and not (openshift.docker.hosted_registry_insecure | default(false) | bool) diff --git a/playbooks/common/openshift-cluster/redeploy-certificates.yml b/playbooks/common/openshift-cluster/redeploy-certificates.yml new file mode 100644 index 000000000..5f008a045 --- /dev/null +++ b/playbooks/common/openshift-cluster/redeploy-certificates.yml @@ -0,0 +1,255 @@ +--- +- include: evaluate_groups.yml + +- include: initialize_facts.yml + +- include: initialize_openshift_version.yml + +- name: Load openshift_facts + hosts: oo_etcd_to_config:oo_masters_to_config:oo_nodes_to_config + roles: + - openshift_facts + +- name: Redeploy etcd certificates + hosts: oo_etcd_to_config + any_errors_fatal: true + vars: + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_conf_dir: /etc/etcd + etcd_generated_certs_dir: "{{ etcd_conf_dir }}/generated_certs" + + pre_tasks: + - stat: + path: "{{ etcd_generated_certs_dir }}" + register: etcd_generated_certs_dir_stat + - name: Backup etcd certificates + command: > + tar -czvf /etc/etcd/etcd-certificate-backup-{{ ansible_date_time.epoch }}.tgz + {{ etcd_conf_dir }}/ca.crt + {{ etcd_conf_dir }}/ca + {{ etcd_generated_certs_dir }} + when: etcd_generated_certs_dir_stat.stat.exists + delegate_to: "{{ etcd_ca_host }}" + run_once: true + - name: Remove existing etcd certificates + file: + path: "{{ item }}" + state: absent + with_items: + - "{{ etcd_conf_dir }}/ca.crt" + - "{{ etcd_conf_dir }}/ca" + - "{{ etcd_generated_certs_dir }}" + roles: + - role: openshift_etcd_server_certificates + etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" + etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" + etcd_certificates_redeploy: true + +- name: Redeploy master certificates + hosts: oo_masters_to_config + any_errors_fatal: true + vars: + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + openshift_master_count: "{{ openshift.master.master_count | default(groups.oo_masters | length) }}" + pre_tasks: + # set_fact task copied from playbooks/common/openshift-master/config.yml + # so that openshift_master_default_subdomain has a default value of "" + # (emptry string). openshift_master_default_subdomain must have a default + # value for openshift_master_facts to set metrics_public_url. + # TODO: clean this up. + - set_fact: + openshift_master_default_subdomain: "{{ lookup('oo_option', 'openshift_master_default_subdomain') | default(None, true) }}" + when: openshift_master_default_subdomain is not defined + - stat: + path: "{{ openshift_generated_configs_dir }}" + register: openshift_generated_configs_dir_stat + - name: Backup generated certificate and config directories + command: > + tar -czvf /etc/origin/master-node-cert-config-backup-{{ ansible_date_time.epoch }}.tgz + {{ openshift_generated_configs_dir }} + {{ openshift.common.config_base }}/master + when: openshift_generated_configs_dir_stat.stat.exists + delegate_to: "{{ openshift_ca_host }}" + run_once: true + - name: Remove generated certificate directories + file: + path: "{{ item }}" + state: absent + with_items: + - "{{ openshift_generated_configs_dir }}" + - name: Remove generated certificates + file: + path: "{{ openshift.common.config_base }}/master/{{ item }}" + state: absent + with_items: + - "{{ hostvars[inventory_hostname] | certificates_to_synchronize(include_keys=false) }}" + - "etcd.server.crt" + - "etcd.server.key" + - "master.etcd-client.crt" + - "master.etcd-client.key" + - "master.server.crt" + - "master.server.key" + - "openshift-master.crt" + - "openshift-master.key" + - "openshift-master.kubeconfig" + - name: Remove CA certificate + file: + path: "{{ openshift.common.config_base }}/master/{{ item }}" + state: absent + when: openshift_certificates_redeploy_ca | default(false) | bool + with_items: + - "ca.crt" + - "ca.key" + - "ca.serial.txt" + - "ca-bundle.crt" + roles: + - role: openshift_master_certificates + openshift_master_etcd_hosts: "{{ hostvars + | oo_select_keys(groups['oo_etcd_to_config'] | default([])) + | oo_collect('openshift.common.hostname') + | default(none, true) }}" + openshift_master_hostnames: "{{ hostvars + | oo_select_keys(groups['oo_masters_to_config'] | default([])) + | oo_collect('openshift.common.all_hostnames') + | oo_flatten | unique }}" + openshift_certificates_redeploy: true + - role: openshift_etcd_client_certificates + etcd_certificates_redeploy: true + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}" + etcd_cert_config_dir: "{{ openshift.common.config_base }}/master" + etcd_cert_prefix: "master.etcd-" + when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config + +- name: Redeploy node certificates + hosts: oo_nodes_to_config + any_errors_fatal: true + pre_tasks: + - name: Remove CA certificate + file: + path: "{{ item }}" + state: absent + with_items: + - "{{ openshift.common.config_base }}/node/ca.crt" + roles: + - role: openshift_node_certificates + openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}" + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + openshift_certificates_redeploy: true + +- name: Restart etcd + hosts: oo_etcd_to_config + tasks: + - name: restart etcd + service: + name: "{{ 'etcd' if not openshift.common.is_containerized | bool else 'etcd_container' }}" + state: restarted + +- name: Stop master services + hosts: oo_masters_to_config + vars: + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + tasks: + - name: stop master + service: name={{ openshift.common.service_type }}-master state=stopped + when: not openshift_master_ha | bool + - name: stop master api + service: name={{ openshift.common.service_type }}-master-api state=stopped + when: openshift_master_ha | bool and openshift_master_cluster_method == 'native' + - name: stop master controllers + service: name={{ openshift.common.service_type }}-master-controllers state=stopped + when: openshift_master_ha | bool and openshift_master_cluster_method == 'native' + +- name: Start master services + hosts: oo_masters_to_config + serial: 1 + vars: + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + tasks: + - name: start master + service: name={{ openshift.common.service_type }}-master state=started + when: not openshift_master_ha | bool + - name: start master api + service: name={{ openshift.common.service_type }}-master-api state=started + when: openshift_master_ha | bool and openshift_master_cluster_method == 'native' + - name: start master controllers + service: name={{ openshift.common.service_type }}-master-controllers state=started + when: openshift_master_ha | bool and openshift_master_cluster_method == 'native' + +- name: Restart masters (pacemaker) + hosts: oo_first_master + vars: + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + tasks: + - name: restart master + command: pcs resource restart master + when: openshift_master_ha | bool and openshift_master_cluster_method == 'pacemaker' + +- name: Restart nodes + hosts: oo_nodes_to_config + tasks: + - name: restart node + service: name={{ openshift.common.service_type }}-node state=restarted + +- name: Copy admin client config(s) + hosts: oo_first_master + tasks: + - name: Create temp directory for kubeconfig + command: mktemp -d /tmp/openshift-ansible-XXXXXX + register: mktemp + changed_when: False + + - name: Copy admin client config(s) + command: > + cp {{ openshift.common.config_base }}/master//admin.kubeconfig {{ mktemp.stdout }}/admin.kubeconfig + changed_when: False + +- name: Serially evacuate all nodes to trigger redeployments + hosts: oo_nodes_to_config + serial: 1 + any_errors_fatal: true + tasks: + - name: Determine if node is currently scheduleable + command: > + {{ openshift.common.client_binary }} --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig + get node {{ openshift.node.nodename }} -o json + register: node_output + when: openshift_certificates_redeploy_ca | default(false) | bool + delegate_to: "{{ groups.oo_first_master.0 }}" + changed_when: false + + - set_fact: + was_schedulable: "{{ 'unschedulable' not in (node_output.stdout | from_json).spec }}" + when: openshift_certificates_redeploy_ca | default(false) | bool + + - name: Prepare for node evacuation + command: > + {{ openshift.common.client_binary }} adm --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig + manage-node {{ openshift.node.nodename }} + --schedulable=false + delegate_to: "{{ groups.oo_first_master.0 }}" + when: openshift_certificates_redeploy_ca | default(false) | bool and was_schedulable | bool + + - name: Evacuate node + command: > + {{ openshift.common.client_binary }} adm --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig + manage-node {{ openshift.node.nodename }} + --evacuate --force + delegate_to: "{{ groups.oo_first_master.0 }}" + when: openshift_certificates_redeploy_ca | default(false) | bool and was_schedulable | bool + + - name: Set node schedulability + command: > + {{ openshift.common.client_binary }} adm --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig + manage-node {{ openshift.node.nodename }} --schedulable=true + delegate_to: "{{ groups.oo_first_master.0 }}" + when: openshift_certificates_redeploy_ca | default(false) | bool and was_schedulable | bool + +- name: Delete temporary directory + hosts: oo_first_master + tasks: + - name: Delete temp directory + file: + name: "{{ mktemp.stdout }}" + state: absent + changed_when: False diff --git a/playbooks/common/openshift-cluster/set_etcd_launch_facts_tasks.yml b/playbooks/common/openshift-cluster/tasks/set_etcd_launch_facts.yml index 1a6580795..1a6580795 100644 --- a/playbooks/common/openshift-cluster/set_etcd_launch_facts_tasks.yml +++ b/playbooks/common/openshift-cluster/tasks/set_etcd_launch_facts.yml diff --git a/playbooks/common/openshift-cluster/set_master_launch_facts_tasks.yml b/playbooks/common/openshift-cluster/tasks/set_master_launch_facts.yml index 36d7b7870..36d7b7870 100644 --- a/playbooks/common/openshift-cluster/set_master_launch_facts_tasks.yml +++ b/playbooks/common/openshift-cluster/tasks/set_master_launch_facts.yml diff --git a/playbooks/common/openshift-cluster/set_node_launch_facts_tasks.yml b/playbooks/common/openshift-cluster/tasks/set_node_launch_facts.yml index 96e1a9a63..278942f8b 100644 --- a/playbooks/common/openshift-cluster/set_node_launch_facts_tasks.yml +++ b/playbooks/common/openshift-cluster/tasks/set_node_launch_facts.yml @@ -1,11 +1,13 @@ --- -- set_fact: k8s_type="node" +- set_fact: k8s_type=node +- set_fact: sub_host_type="{{ type }}" +- set_fact: number_nodes="{{ count }}" - name: Generate node instance names(s) set_fact: - scratch_name: "{{ cluster_id }}-{{ k8s_type }}-{{ '%05x' | format(1048576 | random) }}" + scratch_name: "{{ cluster_id }}-{{ k8s_type }}-{{ sub_host_type }}-{{ '%05x' | format(1048576 | random) }}" register: node_names_output - with_sequence: count={{ num_nodes }} + with_sequence: count={{ number_nodes }} - set_fact: node_names: "{{ node_names_output.results | default([]) diff --git a/playbooks/common/openshift-cluster/update_repos_and_packages.yml b/playbooks/common/openshift-cluster/update_repos_and_packages.yml index e92c6f1ee..e3d16d359 100644 --- a/playbooks/common/openshift-cluster/update_repos_and_packages.yml +++ b/playbooks/common/openshift-cluster/update_repos_and_packages.yml @@ -1,7 +1,19 @@ --- +- include: evaluate_groups.yml + - hosts: oo_hosts_to_update vars: openshift_deployment_type: "{{ deployment_type }}" roles: + # Explicitly calling openshift_facts because it appears that when + # rhel_subscribe is skipped that the openshift_facts dependency for + # openshift_repos is also skipped (this is the case at least for Ansible + # 2.0.2) + - openshift_facts + - role: rhel_subscribe + when: deployment_type in ["enterprise", "atomic-enterprise", "openshift-enterprise"] and + ansible_distribution == "RedHat" and + lookup('oo_option', 'rhel_skip_subscription') | default(rhsub_skip, True) | + default('no', True) | lower in ['no', 'false'] - openshift_repos - os_update_latest diff --git a/playbooks/common/openshift-cluster/upgrades/atomic-openshift-master.j2 b/playbooks/common/openshift-cluster/upgrades/atomic-openshift-master.j2 new file mode 120000 index 000000000..2441f8887 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/atomic-openshift-master.j2 @@ -0,0 +1 @@ +../../../../roles/openshift_master/templates/atomic-openshift-master.j2
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/cleanup_unused_images.yml b/playbooks/common/openshift-cluster/upgrades/cleanup_unused_images.yml new file mode 100644 index 000000000..6e953be69 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/cleanup_unused_images.yml @@ -0,0 +1,22 @@ +--- +- name: Check Docker image count + shell: "docker images -aq | wc -l" + register: docker_image_count + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- debug: var=docker_image_count.stdout + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- name: Remove unused Docker images for Docker 1.10+ migration + shell: "docker rmi `docker images -aq`" + # Will fail on images still in use: + failed_when: false + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- name: Check Docker image count + shell: "docker images -aq | wc -l" + register: docker_image_count + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- debug: var=docker_image_count.stdout + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool diff --git a/playbooks/common/openshift-cluster/upgrades/containerized_node_upgrade.yml b/playbooks/common/openshift-cluster/upgrades/containerized_node_upgrade.yml new file mode 100644 index 000000000..439df5ffd --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/containerized_node_upgrade.yml @@ -0,0 +1,9 @@ +- name: Update systemd units + include: ../../../../roles/openshift_node/tasks/systemd_units.yml openshift_version={{ openshift_image_tag }} + +- name: Verifying the correct version was configured + shell: grep {{ verify_upgrade_version }} {{ item }} + with_items: + - /etc/sysconfig/openvswitch + - /etc/sysconfig/{{ openshift.common.service_type }}* + when: verify_upgrade_version is defined diff --git a/playbooks/common/openshift-cluster/upgrades/create_service_signer_cert.yml b/playbooks/common/openshift-cluster/upgrades/create_service_signer_cert.yml new file mode 100644 index 000000000..23cf8cf76 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/create_service_signer_cert.yml @@ -0,0 +1,76 @@ +--- +- name: Create local temp directory for syncing certs + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - name: Create local temp directory for syncing certs + local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX + register: local_cert_sync_tmpdir + changed_when: false + when: not (hostvars[groups.oo_first_master.0].service_signer_cert_stat.stat.exists | bool) + +- name: Create service signer certificate + hosts: oo_first_master + tasks: + - name: Create remote temp directory for creating certs + command: mktemp -d /tmp/openshift-ansible-XXXXXXX + register: remote_cert_create_tmpdir + changed_when: false + when: not (hostvars[groups.oo_first_master.0].service_signer_cert_stat.stat.exists | bool) + + - name: Create service signer certificate + command: > + {{ openshift.common.client_binary }} adm ca create-signer-cert + --cert="{{ remote_cert_create_tmpdir.stdout }}/"service-signer.crt + --key="{{ remote_cert_create_tmpdir.stdout }}/"service-signer.key + --name="{{ remote_cert_create_tmpdir.stdout }}/"openshift-service-serving-signer + --serial="{{ remote_cert_create_tmpdir.stdout }}/"service-signer.serial.txt + args: + chdir: "{{ remote_cert_create_tmpdir.stdout }}/" + when: not (hostvars[groups.oo_first_master.0].service_signer_cert_stat.stat.exists | bool) + + - name: Retrieve service signer certificate + fetch: + src: "{{ remote_cert_create_tmpdir.stdout }}/{{ item }}" + dest: "{{ hostvars.localhost.local_cert_sync_tmpdir.stdout }}/" + flat: yes + fail_on_missing: yes + validate_checksum: yes + with_items: + - "service-signer.crt" + - "service-signer.key" + when: not (hostvars[groups.oo_first_master.0].service_signer_cert_stat.stat.exists | bool) + + - name: Delete remote temp directory + file: + name: "{{ remote_cert_create_tmpdir.stdout }}" + state: absent + changed_when: false + when: not (hostvars[groups.oo_first_master.0].service_signer_cert_stat.stat.exists | bool) + +- name: Deploy service signer certificate + hosts: oo_masters_to_config + tasks: + - name: Deploy service signer certificate + copy: + src: "{{ hostvars.localhost.local_cert_sync_tmpdir.stdout }}/{{ item }}" + dest: "{{ openshift.common.config_base }}/master/" + with_items: + - "service-signer.crt" + - "service-signer.key" + when: not (hostvars[groups.oo_first_master.0].service_signer_cert_stat.stat.exists | bool) + +- name: Delete local temp directory + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - name: Delete local temp directory + file: + name: "{{ local_cert_sync_tmpdir.stdout }}" + state: absent + changed_when: false + when: not (hostvars[groups.oo_first_master.0].service_signer_cert_stat.stat.exists | bool) diff --git a/playbooks/common/openshift-cluster/upgrades/docker-cluster b/playbooks/common/openshift-cluster/upgrades/docker-cluster new file mode 120000 index 000000000..055ad09fc --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/docker-cluster @@ -0,0 +1 @@ +../../../../roles/openshift_master/templates/docker-cluster
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/docker/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/docker/upgrade.yml new file mode 100644 index 000000000..417096dd0 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/docker/upgrade.yml @@ -0,0 +1,66 @@ +--- +# We need docker service up to remove all the images, but these services will keep +# trying to re-start and thus re-pull the images we're trying to delete. +- name: Stop containerized services + service: name={{ item }} state=stopped + with_items: + - "{{ openshift.common.service_type }}-master" + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + - etcd_container + - openvswitch + failed_when: false + when: openshift.common.is_containerized | bool + +- name: Check Docker image count + shell: "docker images -aq | wc -l" + register: docker_image_count + +- debug: var=docker_image_count.stdout + +- name: Remove all containers and images + script: nuke_images.sh docker + register: nuke_images_result + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- name: Check Docker image count + shell: "docker images -aq | wc -l" + register: docker_image_count + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- debug: var=docker_image_count.stdout + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- service: name=docker state=stopped + +- name: Upgrade Docker + action: "{{ ansible_pkg_mgr }} name=docker{{ '-' + docker_version }} state=present" + +- service: name=docker state=started + +- name: Update docker facts + openshift_facts: + role: docker + +- name: Restart containerized services + service: name={{ item }} state=started + with_items: + - etcd_container + - openvswitch + - "{{ openshift.common.service_type }}-master" + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + failed_when: false + when: openshift.common.is_containerized | bool + +- name: Wait for master API to come back online + become: no + local_action: + module: wait_for + host="{{ inventory_hostname }}" + state=started + delay=10 + port="{{ openshift.master.api_port }}" + when: inventory_hostname in groups.oo_masters_to_config diff --git a/playbooks/common/openshift-cluster/upgrades/docker/upgrade_check.yml b/playbooks/common/openshift-cluster/upgrades/docker/upgrade_check.yml new file mode 100644 index 000000000..ee75aa853 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/docker/upgrade_check.yml @@ -0,0 +1,53 @@ +--- + +# This snippet determines if a Docker upgrade is required by checking the inventory +# variables, the available packages, and sets l_docker_upgrade to True if so. + +- set_fact: + docker_upgrade: True + when: docker_upgrade is not defined + +- name: Check if Docker is installed + command: rpm -q docker + register: pkg_check + failed_when: pkg_check.rc > 1 + changed_when: no + +- name: Get current version of Docker + command: "{{ repoquery_cmd }} --installed --qf '%{version}' docker" + register: curr_docker_version + changed_when: false + +- name: Get latest available version of Docker + command: > + {{ repoquery_cmd }} --qf '%{version}' "docker" + register: avail_docker_version + # Don't expect docker rpm to be available on hosts that don't already have it installed: + when: pkg_check.rc == 0 + failed_when: false + changed_when: false + +- fail: + msg: This playbook requires access to Docker 1.10 or later + # Disable the 1.10 requirement if the user set a specific Docker version + when: docker_version is not defined and (docker_upgrade is not defined or docker_upgrade | bool == True) and (pkg_check.rc == 0 and (avail_docker_version.stdout == "" or avail_docker_version.stdout | version_compare('1.10','<'))) + +# Default l_docker_upgrade to False, we'll set to True if an upgrade is required: +- set_fact: + l_docker_upgrade: False + +# Make sure a docker_version is set if none was requested: +- set_fact: + docker_version: "{{ avail_docker_version.stdout }}" + when: pkg_check.rc == 0 and docker_version is not defined + +- name: Flag for Docker upgrade if necessary + set_fact: + l_docker_upgrade: True + when: pkg_check.rc == 0 and curr_docker_version.stdout | version_compare(docker_version,'<') + +- name: Flag to delete all images prior to upgrade if crossing Docker 1.10 boundary + set_fact: + docker_upgrade_nuke_images: True + when: l_docker_upgrade | bool and docker_upgrade_nuke_images is not defined and curr_docker_version.stdout | version_compare('1.10','<') and docker_version | version_compare('1.10','>=') + diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml new file mode 100644 index 000000000..57b156b1c --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml @@ -0,0 +1,73 @@ +- name: Backup etcd + hosts: etcd_hosts_to_backup + vars: + embedded_etcd: "{{ hostvars[groups.oo_first_master.0].openshift.master.embedded_etcd }}" + timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" + roles: + - openshift_facts + tasks: + # Ensure we persist the etcd role for this host in openshift_facts + - openshift_facts: + role: etcd + local_facts: {} + when: "'etcd' not in openshift" + + - stat: path=/var/lib/openshift + register: var_lib_openshift + + - stat: path=/var/lib/origin + register: var_lib_origin + + - name: Create origin symlink if necessary + file: src=/var/lib/openshift/ dest=/var/lib/origin state=link + when: var_lib_openshift.stat.exists == True and var_lib_origin.stat.exists == False + + # TODO: replace shell module with command and update later checks + # We assume to be using the data dir for all backups. + - name: Check available disk space for etcd backup + shell: df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1 + register: avail_disk + + # TODO: replace shell module with command and update later checks + - name: Check current embedded etcd disk usage + shell: du -k {{ openshift.etcd.etcd_data_dir }} | tail -n 1 | cut -f1 + register: etcd_disk_usage + when: embedded_etcd | bool + + - name: Abort if insufficient disk space for etcd backup + fail: + msg: > + {{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup, + {{ avail_disk.stdout }} Kb available. + when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int) + + - name: Install etcd (for etcdctl) + action: "{{ ansible_pkg_mgr }} name=etcd state=present" + when: not openshift.common.is_atomic | bool + + - name: Generate etcd backup + command: > + etcdctl backup --data-dir={{ openshift.etcd.etcd_data_dir }} + --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ backup_tag | default('') }}{{ timestamp }} + + - set_fact: + etcd_backup_complete: True + + - name: Display location of etcd backup + debug: + msg: "Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ backup_tag | default('') }}{{ timestamp }}" + +- name: Gate on etcd backup + hosts: localhost + connection: local + become: no + tasks: + - set_fact: + etcd_backup_completed: "{{ hostvars + | oo_select_keys(groups.etcd_hosts_to_backup) + | oo_collect('inventory_hostname', {'etcd_backup_complete': true}) }}" + - set_fact: + etcd_backup_failed: "{{ groups.etcd_hosts_to_backup | difference(etcd_backup_completed) }}" + - fail: + msg: "Upgrade cannot continue. The following hosts did not complete etcd backup: {{ etcd_backup_failed | join(',') }}" + when: etcd_backup_failed | length > 0 diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/containerized_tasks.yml b/playbooks/common/openshift-cluster/upgrades/etcd/containerized_tasks.yml new file mode 100644 index 000000000..35f391f8c --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/containerized_tasks.yml @@ -0,0 +1,47 @@ +--- +- name: Verify cluster is healthy pre-upgrade + command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health" + +- name: Get current image + shell: grep 'ExecStart=' /etc/systemd/system/etcd_container.service | awk '{print $NF}' + register: current_image + +- name: Set new_etcd_image + set_fact: + new_etcd_image: "{{ current_image.stdout | regex_replace('/etcd.*$','/etcd3:' ~ upgrade_version ) if upgrade_version | version_compare('3.0','>=') + else current_image.stdout.split(':')[0] ~ ':' ~ upgrade_version }}" + +- name: Pull new etcd image + command: "docker pull {{ new_etcd_image }}" + +- name: Update to latest etcd image + replace: + dest: /etc/systemd/system/etcd_container.service + regexp: "{{ current_image.stdout }}$" + replace: "{{ new_etcd_image }}" + +- name: Restart etcd_container + systemd: + name: etcd_container + daemon_reload: yes + state: restarted + +## TODO: probably should just move this into the backup playbooks, also this +## will fail on atomic host. We need to revisit how to do etcd backups there as +## the container may be newer than etcdctl on the host. Assumes etcd3 obsoletes etcd (7.3.1) +- name: Upgrade etcd for etcdctl when not atomic + action: "{{ ansible_pkg_mgr }} name=etcd ensure=latest" + when: not openshift.common.is_atomic | bool + +- name: Verify cluster is healthy + command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health" + register: etcdctl + until: etcdctl.rc == 0 + retries: 3 + delay: 10 + +- name: Store new etcd_image + openshift_facts: + role: etcd + local_facts: + etcd_image: "{{ new_etcd_image }}" diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/fedora_tasks.yml b/playbooks/common/openshift-cluster/upgrades/etcd/fedora_tasks.yml new file mode 100644 index 000000000..30232110e --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/fedora_tasks.yml @@ -0,0 +1,23 @@ +--- +# F23 GA'd with etcd 2.0, currently has 2.2 in updates +# F24 GA'd with etcd-2.2, currently has 2.2 in updates +# F25 Beta currently has etcd 3.0 +- name: Verify cluster is healthy pre-upgrade + command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health" + +- name: Update etcd + package: + name: "etcd" + state: "latest" + +- name: Restart etcd + service: + name: etcd + state: restarted + +- name: Verify cluster is healthy + command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health" + register: etcdctl + until: etcdctl.rc == 0 + retries: 3 + delay: 10 diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/files/etcdctl.sh b/playbooks/common/openshift-cluster/upgrades/etcd/files/etcdctl.sh new file mode 120000 index 000000000..641e04e44 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/files/etcdctl.sh @@ -0,0 +1 @@ +../roles/etcd/files/etcdctl.sh
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/filter_plugins b/playbooks/common/openshift-cluster/upgrades/etcd/filter_plugins new file mode 120000 index 000000000..27ddaa18b --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/filter_plugins @@ -0,0 +1 @@ +../../../../../filter_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/lookup_plugins b/playbooks/common/openshift-cluster/upgrades/etcd/lookup_plugins new file mode 120000 index 000000000..cf407f69b --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/lookup_plugins @@ -0,0 +1 @@ +../../../../../lookup_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/main.yml b/playbooks/common/openshift-cluster/upgrades/etcd/main.yml new file mode 100644 index 000000000..cce844403 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/main.yml @@ -0,0 +1,122 @@ +--- +# For 1.4/3.4 we want to upgrade everyone to etcd-3.0. etcd docs say to +# upgrade from 2.0.x to 2.1.x to 2.2.x to 2.3.x to 3.0.x. While this is a tedius +# task for RHEL and CENTOS it's simply not possible in Fedora unless you've +# mirrored packages on your own because only the GA and latest versions are +# available in the repos. So for Fedora we'll simply skip this, sorry. + +- include: ../../evaluate_groups.yml + tags: + - always + +- name: Evaluate additional groups for upgrade + hosts: localhost + connection: local + become: no + tasks: + - name: Evaluate etcd_hosts_to_upgrade + add_host: + name: "{{ item }}" + groups: etcd_hosts_to_upgrade, etcd_hosts_to_backup + with_items: "{{ groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else groups.oo_first_master }}" + +- name: Backup etcd before upgrading anything + include: backup.yml + vars: + backup_tag: "pre-upgrade-" + +- name: Drop etcdctl profiles + hosts: etcd_hosts_to_upgrade + tasks: + - include: roles/etcd/tasks/etcdctl.yml + +- name: Determine etcd version + hosts: etcd_hosts_to_upgrade + tasks: + - name: Record RPM based etcd version + command: rpm -qa --qf '%{version}' etcd\* + register: etcd_installed_version + failed_when: false + when: not openshift.common.is_containerized | bool + - name: Record containerized etcd version + command: docker exec etcd_container rpm -qa --qf '%{version}' etcd\* + register: etcd_installed_version + failed_when: false + when: openshift.common.is_containerized | bool + +# I really dislike this copy/pasta but I wasn't able to find a way to get it to loop +# through hosts, then loop through tasks only when appropriate +- name: Upgrade to 2.1 + hosts: etcd_hosts_to_upgrade + serial: 1 + vars: + upgrade_version: '2.1' + tasks: + - include: rhel_tasks.yml + when: etcd_installed_version.stdout | default('99') | version_compare('2.1','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool + +- name: Upgrade RPM hosts to 2.2 + hosts: etcd_hosts_to_upgrade + serial: 1 + vars: + upgrade_version: '2.2' + tasks: + - include: rhel_tasks.yml + when: etcd_installed_version.stdout | default('99') | version_compare('2.2','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool + +- name: Upgrade containerized hosts to 2.2.5 + hosts: etcd_hosts_to_upgrade + serial: 1 + vars: + upgrade_version: 2.2.5 + tasks: + - include: containerized_tasks.yml + when: etcd_installed_version.stdout | default('99') | version_compare('2.2','<') and openshift.common.is_containerized | bool + +- name: Upgrade RPM hosts to 2.3 + hosts: etcd_hosts_to_upgrade + serial: 1 + vars: + upgrade_version: '2.3' + tasks: + - include: rhel_tasks.yml + when: etcd_installed_version.stdout | default('99') | version_compare('2.3','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool + +- name: Upgrade containerized hosts to 2.3.7 + hosts: etcd_hosts_to_upgrade + serial: 1 + vars: + upgrade_version: 2.3.7 + tasks: + - include: containerized_tasks.yml + when: etcd_installed_version.stdout | default('99') | version_compare('2.3','<') and openshift.common.is_containerized | bool + +- name: Upgrade RPM hosts to 3.0 + hosts: etcd_hosts_to_upgrade + serial: 1 + vars: + upgrade_version: '3.0' + tasks: + - include: rhel_tasks.yml + when: etcd_installed_version.stdout | default('99') | version_compare('3.0','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool + +- name: Upgrade containerized hosts to etcd3 image + hosts: etcd_hosts_to_upgrade + serial: 1 + vars: + upgrade_version: 3.0.3 + tasks: + - include: containerized_tasks.yml + when: etcd_installed_version.stdout | default('99') | version_compare('3.0','<') and openshift.common.is_containerized | bool + +- name: Upgrade fedora to latest + hosts: etcd_hosts_to_upgrade + serial: 1 + tasks: + - include: fedora_tasks.yml + when: ansible_distribution == 'Fedora' and not openshift.common.is_containerized | bool + +- name: Backup etcd + include: backup.yml + vars: + backup_tag: "post-3.0-" diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/rhel_tasks.yml b/playbooks/common/openshift-cluster/upgrades/etcd/rhel_tasks.yml new file mode 100644 index 000000000..8e7dc9d9b --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/rhel_tasks.yml @@ -0,0 +1,23 @@ +--- +- name: Verify cluster is healthy pre-upgrade + command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health" + +- name: Update etcd package but exclude etcd3 + command: "{{ ansible_pkg_mgr }} install -y etcd-{{ upgrade_version }}\\* --exclude etcd3" + when: upgrade_version | version_compare('3.0','<') + +- name: Update etcd package not excluding etcd3 + command: "{{ ansible_pkg_mgr }} install -y etcd3-{{ upgrade_version }}\\*" + when: not upgrade_version | version_compare('3.0','<') + +- name: Restart etcd + service: + name: etcd + state: restarted + +- name: Verify cluster is healthy + command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health" + register: etcdctl + until: etcdctl.rc == 0 + retries: 3 + delay: 10 diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/roles b/playbooks/common/openshift-cluster/upgrades/etcd/roles new file mode 120000 index 000000000..6bc1a7aef --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/etcd/roles @@ -0,0 +1 @@ +../../../../../roles
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/files/nuke_images.sh b/playbooks/common/openshift-cluster/upgrades/files/nuke_images.sh new file mode 100644 index 000000000..8635eab0d --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/files/nuke_images.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Stop any running containers +running_container_ids=`docker ps -q` +if test -n "$running_container_ids" +then + docker stop $running_container_ids +fi + +# Delete all containers +container_ids=`docker ps -a -q` +if test -n "$container_ids" +then + docker rm -f -v $container_ids +fi + +# Delete all images (forcefully) +image_ids=`docker images -aq` +if test -n "$image_ids" +then + # Some layers are deleted recursively and are no longer present + # when docker goes to remove them: + docker rmi -f `docker images -aq` || true +fi + diff --git a/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check b/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check new file mode 100644 index 000000000..e5c958ebb --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check @@ -0,0 +1,193 @@ +#!/usr/bin/env python +""" +Pre-upgrade checks that must be run on a master before proceeding with upgrade. +""" +# This is a script not a python module: +# pylint: disable=invalid-name + +# NOTE: This script should not require any python libs other than what is +# in the standard library. + +__license__ = "ASL 2.0" + +import json +import os +import subprocess +import re + +# The maximum length of container.ports.name +ALLOWED_LENGTH = 15 +# The valid structure of container.ports.name +ALLOWED_CHARS = re.compile('^[a-z0-9][a-z0-9\\-]*[a-z0-9]$') +AT_LEAST_ONE_LETTER = re.compile('[a-z]') +# look at OS_PATH for the full path. Default ot 'oc' +OC_PATH = os.getenv('OC_PATH', 'oc') + + +def validate(value): + """ + validate verifies that value matches required conventions + + Rules of container.ports.name validation: + + * must be less that 16 chars + * at least one letter + * only a-z0-9- + * hyphens can not be leading or trailing or next to each other + + :Parameters: + - `value`: Value to validate + """ + if len(value) > ALLOWED_LENGTH: + return False + + if '--' in value: + return False + + # We search since it can be anywhere + if not AT_LEAST_ONE_LETTER.search(value): + return False + + # We match because it must start at the beginning + if not ALLOWED_CHARS.match(value): + return False + return True + + +def list_items(kind): + """ + list_items returns a list of items from the api + + :Parameters: + - `kind`: Kind of item to access + """ + response = subprocess.check_output([OC_PATH, 'get', '--all-namespaces', '-o', 'json', kind]) + items = json.loads(response) + return items.get("items", []) + + +def get(obj, *paths): + """ + Gets an object + + :Parameters: + - `obj`: A dictionary structure + - `path`: All other non-keyword arguments + """ + ret_obj = obj + for path in paths: + if ret_obj.get(path, None) is None: + return [] + ret_obj = ret_obj[path] + return ret_obj + + +# pylint: disable=too-many-arguments +def pretty_print_errors(namespace, kind, item_name, container_name, invalid_label, port_name, valid): + """ + Prints out results in human friendly way. + + :Parameters: + - `namespace`: Namespace of the resource + - `kind`: Kind of the resource + - `item_name`: Name of the resource + - `container_name`: Name of the container. May be "" when kind=Service. + - `port_name`: Name of the port + - `invalid_label`: The label of the invalid port. Port.name/targetPort + - `valid`: True if the port is valid + """ + if not valid: + if len(container_name) > 0: + print('%s/%s -n %s (Container="%s" %s="%s")' % ( + kind, item_name, namespace, container_name, invalid_label, port_name)) + else: + print('%s/%s -n %s (%s="%s")' % ( + kind, item_name, namespace, invalid_label, port_name)) + + +def print_validation_header(): + """ + Prints the error header. Should run on the first error to avoid + overwhelming the user. + """ + print """\ +At least one port name is invalid and must be corrected before upgrading. +Please update or remove any resources with invalid port names. + + Valid port names must: + + * be less that 16 characters + * have at least one letter + * contain only a-z0-9- + * not start or end with - + * not contain dashes next to each other ('--') +""" + + +def main(): + """ + main is the main entry point to this script + """ + try: + # the comma at the end suppresses the newline + print "Checking for oc ...", + subprocess.check_output([OC_PATH, 'whoami']) + print "found" + except: + print( + 'Unable to run "%s whoami"\n' + 'Please ensure OpenShift is running, and "oc" is on your system ' + 'path.\n' + 'You can override the path with the OC_PATH environment variable.' + % OC_PATH) + raise SystemExit(1) + + # Where the magic happens + first_error = True + for kind, path in [ + ('deploymentconfigs', ("spec", "template", "spec", "containers")), + ('replicationcontrollers', ("spec", "template", "spec", "containers")), + ('pods', ("spec", "containers"))]: + for item in list_items(kind): + namespace = item["metadata"]["namespace"] + item_name = item["metadata"]["name"] + for container in get(item, *path): + container_name = container["name"] + for port in get(container, "ports"): + port_name = port.get("name", None) + if not port_name: + # Unnamed ports are OK + continue + valid = validate(port_name) + if not valid and first_error: + first_error = False + print_validation_header() + pretty_print_errors( + namespace, kind, item_name, + container_name, "Port.name", port_name, valid) + + # Services follow a different flow + for item in list_items('services'): + namespace = item["metadata"]["namespace"] + item_name = item["metadata"]["name"] + for port in get(item, "spec", "ports"): + port_name = port.get("targetPort", None) + if isinstance(port_name, int) or port_name is None: + # Integer only or unnamed ports are OK + continue + valid = validate(port_name) + if not valid and first_error: + first_error = False + print_validation_header() + pretty_print_errors( + namespace, "services", item_name, "", + "targetPort", port_name, valid) + + # If we had at least 1 error then exit with 1 + if not first_error: + raise SystemExit(1) + + +if __name__ == '__main__': + main() + diff --git a/playbooks/common/openshift-cluster/upgrades/files/rpm_versions.sh b/playbooks/common/openshift-cluster/upgrades/files/rpm_versions.sh new file mode 100644 index 000000000..7bf249742 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/files/rpm_versions.sh @@ -0,0 +1,12 @@ +#!/bin/bash +if [ `which dnf 2> /dev/null` ]; then + installed=$(dnf repoquery --installed --latest-limit 1 -d 0 --qf '%{version}-%{release}' "${@}" 2> /dev/null) + available=$(dnf repoquery --available --latest-limit 1 -d 0 --qf '%{version}-%{release}' "${@}" 2> /dev/null) +else + installed=$(repoquery --plugins --pkgnarrow=installed --qf '%{version}-%{release}' "${@}" 2> /dev/null) + available=$(repoquery --plugins --pkgnarrow=available --qf '%{version}-%{release}' "${@}" 2> /dev/null) +fi + +echo "---" +echo "curr_version: ${installed}" +echo "avail_version: ${available}" diff --git a/playbooks/common/openshift-cluster/upgrades/filter_plugins b/playbooks/common/openshift-cluster/upgrades/filter_plugins new file mode 120000 index 000000000..b1213dedb --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/filter_plugins @@ -0,0 +1 @@ +../../../../filter_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/init.yml b/playbooks/common/openshift-cluster/upgrades/init.yml new file mode 100644 index 000000000..fbdb7900a --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/init.yml @@ -0,0 +1,50 @@ +--- +- include: ../verify_ansible_version.yml + +- hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - include_vars: ../../../byo/openshift-cluster/cluster_hosts.yml + - add_host: + name: "{{ item }}" + groups: l_oo_all_hosts + with_items: "{{ g_all_hosts | default([]) }}" + +- hosts: l_oo_all_hosts + gather_facts: no + tasks: + - include_vars: ../../../byo/openshift-cluster/cluster_hosts.yml + +- include: ../evaluate_groups.yml + vars: + # Do not allow adding hosts during upgrade. + g_new_master_hosts: [] + g_new_node_hosts: [] + openshift_cluster_id: "{{ cluster_id | default('default') }}" + openshift_deployment_type: "{{ deployment_type }}" + +- name: Set oo_options + hosts: oo_all_hosts + tasks: + - set_fact: + openshift_docker_additional_registries: "{{ lookup('oo_option', 'docker_additional_registries') }}" + when: openshift_docker_additional_registries is not defined + - set_fact: + openshift_docker_insecure_registries: "{{ lookup('oo_option', 'docker_insecure_registries') }}" + when: openshift_docker_insecure_registries is not defined + - set_fact: + openshift_docker_blocked_registries: "{{ lookup('oo_option', 'docker_blocked_registries') }}" + when: openshift_docker_blocked_registries is not defined + - set_fact: + openshift_docker_options: "{{ lookup('oo_option', 'docker_options') }}" + when: openshift_docker_options is not defined + - set_fact: + openshift_docker_log_driver: "{{ lookup('oo_option', 'docker_log_driver') }}" + when: openshift_docker_log_driver is not defined + - set_fact: + openshift_docker_log_options: "{{ lookup('oo_option', 'docker_log_options') }}" + when: openshift_docker_log_options is not defined + +- include: ../initialize_facts.yml diff --git a/playbooks/common/openshift-cluster/upgrades/initialize_nodes_to_upgrade.yml b/playbooks/common/openshift-cluster/upgrades/initialize_nodes_to_upgrade.yml new file mode 100644 index 000000000..4e375ac26 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/initialize_nodes_to_upgrade.yml @@ -0,0 +1,40 @@ +--- +- name: Filter list of nodes to be upgraded if necessary + hosts: oo_first_master + tasks: + - name: Retrieve list of openshift nodes matching upgrade label + command: > + {{ openshift.common.client_binary }} + get nodes + --config={{ openshift.common.config_base }}/master/admin.kubeconfig + --selector={{ openshift_upgrade_nodes_label }} + -o jsonpath='{.items[*].metadata.name}' + register: matching_nodes + changed_when: false + when: openshift_upgrade_nodes_label is defined + + - set_fact: + nodes_to_upgrade: "{{ matching_nodes.stdout.split(' ') }}" + when: openshift_upgrade_nodes_label is defined + + # We got a list of nodes with the label, now we need to match these with inventory hosts + # using their openshift.common.hostname fact. + - name: Map labelled nodes to inventory hosts + add_host: + name: "{{ item }}" + groups: temp_nodes_to_upgrade + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_become: "{{ g_sudo | default(omit) }}" + with_items: " {{ groups['oo_nodes_to_config'] }}" + when: openshift_upgrade_nodes_label is defined and hostvars[item].openshift.common.hostname in nodes_to_upgrade + changed_when: false + + # Build up the oo_nodes_to_upgrade group, use the list filtered by label if + # present, otherwise hit all nodes: + - name: Evaluate oo_nodes_to_upgrade + add_host: + name: "{{ item }}" + groups: oo_nodes_to_upgrade + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_become: "{{ g_sudo | default(omit) }}" + with_items: "{{ groups['temp_nodes_to_upgrade'] | default(groups['oo_nodes_to_config']) }}" diff --git a/playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py b/playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py new file mode 100755 index 000000000..9a065fd1c --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py @@ -0,0 +1,158 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# vim: expandtab:tabstop=4:shiftwidth=4 + +"""Ansible module for modifying OpenShift configs during an upgrade""" + +import os +import yaml + +DOCUMENTATION = ''' +--- +module: openshift_upgrade_config +short_description: OpenShift Upgrade Config +author: Jason DeTiberus +requirements: [ ] +''' +EXAMPLES = ''' +''' + +def modify_api_levels(level_list, remove, ensure, msg_prepend='', + msg_append=''): + """ modify_api_levels """ + changed = False + changes = [] + + if not isinstance(remove, list): + remove = [] + + if not isinstance(ensure, list): + ensure = [] + + if not isinstance(level_list, list): + new_list = [] + changed = True + changes.append("%s created missing %s" % (msg_prepend, msg_append)) + else: + new_list = level_list + for level in remove: + if level in new_list: + new_list.remove(level) + changed = True + changes.append("%s removed %s %s" % (msg_prepend, level, msg_append)) + + for level in ensure: + if level not in new_list: + new_list.append(level) + changed = True + changes.append("%s added %s %s" % (msg_prepend, level, msg_append)) + + return {'new_list': new_list, 'changed': changed, 'changes': changes} + + +def upgrade_master_3_0_to_3_1(ansible_module, config_base, backup): + """Main upgrade method for 3.0 to 3.1.""" + changes = [] + + # Facts do not get transferred to the hosts where custom modules run, + # need to make some assumptions here. + master_config = os.path.join(config_base, 'master/master-config.yaml') + + master_cfg_file = open(master_config, 'r') + config = yaml.safe_load(master_cfg_file.read()) + master_cfg_file.close() + + + # Remove unsupported api versions and ensure supported api versions from + # master config + unsupported_levels = ['v1beta1', 'v1beta2', 'v1beta3'] + supported_levels = ['v1'] + + result = modify_api_levels(config.get('apiLevels'), unsupported_levels, + supported_levels, 'master-config.yaml:', 'from apiLevels') + if result['changed']: + config['apiLevels'] = result['new_list'] + changes.append(result['changes']) + + if 'kubernetesMasterConfig' in config and 'apiLevels' in config['kubernetesMasterConfig']: + config['kubernetesMasterConfig'].pop('apiLevels') + changes.append('master-config.yaml: removed kubernetesMasterConfig.apiLevels') + + # Add masterCA to serviceAccountConfig + if 'serviceAccountConfig' in config and 'masterCA' not in config['serviceAccountConfig']: + config['serviceAccountConfig']['masterCA'] = config['oauthConfig'].get('masterCA', 'ca.crt') + + # Add proxyClientInfo to master-config + if 'proxyClientInfo' not in config['kubernetesMasterConfig']: + config['kubernetesMasterConfig']['proxyClientInfo'] = { + 'certFile': 'master.proxy-client.crt', + 'keyFile': 'master.proxy-client.key' + } + changes.append("master-config.yaml: added proxyClientInfo") + + if len(changes) > 0: + if backup: + # TODO: Check success: + ansible_module.backup_local(master_config) + + # Write the modified config: + out_file = open(master_config, 'w') + out_file.write(yaml.safe_dump(config, default_flow_style=False)) + out_file.close() + + return changes + + +def upgrade_master(ansible_module, config_base, from_version, to_version, backup): + """Upgrade entry point.""" + if from_version == '3.0': + if to_version == '3.1': + return upgrade_master_3_0_to_3_1(ansible_module, config_base, backup) + + +def main(): + """ main """ + # disabling pylint errors for global-variable-undefined and invalid-name + # for 'global module' usage, since it is required to use ansible_facts + # pylint: disable=global-variable-undefined, invalid-name, + # redefined-outer-name + global module + + module = AnsibleModule( + argument_spec=dict( + config_base=dict(required=True), + from_version=dict(required=True, choices=['3.0']), + to_version=dict(required=True, choices=['3.1']), + role=dict(required=True, choices=['master']), + backup=dict(required=False, default=True, type='bool') + ), + supports_check_mode=True, + ) + + from_version = module.params['from_version'] + to_version = module.params['to_version'] + role = module.params['role'] + backup = module.params['backup'] + config_base = module.params['config_base'] + + try: + changes = [] + if role == 'master': + changes = upgrade_master(module, config_base, from_version, + to_version, backup) + + changed = len(changes) > 0 + return module.exit_json(changed=changed, changes=changes) + + # ignore broad-except error to avoid stack trace to ansible user + # pylint: disable=broad-except + except Exception, e: + return module.fail_json(msg=str(e)) + +# ignore pylint errors related to the module_utils import +# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import +# import module snippets +from ansible.module_utils.basic import * + +if __name__ == '__main__': + main() diff --git a/playbooks/common/openshift-cluster/upgrades/lookup_plugins b/playbooks/common/openshift-cluster/upgrades/lookup_plugins new file mode 120000 index 000000000..aff753026 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/lookup_plugins @@ -0,0 +1 @@ +../../../../lookup_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/master_docker b/playbooks/common/openshift-cluster/upgrades/master_docker new file mode 120000 index 000000000..6aeca2842 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/master_docker @@ -0,0 +1 @@ +../../../../roles/openshift_master/templates/master_docker
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/native-cluster b/playbooks/common/openshift-cluster/upgrades/native-cluster new file mode 120000 index 000000000..4af88e666 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/native-cluster @@ -0,0 +1 @@ +../../../../roles/openshift_master/templates/native-cluster
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/openshift.docker.node.dep.service b/playbooks/common/openshift-cluster/upgrades/openshift.docker.node.dep.service new file mode 120000 index 000000000..add8b7fa9 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/openshift.docker.node.dep.service @@ -0,0 +1 @@ +../../../../roles/openshift_node/templates/openshift.docker.node.dep.service
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/openshift.docker.node.service b/playbooks/common/openshift-cluster/upgrades/openshift.docker.node.service new file mode 120000 index 000000000..ed181633d --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/openshift.docker.node.service @@ -0,0 +1 @@ +../../../../roles/openshift_node/templates/openshift.docker.node.service
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/openvswitch-avoid-oom.conf b/playbooks/common/openshift-cluster/upgrades/openvswitch-avoid-oom.conf new file mode 120000 index 000000000..514526fe2 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/openvswitch-avoid-oom.conf @@ -0,0 +1 @@ +../../../../roles/openshift_node/templates/openvswitch-avoid-oom.conf
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/openvswitch.docker.service b/playbooks/common/openshift-cluster/upgrades/openvswitch.docker.service new file mode 120000 index 000000000..c21e895f2 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/openvswitch.docker.service @@ -0,0 +1 @@ +../../../../roles/openshift_node/templates/openvswitch.docker.service
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/openvswitch.sysconfig.j2 b/playbooks/common/openshift-cluster/upgrades/openvswitch.sysconfig.j2 new file mode 120000 index 000000000..ead6904c4 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/openvswitch.sysconfig.j2 @@ -0,0 +1 @@ +../../../../roles/openshift_node/templates/openvswitch.sysconfig.j2
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/post_control_plane.yml b/playbooks/common/openshift-cluster/upgrades/post_control_plane.yml new file mode 100644 index 000000000..2bbcbe1f8 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/post_control_plane.yml @@ -0,0 +1,76 @@ +--- +############################################################################### +# Post upgrade - Upgrade default router, default registry and examples +############################################################################### +- name: Upgrade default router and default registry + hosts: oo_first_master + vars: + openshift_deployment_type: "{{ deployment_type }}" + registry_image: "{{ openshift.master.registry_url | replace( '${component}', 'docker-registry' ) | replace ( '${version}', openshift_image_tag ) }}" + router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', openshift_image_tag ) }}" + oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig" + roles: + - openshift_manageiq + # Create the new templates shipped in 3.2, existing templates are left + # unmodified. This prevents the subsequent role definition for + # openshift_examples from failing when trying to replace templates that do + # not already exist. We could have potentially done a replace --force to + # create and update in one step. + - openshift_examples + - openshift_hosted_templates + # Update the existing templates + - role: openshift_examples + registry_url: "{{ openshift.master.registry_url }}" + openshift_examples_import_command: replace + - role: openshift_hosted_templates + registry_url: "{{ openshift.master.registry_url }}" + openshift_hosted_templates_import_command: replace + pre_tasks: + - name: Collect all routers + command: > + {{ oc_cmd }} get pods --all-namespaces -l 'router' -o json + register: all_routers + failed_when: false + changed_when: false + + - set_fact: haproxy_routers="{{ (all_routers.stdout | from_json)['items'] | oo_pods_match_component(openshift_deployment_type, 'haproxy-router') | oo_select_keys_from_list(['metadata']) }}" + when: all_routers.rc == 0 + + - set_fact: haproxy_routers=[] + when: all_routers.rc != 0 + + - name: Update router image to current version + when: all_routers.rc == 0 + command: > + {{ oc_cmd }} patch dc/{{ item['labels']['deploymentconfig'] }} -n {{ item['namespace'] }} -p + '{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}","livenessProbe":{"tcpSocket":null,"httpGet":{"path": "/healthz", "port": 1936, "host": "localhost", "scheme": "HTTP"},"initialDelaySeconds":10,"timeoutSeconds":1}}]}}}}' + --api-version=v1 + with_items: "{{ haproxy_routers }}" + + - name: Check for default registry + command: > + {{ oc_cmd }} get -n default dc/docker-registry + register: _default_registry + failed_when: false + changed_when: false + + - name: Update registry image to current version + when: _default_registry.rc == 0 + command: > + {{ oc_cmd }} patch dc/docker-registry -n default -p + '{"spec":{"template":{"spec":{"containers":[{"name":"registry","image":"{{ registry_image }}"}]}}}}' + --api-version=v1 + +# Check for warnings to be printed at the end of the upgrade: +- name: Check for warnings + hosts: oo_masters_to_config + tasks: + # Check if any masters are using pluginOrderOverride and warn if so, only for 1.3/3.3 and beyond: + - command: > + grep pluginOrderOverride {{ openshift.common.config_base }}/master/master-config.yaml + register: grep_plugin_order_override + when: openshift.common.version_gte_3_3_or_1_3 | bool + failed_when: false + - name: Warn if pluginOrderOverride is in use in master-config.yaml + debug: msg="WARNING pluginOrderOverride is being deprecated in master-config.yaml, please see https://docs.openshift.com/enterprise/latest/architecture/additional_concepts/admission_controllers.html for more information." + when: not grep_plugin_order_override | skipped and grep_plugin_order_override.rc == 0 diff --git a/playbooks/common/openshift-cluster/upgrades/pre/gate_checks.yml b/playbooks/common/openshift-cluster/upgrades/pre/gate_checks.yml new file mode 100644 index 000000000..8ecae4539 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/pre/gate_checks.yml @@ -0,0 +1,6 @@ +--- +- name: Flag pre-upgrade checks complete for hosts without errors + hosts: oo_masters_to_config:oo_nodes_to_upgrade:oo_etcd_to_config + tasks: + - set_fact: + pre_upgrade_complete: True diff --git a/playbooks/common/openshift-cluster/upgrades/pre/roles b/playbooks/common/openshift-cluster/upgrades/pre/roles new file mode 120000 index 000000000..415645be6 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/pre/roles @@ -0,0 +1 @@ +../../../../../roles/
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/pre/verify_control_plane_running.yml b/playbooks/common/openshift-cluster/upgrades/pre/verify_control_plane_running.yml new file mode 100644 index 000000000..06eb5f936 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/pre/verify_control_plane_running.yml @@ -0,0 +1,31 @@ +--- +- name: Verify master processes + hosts: oo_masters_to_config + roles: + - openshift_facts + tasks: + - openshift_facts: + role: master + local_facts: + ha: "{{ groups.oo_masters_to_config | length > 1 }}" + + - name: Ensure Master is running + service: + name: "{{ openshift.common.service_type }}-master" + state: started + enabled: yes + when: openshift.master.ha is defined and not openshift.master.ha | bool and openshift.common.is_containerized | bool + + - name: Ensure HA Master is running + service: + name: "{{ openshift.common.service_type }}-master-api" + state: started + enabled: yes + when: openshift.master.ha is defined and openshift.master.ha | bool and openshift.common.is_containerized | bool + + - name: Ensure HA Master is running + service: + name: "{{ openshift.common.service_type }}-master-controllers" + state: started + enabled: yes + when: openshift.master.ha is defined and openshift.master.ha | bool and openshift.common.is_containerized | bool diff --git a/playbooks/common/openshift-cluster/upgrades/pre/verify_docker_upgrade_targets.yml b/playbooks/common/openshift-cluster/upgrades/pre/verify_docker_upgrade_targets.yml new file mode 100644 index 000000000..ba4d77617 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/pre/verify_docker_upgrade_targets.yml @@ -0,0 +1,23 @@ +--- +- name: Verify docker upgrade targets + hosts: oo_masters_to_config:oo_nodes_to_upgrade:oo_etcd_to_config + tasks: + # Only check if docker upgrade is required if docker_upgrade is not + # already set to False. + - include: ../docker/upgrade_check.yml + when: docker_upgrade is not defined or docker_upgrade | bool and not openshift.common.is_atomic | bool + + # Additional checks for Atomic hosts: + + - name: Determine available Docker + shell: "rpm -q --queryformat '---\ncurr_version: %{VERSION}\navail_version: \n' docker" + register: g_atomic_docker_version_result + when: openshift.common.is_atomic | bool + + - set_fact: + l_docker_version: "{{ g_atomic_docker_version_result.stdout | from_yaml }}" + when: openshift.common.is_atomic | bool + + - fail: + msg: This playbook requires access to Docker 1.10 or later + when: openshift.common.is_atomic | bool and l_docker_version.avail_version | default(l_docker_version.curr_version, true) | version_compare('1.10','<') diff --git a/playbooks/common/openshift-cluster/upgrades/pre/verify_inventory_vars.yml b/playbooks/common/openshift-cluster/upgrades/pre/verify_inventory_vars.yml new file mode 100644 index 000000000..9a959a959 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/pre/verify_inventory_vars.yml @@ -0,0 +1,37 @@ +--- +- name: Verify upgrade can proceed on first master + hosts: oo_first_master + gather_facts: no + tasks: + - fail: + msg: > + This upgrade is only supported for origin, openshift-enterprise, and online + deployment types + when: deployment_type not in ['origin','openshift-enterprise', 'online'] + + # Error out in situations where the user has older versions specified in their + # inventory in any of the openshift_release, openshift_image_tag, and + # openshift_pkg_version variables. These must be removed or updated to proceed + # with upgrade. + # TODO: Should we block if you're *over* the next major release version as well? + - fail: + msg: > + openshift_pkg_version is {{ openshift_pkg_version }} which is not a + valid version for a {{ openshift_upgrade_target }} upgrade + when: openshift_pkg_version is defined and openshift_pkg_version.split('-',1).1 | version_compare(openshift_upgrade_target ,'<') + + - fail: + msg: > + openshift_image_tag is {{ openshift_image_tag }} which is not a + valid version for a {{ openshift_upgrade_target }} upgrade + when: openshift_image_tag is defined and openshift_image_tag.split('v',1).1 | version_compare(openshift_upgrade_target ,'<') + + - set_fact: + openshift_release: "{{ openshift_release[1:] }}" + when: openshift_release is defined and openshift_release[0] == 'v' + + - fail: + msg: > + openshift_release is {{ openshift_release }} which is not a + valid release for a {{ openshift_upgrade_target }} upgrade + when: openshift_release is defined and not openshift_release | version_compare(openshift_upgrade_target ,'=') diff --git a/playbooks/common/openshift-cluster/upgrades/pre/verify_nodes_running.yml b/playbooks/common/openshift-cluster/upgrades/pre/verify_nodes_running.yml new file mode 100644 index 000000000..354af3cde --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/pre/verify_nodes_running.yml @@ -0,0 +1,13 @@ +--- +- name: Verify node processes + hosts: oo_nodes_to_config + roles: + - openshift_facts + - openshift_docker_facts + tasks: + - name: Ensure Node is running + service: + name: "{{ openshift.common.service_type }}-node" + state: started + enabled: yes + when: openshift.common.is_containerized | bool diff --git a/playbooks/common/openshift-cluster/upgrades/pre/verify_upgrade_targets.yml b/playbooks/common/openshift-cluster/upgrades/pre/verify_upgrade_targets.yml new file mode 100644 index 000000000..9632626a4 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/pre/verify_upgrade_targets.yml @@ -0,0 +1,45 @@ +--- +- name: Verify upgrade targets + hosts: oo_masters_to_config:oo_nodes_to_upgrade + vars: + openshift_docker_hosted_registry_network: "{{ hostvars[groups.oo_first_master.0].openshift.common.portal_net }}" + pre_tasks: + - fail: + msg: Verify OpenShift is already installed + when: openshift.common.version is not defined + + - fail: + msg: Verify the correct version was found + when: verify_upgrade_version is defined and openshift_version != verify_upgrade_version + + - name: Clean package cache + command: "{{ ansible_pkg_mgr }} clean all" + when: not openshift.common.is_atomic | bool + + - set_fact: + g_new_service_name: "{{ 'origin' if deployment_type =='origin' else 'atomic-openshift' }}" + when: not openshift.common.is_containerized | bool + + - name: Verify containers are available for upgrade + command: > + docker pull {{ openshift.common.cli_image }}:{{ openshift_image_tag }} + register: pull_result + changed_when: "'Downloaded newer image' in pull_result.stdout" + when: openshift.common.is_containerized | bool + + - name: Check latest available OpenShift RPM version + command: > + {{ repoquery_cmd }} --qf '%{version}' "{{ openshift.common.service_type }}" + failed_when: false + changed_when: false + register: avail_openshift_version + when: not openshift.common.is_containerized | bool + + - name: Verify OpenShift RPMs are available for upgrade + fail: + msg: "OpenShift {{ avail_openshift_version.stdout }} is available, but {{ openshift_upgrade_target }} or greater is required" + when: not openshift.common.is_containerized | bool and not avail_openshift_version | skipped and avail_openshift_version.stdout | default('0.0', True) | version_compare(openshift_release, '<') + + - fail: + msg: "This upgrade playbook must be run against OpenShift {{ openshift_upgrade_min }} or later" + when: deployment_type == 'origin' and openshift.common.version | version_compare(openshift_upgrade_min,'<') diff --git a/playbooks/common/openshift-cluster/upgrades/roles b/playbooks/common/openshift-cluster/upgrades/roles new file mode 120000 index 000000000..4bdbcbad3 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/roles @@ -0,0 +1 @@ +../../../../roles
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/rpm_upgrade.yml b/playbooks/common/openshift-cluster/upgrades/rpm_upgrade.yml new file mode 100644 index 000000000..cd1139b29 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/rpm_upgrade.yml @@ -0,0 +1,11 @@ +# We verified latest rpm available is suitable, so just yum update. +- name: Upgrade packages + action: "{{ ansible_pkg_mgr }} name={{ openshift.common.service_type }}-{{ component }}{{ openshift_pkg_version }} state=present" + +- name: Ensure python-yaml present for config upgrade + action: "{{ ansible_pkg_mgr }} name=PyYAML state=present" + when: not openshift.common.is_atomic | bool + +- name: Restart node service + service: name="{{ openshift.common.service_type }}-node" state=restarted + when: component == "node" diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml new file mode 100644 index 000000000..57c25aa41 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml @@ -0,0 +1,200 @@ +--- +############################################################################### +# Upgrade Masters +############################################################################### +- name: Evaluate additional groups for upgrade + hosts: localhost + connection: local + become: no + tasks: + - name: Evaluate etcd_hosts_to_backup + add_host: + name: "{{ item }}" + groups: etcd_hosts_to_backup + with_items: "{{ groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else groups.oo_first_master }}" + +# If facts cache were for some reason deleted, this fact may not be set, and if not set +# it will always default to true. This causes problems for the etcd data dir fact detection +# so we must first make sure this is set correctly before attempting the backup. +- name: Set master embedded_etcd fact + hosts: oo_masters_to_config + roles: + - openshift_facts + tasks: + - openshift_facts: + role: master + local_facts: + embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" + debug_level: "{{ openshift_master_debug_level | default(openshift.common.debug_level | default(2)) }}" + +- name: Backup etcd + include: ./etcd/backup.yml + +- name: Upgrade master packages + hosts: oo_masters_to_config + handlers: + - include: ../../../../roles/openshift_master/handlers/main.yml + static: yes + roles: + - openshift_facts + tasks: + - include: rpm_upgrade.yml component=master + when: not openshift.common.is_containerized | bool + +# Create service signer cert when missing. Service signer certificate +# is added to master config in the master config hook for v3_3. +- name: Determine if service signer cert must be created + hosts: oo_first_master + tasks: + - name: Determine if service signer certificate must be created + stat: + path: "{{ openshift.common.config_base }}/master/service-signer.crt" + register: service_signer_cert_stat + changed_when: false + +- include: create_service_signer_cert.yml + +- name: Upgrade master config and systemd units + hosts: oo_masters_to_config + handlers: + - include: ../../../../roles/openshift_master/handlers/main.yml + static: yes + roles: + - openshift_facts + tasks: + - include: "{{ master_config_hook }}" + when: master_config_hook is defined + + - include_vars: ../../../../roles/openshift_master/vars/main.yml + + - name: Update systemd units + include: ../../../../roles/openshift_master/tasks/systemd_units.yml + + - name: Check for ca-bundle.crt + stat: + path: "{{ openshift.common.config_base }}/master/ca-bundle.crt" + register: ca_bundle_stat + failed_when: false + + - name: Check for ca.crt + stat: + path: "{{ openshift.common.config_base }}/master/ca.crt" + register: ca_crt_stat + failed_when: false + + - name: Migrate ca.crt to ca-bundle.crt + command: mv ca.crt ca-bundle.crt + args: + chdir: "{{ openshift.common.config_base }}/master" + when: ca_crt_stat.stat.isreg and not ca_bundle_stat.stat.exists + + - name: Link ca.crt to ca-bundle.crt + file: + src: "{{ openshift.common.config_base }}/master/ca-bundle.crt" + path: "{{ openshift.common.config_base }}/master/ca.crt" + state: link + when: ca_crt_stat.stat.isreg and not ca_bundle_stat.stat.exists + +- name: Set master update status to complete + hosts: oo_masters_to_config + tasks: + - set_fact: + master_update_complete: True + +############################################################################## +# Gate on master update complete +############################################################################## +- name: Gate on master update + hosts: localhost + connection: local + become: no + tasks: + - set_fact: + master_update_completed: "{{ hostvars + | oo_select_keys(groups.oo_masters_to_config) + | oo_collect('inventory_hostname', {'master_update_complete': true}) }}" + - set_fact: + master_update_failed: "{{ groups.oo_masters_to_config | difference(master_update_completed) }}" + - fail: + msg: "Upgrade cannot continue. The following masters did not finish updating: {{ master_update_failed | join(',') }}" + when: master_update_failed | length > 0 + +# We are now ready to restart master services (or entire system +# depending on openshift_rolling_restart_mode): +- include: ../../openshift-master/restart.yml + +############################################################################### +# Reconcile Cluster Roles, Cluster Role Bindings and Security Context Constraints +############################################################################### + +- name: Reconcile Cluster Roles and Cluster Role Bindings and Security Context Constraints + hosts: oo_masters_to_config + roles: + - { role: openshift_cli } + vars: + origin_reconcile_bindings: "{{ deployment_type == 'origin' and openshift_version | version_compare('1.0.6', '>') }}" + ent_reconcile_bindings: true + openshift_docker_hosted_registry_network: "{{ hostvars[groups.oo_first_master.0].openshift.common.portal_net }}" + # Another spot where we assume docker is running and do not want to accidentally trigger an unsafe + # restart. + skip_docker_role: True + tasks: + - name: Reconcile Cluster Roles + command: > + {{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig + policy reconcile-cluster-roles --additive-only=true --confirm + run_once: true + + - name: Reconcile Cluster Role Bindings + command: > + {{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig + policy reconcile-cluster-role-bindings + --exclude-groups=system:authenticated + --exclude-groups=system:authenticated:oauth + --exclude-groups=system:unauthenticated + --exclude-users=system:anonymous + --additive-only=true --confirm + when: origin_reconcile_bindings | bool or ent_reconcile_bindings | bool + run_once: true + + - name: Reconcile Jenkins Pipeline Role Bindings + command: > + {{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig policy reconcile-cluster-role-bindings system:build-strategy-jenkinspipeline --confirm + run_once: true + when: openshift.common.version_gte_3_4_or_1_4 | bool + + - name: Reconcile Security Context Constraints + command: > + {{ openshift.common.client_binary }} adm policy reconcile-sccs --confirm --additive-only=true + run_once: true + + - set_fact: + reconcile_complete: True + +############################################################################## +# Gate on reconcile +############################################################################## +- name: Gate on reconcile + hosts: localhost + connection: local + become: no + tasks: + - set_fact: + reconcile_completed: "{{ hostvars + | oo_select_keys(groups.oo_masters_to_config) + | oo_collect('inventory_hostname', {'reconcile_complete': true}) }}" + - set_fact: + reconcile_failed: "{{ groups.oo_masters_to_config | difference(reconcile_completed) }}" + - fail: + msg: "Upgrade cannot continue. The following masters did not finish reconciling: {{ reconcile_failed | join(',') }}" + when: reconcile_failed | length > 0 + +- name: Upgrade Docker on dedicated containerized etcd hosts + hosts: oo_etcd_to_config:!oo_nodes_to_upgrade + serial: 1 + any_errors_fatal: true + roles: + - openshift_facts + tasks: + - include: docker/upgrade.yml + when: l_docker_upgrade is defined and l_docker_upgrade | bool and not openshift.common.is_atomic | bool diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml b/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml new file mode 100644 index 000000000..1f314c854 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml @@ -0,0 +1,75 @@ +--- +- name: Evacuate and upgrade nodes + hosts: oo_nodes_to_upgrade + # This var must be set with -e on invocation, as it is not a per-host inventory var + # and is evaluated early. Values such as "20%" can also be used. + serial: "{{ openshift_upgrade_nodes_serial | default(1) }}" + any_errors_fatal: true + roles: + - openshift_facts + - docker + handlers: + - include: ../../../../roles/openshift_node/handlers/main.yml + static: yes + pre_tasks: + # TODO: To better handle re-trying failed upgrades, it would be nice to check if the node + # or docker actually needs an upgrade before proceeding. Perhaps best to save this until + # we merge upgrade functionality into the base roles and a normal config.yml playbook run. + - name: Determine if node is currently scheduleable + command: > + {{ openshift.common.client_binary }} get node {{ openshift.node.nodename | lower }} -o json + register: node_output + delegate_to: "{{ groups.oo_first_master.0 }}" + changed_when: false + when: inventory_hostname in groups.oo_nodes_to_upgrade + + - set_fact: + was_schedulable: "{{ 'unschedulable' not in (node_output.stdout | from_json).spec }}" + when: inventory_hostname in groups.oo_nodes_to_upgrade + + - name: Mark unschedulable if host is a node + command: > + {{ openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --schedulable=false + delegate_to: "{{ groups.oo_first_master.0 }}" + when: inventory_hostname in groups.oo_nodes_to_upgrade + # NOTE: There is a transient "object has been modified" error here, allow a couple + # retries for a more reliable upgrade. + register: node_unsched + until: node_unsched.rc == 0 + retries: 3 + delay: 1 + + - name: Evacuate Node for Kubelet upgrade + command: > + {{ openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --evacuate --force + delegate_to: "{{ groups.oo_first_master.0 }}" + when: inventory_hostname in groups.oo_nodes_to_upgrade + tasks: + - include: docker/upgrade.yml + when: l_docker_upgrade is defined and l_docker_upgrade | bool and not openshift.common.is_atomic | bool + + - include: "{{ node_config_hook }}" + when: node_config_hook is defined and inventory_hostname in groups.oo_nodes_to_upgrade + + - include: rpm_upgrade.yml + vars: + component: "node" + openshift_version: "{{ openshift_pkg_version | default('') }}" + when: inventory_hostname in groups.oo_nodes_to_upgrade and not openshift.common.is_containerized | bool + + - include: containerized_node_upgrade.yml + when: inventory_hostname in groups.oo_nodes_to_upgrade and openshift.common.is_containerized | bool + + - meta: flush_handlers + + - name: Set node schedulability + command: > + {{ openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --schedulable=true + delegate_to: "{{ groups.oo_first_master.0 }}" + when: inventory_hostname in groups.oo_nodes_to_upgrade and was_schedulable | bool + register: node_sched + until: node_sched.rc == 0 + retries: 3 + delay: 1 + + diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/nuke_images.sh b/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/nuke_images.sh new file mode 120000 index 000000000..49a51bba9 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/nuke_images.sh @@ -0,0 +1 @@ +../files/nuke_images.sh
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/v3_3/master_config_upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_3/master_config_upgrade.yml new file mode 100644 index 000000000..684eea343 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_3/master_config_upgrade.yml @@ -0,0 +1,50 @@ +--- +- modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'masterClients.externalKubernetesClientConnectionOverrides.acceptContentTypes' + yaml_value: 'application/vnd.kubernetes.protobuf,application/json' + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'masterClients.externalKubernetesClientConnectionOverrides.contentType' + yaml_value: 'application/vnd.kubernetes.protobuf' + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'masterClients.externalKubernetesClientConnectionOverrides.burst' + yaml_value: 400 + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'masterClients.externalKubernetesClientConnectionOverrides.qps' + yaml_value: 200 + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'masterClients.openshiftLoopbackClientConnectionOverrides.acceptContentTypes' + yaml_value: 'application/vnd.kubernetes.protobuf,application/json' + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'masterClients.openshiftLoopbackClientConnectionOverrides.contentType' + yaml_value: 'application/vnd.kubernetes.protobuf' + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'masterClients.openshiftLoopbackClientConnectionOverrides.burst' + yaml_value: 600 + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'masterClients.openshiftLoopbackClientConnectionOverrides.qps' + yaml_value: 300 + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'controllerConfig.servicesServingCert.signer.certFile' + yaml_value: service-signer.crt + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'controllerConfig.servicesServingCert.signer.keyFile' + yaml_value: service-signer.key diff --git a/playbooks/common/openshift-cluster/upgrades/v3_3/node_config_upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_3/node_config_upgrade.yml new file mode 100644 index 000000000..8f64636ae --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_3/node_config_upgrade.yml @@ -0,0 +1,21 @@ +--- +- modify_yaml: + dest: "{{ openshift.common.config_base}}/node/node-config.yaml" + yaml_key: 'masterClientConnectionOverrides.acceptContentTypes' + yaml_value: 'application/vnd.kubernetes.protobuf,application/json' + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/node/node-config.yaml" + yaml_key: 'masterClientConnectionOverrides.contentType' + yaml_value: 'application/vnd.kubernetes.protobuf' + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/node/node-config.yaml" + yaml_key: 'masterClientConnectionOverrides.burst' + yaml_value: 40 + +- modify_yaml: + dest: "{{ openshift.common.config_base}}/node/node-config.yaml" + yaml_key: 'masterClientConnectionOverrides.qps' + yaml_value: 20 + diff --git a/playbooks/common/openshift-cluster/validate_hostnames.yml b/playbooks/common/openshift-cluster/validate_hostnames.yml new file mode 100644 index 000000000..50e25984f --- /dev/null +++ b/playbooks/common/openshift-cluster/validate_hostnames.yml @@ -0,0 +1,16 @@ +--- +- name: Gather and set facts for node hosts + hosts: oo_nodes_to_config + roles: + - openshift_facts + tasks: + - shell: + getent ahostsv4 {{ openshift.common.hostname }} | head -n 1 | awk '{ print $1 }' + register: lookupip + changed_when: false + failed_when: false + - name: Warn user about bad openshift_hostname values + pause: + prompt: "The hostname \"{{ openshift.common.hostname }}\" for \"{{ ansible_nodename }}\" doesn't resolve to an ip address owned by this host. Please set openshift_hostname variable to a hostname that when resolved on the host in question resolves to an IP address matching an interface on this host. This host will fail liveness checks for pods utilizing hostPorts, press ENTER to continue or CTRL-C to abort." + seconds: "{{ 10 if openshift_override_hostname_check | default(false) | bool else omit }}" + when: lookupip.stdout not in ansible_all_ipv4_addresses diff --git a/playbooks/common/openshift-cluster/verify_ansible_version.yml b/playbooks/common/openshift-cluster/verify_ansible_version.yml new file mode 100644 index 000000000..d75b23bf7 --- /dev/null +++ b/playbooks/common/openshift-cluster/verify_ansible_version.yml @@ -0,0 +1,11 @@ +--- +- name: Verify Ansible version is greater than or equal to 2.1.0.0 + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - name: Verify Ansible version is greater than or equal to 2.1.0.0 + fail: + msg: "Unsupported ansible version: {{ ansible_version.full }} found" + when: not ansible_version.full | version_compare('2.1.0.0', 'ge') diff --git a/playbooks/common/openshift-etcd/config.yml b/playbooks/common/openshift-etcd/config.yml index 3cc561ba0..1b8106e0e 100644 --- a/playbooks/common/openshift-etcd/config.yml +++ b/playbooks/common/openshift-etcd/config.yml @@ -1,96 +1,10 @@ --- -- name: Set etcd facts needed for generating certs +- name: Configure etcd hosts: oo_etcd_to_config + any_errors_fatal: true roles: - - openshift_facts - tasks: - - openshift_facts: - role: "{{ item.role }}" - local_facts: "{{ item.local_facts }}" - with_items: - - role: common - local_facts: - hostname: "{{ openshift_hostname | default(None) }}" - public_hostname: "{{ openshift_public_hostname | default(None) }}" - deployment_type: "{{ openshift_deployment_type }}" - - name: Check status of etcd certificates - stat: - path: "{{ item }}" - with_items: - - /etc/etcd/server.crt - - /etc/etcd/peer.crt - - /etc/etcd/ca.crt - register: g_etcd_server_cert_stat_result - - set_fact: - etcd_server_certs_missing: "{{ g_etcd_server_cert_stat_result.results | map(attribute='stat.exists') - | list | intersect([false])}}" - etcd_cert_subdir: etcd-{{ openshift.common.hostname }} - etcd_cert_config_dir: /etc/etcd - etcd_cert_prefix: - -- name: Create temp directory for syncing certs - hosts: localhost - connection: local - sudo: false - gather_facts: no - tasks: - - name: Create local temp directory for syncing certs - local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX - register: g_etcd_mktemp - changed_when: False - -- name: Configure etcd certificates - hosts: oo_first_etcd - vars: - etcd_generated_certs_dir: /etc/etcd/generated_certs - etcd_needing_server_certs: "{{ hostvars - | oo_select_keys(groups['oo_etcd_to_config']) - | oo_filter_list(filter_attr='etcd_server_certs_missing') }}" - sync_tmpdir: "{{ hostvars.localhost.g_etcd_mktemp.stdout }}" - roles: - - etcd_certificates - post_tasks: - - name: Create a tarball of the etcd certs - command: > - tar -czvf {{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz - -C {{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }} . - args: - creates: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz" - with_items: etcd_needing_server_certs - - name: Retrieve the etcd cert tarballs - fetch: - src: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz" - dest: "{{ sync_tmpdir }}/" - flat: yes - fail_on_missing: yes - validate_checksum: yes - with_items: etcd_needing_server_certs - -- name: Configure etcd hosts - hosts: oo_etcd_to_config - vars: - sync_tmpdir: "{{ hostvars.localhost.g_etcd_mktemp.stdout }}" - etcd_url_scheme: https - etcd_peer_url_scheme: https - etcd_peers_group: oo_etcd_to_config - pre_tasks: - - name: Ensure certificate directory exists - file: - path: "{{ etcd_cert_config_dir }}" - state: directory - - name: Unarchive the tarball on the etcd host - unarchive: - src: "{{ sync_tmpdir }}/{{ etcd_cert_subdir }}.tgz" - dest: "{{ etcd_cert_config_dir }}" - when: etcd_server_certs_missing - roles: - - etcd - -- name: Delete temporary directory on localhost - hosts: localhost - connection: local - sudo: false - gather_facts: no - tasks: - - file: name={{ g_etcd_mktemp.stdout }} state=absent - changed_when: False + - role: openshift_etcd + etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" + - role: nickhammond.logrotate diff --git a/playbooks/common/openshift-etcd/service.yml b/playbooks/common/openshift-etcd/service.yml index 0bf69b22f..f460612ba 100644 --- a/playbooks/common/openshift-etcd/service.yml +++ b/playbooks/common/openshift-etcd/service.yml @@ -1,6 +1,8 @@ --- - name: Populate g_service_masters host group if needed hosts: localhost + connection: local + become: no gather_facts: no tasks: - fail: msg="new_cluster_state is required to be injected in this playbook" @@ -8,7 +10,7 @@ - name: Evaluate g_service_etcd add_host: name={{ item }} groups=g_service_etcd - with_items: oo_host_group_exp | default([]) + with_items: "{{ oo_host_group_exp | default([]) }}" - name: Change etcd state on etcd instance(s) hosts: g_service_etcd diff --git a/playbooks/common/openshift-loadbalancer/config.yml b/playbooks/common/openshift-loadbalancer/config.yml new file mode 100644 index 000000000..c414913bf --- /dev/null +++ b/playbooks/common/openshift-loadbalancer/config.yml @@ -0,0 +1,16 @@ +--- +- name: Configure load balancers + hosts: oo_lb_to_config + vars: + openshift_loadbalancer_frontends: "{{ (openshift_master_api_port | default(8443) + | oo_openshift_loadbalancer_frontends(hostvars | oo_select_keys(groups['oo_masters']), + openshift_use_nuage | default(false), + nuage_mon_rest_server_port | default(none))) + + openshift_loadbalancer_additional_frontends | default([]) }}" + openshift_loadbalancer_backends: "{{ (openshift_master_api_port | default(8443) + | oo_openshift_loadbalancer_backends(hostvars | oo_select_keys(groups['oo_masters']), + openshift_use_nuage | default(false), + nuage_mon_rest_server_port | default(none))) + + openshift_loadbalancer_additional_backends | default([]) }}" + roles: + - role: openshift_loadbalancer diff --git a/playbooks/common/openshift-loadbalancer/filter_plugins b/playbooks/common/openshift-loadbalancer/filter_plugins new file mode 120000 index 000000000..99a95e4ca --- /dev/null +++ b/playbooks/common/openshift-loadbalancer/filter_plugins @@ -0,0 +1 @@ +../../../filter_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-loadbalancer/lookup_plugins b/playbooks/common/openshift-loadbalancer/lookup_plugins new file mode 120000 index 000000000..ac79701db --- /dev/null +++ b/playbooks/common/openshift-loadbalancer/lookup_plugins @@ -0,0 +1 @@ +../../../lookup_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-loadbalancer/roles b/playbooks/common/openshift-loadbalancer/roles new file mode 120000 index 000000000..e2b799b9d --- /dev/null +++ b/playbooks/common/openshift-loadbalancer/roles @@ -0,0 +1 @@ +../../../roles/
\ No newline at end of file diff --git a/playbooks/common/openshift-loadbalancer/service.yml b/playbooks/common/openshift-loadbalancer/service.yml new file mode 100644 index 000000000..efc80edf9 --- /dev/null +++ b/playbooks/common/openshift-loadbalancer/service.yml @@ -0,0 +1,20 @@ +--- +- name: Populate g_service_nodes host group if needed + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - fail: msg="new_cluster_state is required to be injected in this playbook" + when: new_cluster_state is not defined + + - name: Evaluate g_service_lb + add_host: name={{ item }} groups=g_service_lb + with_items: "{{ oo_host_group_exp | default([]) }}" + +- name: Change state on lb instance(s) + hosts: g_service_lb + connection: ssh + gather_facts: no + tasks: + - service: name=haproxy state="{{ new_cluster_state }}" diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml index 3956128e1..5fcb850a2 100644 --- a/playbooks/common/openshift-master/config.yml +++ b/playbooks/common/openshift-master/config.yml @@ -1,7 +1,29 @@ --- -- name: Set master facts and determine if external etcd certs need to be generated +- name: Gather and set facts for master hosts hosts: oo_masters_to_config + vars: + t_oo_option_master_debug_level: "{{ lookup('oo_option', 'openshift_master_debug_level') }}" + pre_tasks: + - name: Check for RPM generated config marker file .config_managed + stat: + path: /etc/origin/.config_managed + register: rpmgenerated_config + + - name: Remove RPM generated config files if present + file: + path: "/etc/origin/{{ item }}" + state: absent + when: rpmgenerated_config.stat.exists == true and deployment_type in ['openshift-enterprise', 'atomic-enterprise'] + with_items: + - master + - node + - .config_managed + + - set_fact: + openshift_master_pod_eviction_timeout: "{{ lookup('oo_option', 'openshift_master_pod_eviction_timeout') | default(none, true) }}" + when: openshift_master_pod_eviction_timeout is not defined + - set_fact: openshift_master_etcd_port: "{{ (etcd_client_port | default('2379')) if (groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config) else none }}" openshift_master_etcd_hosts: "{{ hostvars @@ -9,49 +31,59 @@ | default([])) | oo_collect('openshift.common.hostname') | default(none, true) }}" + + - set_fact: + openshift_master_debug_level: "{{ t_oo_option_master_debug_level }}" + when: openshift_master_debug_level is not defined and t_oo_option_master_debug_level != "" + + - set_fact: + openshift_master_default_subdomain: "{{ lookup('oo_option', 'openshift_master_default_subdomain') | default(None, true) }}" + when: openshift_master_default_subdomain is not defined + - set_fact: + openshift_hosted_metrics_deploy: "{{ lookup('oo_option', 'openshift_hosted_metrics_deploy') | default(false, true) }}" + when: openshift_hosted_metrics_deploy is not defined + - set_fact: + openshift_hosted_metrics_duration: "{{ lookup('oo_option', 'openshift_hosted_metrics_duration') | default(7) }}" + when: openshift_hosted_metrics_duration is not defined + - set_fact: + openshift_hosted_metrics_resolution: "{{ lookup('oo_option', 'openshift_hosted_metrics_resolution') | default('10s', true) }}" + when: openshift_hosted_metrics_resolution is not defined + - set_fact: + openshift_hosted_metrics_deployer_prefix: "{{ lookup('oo_option', 'openshift_hosted_metrics_deployer_prefix') | default('openshift') }}" + when: openshift_hosted_metrics_deployer_prefix is not defined + - set_fact: + openshift_hosted_metrics_deployer_version: "{{ lookup('oo_option', 'openshift_hosted_metrics_deployer_version') | default('latest') }}" + when: openshift_hosted_metrics_deployer_version is not defined roles: - openshift_facts post_tasks: - openshift_facts: - role: "{{ item.role }}" - local_facts: "{{ item.local_facts }}" - with_items: - - role: common - local_facts: - hostname: "{{ openshift_hostname | default(None) }}" - public_hostname: "{{ openshift_public_hostname | default(None) }}" - deployment_type: "{{ openshift_deployment_type }}" - - role: master - local_facts: - api_port: "{{ openshift_master_api_port | default(None) }}" - api_url: "{{ openshift_master_api_url | default(None) }}" - api_use_ssl: "{{ openshift_master_api_use_ssl | default(None) }}" - public_api_url: "{{ openshift_master_public_api_url | default(None) }}" - console_path: "{{ openshift_master_console_path | default(None) }}" - console_port: "{{ openshift_master_console_port | default(None) }}" - console_url: "{{ openshift_master_console_url | default(None) }}" - console_use_ssl: "{{ openshift_master_console_use_ssl | default(None) }}" - public_console_url: "{{ openshift_master_public_console_url | default(None) }}" - - name: Check status of external etcd certificatees - stat: - path: "/etc/openshift/master/{{ item }}" - with_items: - - master.etcd-client.crt - - master.etcd-ca.crt - register: g_external_etcd_cert_stat_result - - set_fact: - etcd_client_certs_missing: "{{ g_external_etcd_cert_stat_result.results - | map(attribute='stat.exists') - | list | intersect([false])}}" - etcd_cert_subdir: openshift-master-{{ openshift.common.hostname }} - etcd_cert_config_dir: /etc/openshift/master - etcd_cert_prefix: master.etcd- - when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config + role: master + local_facts: + api_port: "{{ openshift_master_api_port | default(None) }}" + api_url: "{{ openshift_master_api_url | default(None) }}" + api_use_ssl: "{{ openshift_master_api_use_ssl | default(None) }}" + controllers_port: "{{ openshift_master_controllers_port | default(None) }}" + public_api_url: "{{ openshift_master_public_api_url | default(None) }}" + cluster_hostname: "{{ openshift_master_cluster_hostname | default(None) }}" + cluster_public_hostname: "{{ openshift_master_cluster_public_hostname | default(None) }}" + console_path: "{{ openshift_master_console_path | default(None) }}" + console_port: "{{ openshift_master_console_port | default(None) }}" + console_url: "{{ openshift_master_console_url | default(None) }}" + console_use_ssl: "{{ openshift_master_console_use_ssl | default(None) }}" + public_console_url: "{{ openshift_master_public_console_url | default(None) }}" + ha: "{{ openshift_master_ha | default(groups.oo_masters | length > 1) }}" + master_count: "{{ openshift_master_count | default(groups.oo_masters | length) }}" + - openshift_facts: + role: hosted + openshift_env: + openshift_hosted_registry_storage_kind: 'nfs' + when: openshift_hosted_registry_storage_kind is not defined and groups.oo_nfs_to_config is defined and groups.oo_nfs_to_config | length > 0 - name: Create temp directory for syncing certs hosts: localhost connection: local - sudo: false + become: no gather_facts: no tasks: - name: Create local temp directory for syncing certs @@ -59,155 +91,105 @@ register: g_master_mktemp changed_when: False -- name: Configure etcd certificates - hosts: oo_first_etcd - vars: - etcd_generated_certs_dir: /etc/etcd/generated_certs - etcd_needing_client_certs: "{{ hostvars - | oo_select_keys(groups['oo_masters_to_config']) - | oo_filter_list(filter_attr='etcd_client_certs_missing') }}" - sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" +- name: Determine if session secrets must be generated + hosts: oo_first_master roles: - - etcd_certificates + - role: openshift_facts post_tasks: - - name: Create a tarball of the etcd certs - command: > - tar -czvf {{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz - -C {{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }} . - args: - creates: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz" - with_items: etcd_needing_client_certs - - name: Retrieve the etcd cert tarballs - fetch: - src: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz" - dest: "{{ sync_tmpdir }}/" - flat: yes - fail_on_missing: yes - validate_checksum: yes - with_items: etcd_needing_client_certs - -- name: Copy the external etcd certs to the masters - hosts: oo_masters_to_config - vars: - sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" - tasks: - - name: Ensure certificate directory exists - file: - path: /etc/openshift/master - state: directory - when: etcd_client_certs_missing is defined and etcd_client_certs_missing - - name: Unarchive the tarball on the master - unarchive: - src: "{{ sync_tmpdir }}/{{ etcd_cert_subdir }}.tgz" - dest: "{{ etcd_cert_config_dir }}" - when: etcd_client_certs_missing is defined and etcd_client_certs_missing - - file: - path: "{{ etcd_cert_config_dir }}/{{ item }}" - owner: root - group: root - mode: 0600 - with_items: - - master.etcd-client.crt - - master.etcd-client.key - - master.etcd-ca.crt - when: etcd_client_certs_missing is defined and etcd_client_certs_missing - -- name: Determine if master certificates need to be generated - hosts: oo_masters_to_config - tasks: - - set_fact: - openshift_master_certs_no_etcd: - - admin.crt - - master.kubelet-client.crt - - master.server.crt - - openshift-master.crt - - openshift-registry.crt - - openshift-router.crt - - etcd.server.crt - openshift_master_certs_etcd: - - master.etcd-client.crt - - set_fact: - openshift_master_certs: "{{ (openshift_master_certs_no_etcd | union(openshift_master_certs_etcd)) if (groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config) else openshift_master_certs_no_etcd }}" - - - name: Check status of master certificates - stat: - path: "/etc/openshift/master/{{ item }}" - with_items: openshift_master_certs - register: g_master_cert_stat_result - - set_fact: - master_certs_missing: "{{ g_master_cert_stat_result.results - | map(attribute='stat.exists') - | list | intersect([false])}}" - master_cert_subdir: master-{{ openshift.common.hostname }} - master_cert_config_dir: /etc/openshift/master + - openshift_facts: + role: master + local_facts: + session_auth_secrets: "{{ openshift_master_session_auth_secrets | default(openshift.master.session_auth_secrets | default(None)) }}" + session_encryption_secrets: "{{ openshift_master_session_encryption_secrets | default(openshift.master.session_encryption_secrets | default(None)) }}" -- name: Configure master certificates +- name: Generate master session secrets hosts: oo_first_master vars: - master_generated_certs_dir: /etc/openshift/generated-configs - masters_needing_certs: "{{ hostvars - | oo_select_keys(groups['oo_masters_to_config'] | difference(groups['oo_first_master'])) - | oo_filter_list(filter_attr='master_certs_missing') }}" - sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" + g_session_secrets_present: "{{ (openshift.master.session_auth_secrets | default([])) | length > 0 and (openshift.master.session_encryption_secrets | default([])) | length > 0 }}" + g_session_auth_secrets: "{{ [ 24 | oo_generate_secret ] }}" + g_session_encryption_secrets: "{{ [ 24 | oo_generate_secret ] }}" roles: - - openshift_master_certificates - post_tasks: - - name: Create a tarball of the master certs - command: > - tar -czvf {{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz - -C {{ master_generated_certs_dir }}/{{ item.master.cert_subdir }} . - args: - creates: "{{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz" - with_items: masters_needing_certs - - name: Retrieve the master cert tarball from the master - fetch: - src: "{{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz" - dest: "{{ sync_tmpdir }}/" - flat: yes - fail_on_missing: yes - validate_checksum: yes - with_items: masters_needing_certs + - role: openshift_facts + tasks: + - openshift_facts: + role: master + local_facts: + session_auth_secrets: "{{ g_session_auth_secrets }}" + session_encryption_secrets: "{{ g_session_encryption_secrets }}" + when: not g_session_secrets_present | bool -- name: Configure master instances +- name: Configure masters hosts: oo_masters_to_config + any_errors_fatal: true vars: sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" - pre_tasks: - - name: Ensure certificate directory exists - file: - path: /etc/openshift/master - state: directory - when: master_certs_missing and 'oo_first_master' not in group_names - - name: Unarchive the tarball on the master - unarchive: - src: "{{ sync_tmpdir }}/{{ master_cert_subdir }}.tgz" - dest: "{{ master_cert_config_dir }}" - when: master_certs_missing and 'oo_first_master' not in group_names + openshift_master_ha: "{{ openshift.master.ha }}" + openshift_master_count: "{{ openshift.master.master_count }}" + openshift_master_session_auth_secrets: "{{ hostvars[groups.oo_first_master.0].openshift.master.session_auth_secrets }}" + openshift_master_session_encryption_secrets: "{{ hostvars[groups.oo_first_master.0].openshift.master.session_encryption_secrets }}" + openshift_no_proxy_internal_hostnames: "{{ hostvars | oo_select_keys(groups['oo_nodes_to_config'] + | union(groups['oo_masters_to_config']) + | union(groups['oo_etcd_to_config'] | default([]))) + | oo_collect('openshift.common.hostname') | default([]) | join (',') + }}" roles: - - openshift_master - - role: fluentd_master - when: openshift.common.use_fluentd | bool + - role: openshift_master_facts + - role: openshift_hosted_facts + - role: openshift_master_certificates + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + openshift_master_etcd_hosts: "{{ hostvars + | oo_select_keys(groups['oo_etcd_to_config'] | default([])) + | oo_collect('openshift.common.hostname') + | default(none, true) }}" + openshift_master_hostnames: "{{ hostvars + | oo_select_keys(groups['oo_masters_to_config'] | default([])) + | oo_collect('openshift.common.all_hostnames') + | oo_flatten | unique }}" + - role: openshift_etcd_client_certificates + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}" + etcd_cert_config_dir: "{{ openshift.common.config_base }}/master" + etcd_cert_prefix: "master.etcd-" + when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config + - role: openshift_clock + - role: openshift_cloud_provider + - role: openshift_builddefaults + - role: os_firewall + os_firewall_allow: + - service: etcd embedded + port: 4001/tcp + - service: api server https + port: "{{ openshift.master.api_port }}/tcp" + - service: api controllers https + port: "{{ openshift.master.controllers_port }}/tcp" + - service: skydns tcp + port: "{{ openshift.master.dns_port }}/tcp" + - service: skydns udp + port: "{{ openshift.master.dns_port }}/udp" + - service: Fluentd td-agent tcp + port: 24224/tcp + - service: Fluentd td-agent udp + port: 24224/udp + - service: pcsd + port: 2224/tcp + - service: Corosync UDP + port: 5404/udp + - service: Corosync UDP + port: 5405/udp + - role: openshift_master + openshift_master_hosts: "{{ groups.oo_masters_to_config }}" + - role: nickhammond.logrotate + - role: nuage_master + when: openshift.common.use_nuage | bool post_tasks: - name: Create group for deployment type group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }} changed_when: False -- name: Deploy OpenShift examples - hosts: oo_first_master - roles: - - openshift_examples - -# Additional instance config for online deployments -- name: Additional instance config - hosts: oo_masters_deployment_type_online - roles: - - pods - - os_env_extras - - name: Delete temporary directory on localhost hosts: localhost connection: local - sudo: false + become: no gather_facts: no tasks: - file: name={{ g_master_mktemp.stdout }} state=absent diff --git a/playbooks/common/openshift-master/library b/playbooks/common/openshift-master/library new file mode 120000 index 000000000..d0b7393d3 --- /dev/null +++ b/playbooks/common/openshift-master/library @@ -0,0 +1 @@ +../../../library/
\ No newline at end of file diff --git a/playbooks/common/openshift-master/restart.yml b/playbooks/common/openshift-master/restart.yml new file mode 100644 index 000000000..5769ef5cd --- /dev/null +++ b/playbooks/common/openshift-master/restart.yml @@ -0,0 +1,79 @@ +--- +- include: ../openshift-cluster/evaluate_groups.yml + +- name: Validate configuration for rolling restart + hosts: oo_masters_to_config + roles: + - openshift_facts + tasks: + - fail: + msg: "openshift_rolling_restart_mode must be set to either 'services' or 'system'" + when: openshift_rolling_restart_mode is defined and openshift_rolling_restart_mode not in ["services", "system"] + - openshift_facts: + role: "{{ item.role }}" + local_facts: "{{ item.local_facts }}" + with_items: + - role: common + local_facts: + rolling_restart_mode: "{{ openshift_rolling_restart_mode | default('services') }}" + - role: master + local_facts: + cluster_method: "{{ openshift_master_cluster_method | default(None) }}" + +# Creating a temp file on localhost, we then check each system that will +# be rebooted to see if that file exists, if so we know we're running +# ansible on a machine that needs a reboot, and we need to error out. +- name: Create temp file on localhost + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - local_action: command mktemp + register: mktemp + changed_when: false + +- name: Check if temp file exists on any masters + hosts: oo_masters_to_config + tasks: + - stat: path="{{ hostvars.localhost.mktemp.stdout }}" + register: exists + changed_when: false + +- name: Cleanup temp file on localhost + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - file: path="{{ hostvars.localhost.mktemp.stdout }}" state=absent + changed_when: false + +- name: Warn if restarting the system where ansible is running + hosts: oo_masters_to_config + tasks: + - pause: + prompt: > + Warning: Running playbook from a host that will be restarted! + Press CTRL+C and A to abort playbook execution. You may + continue by pressing ENTER but the playbook will stop + executing after this system has been restarted and services + must be verified manually. To only restart services, set + openshift_master_rolling_restart_mode=services in host + inventory and relaunch the playbook. + when: exists.stat.exists and openshift.common.rolling_restart_mode == 'system' + - set_fact: + current_host: "{{ exists.stat.exists }}" + when: openshift.common.rolling_restart_mode == 'system' + +- name: Restart masters + hosts: oo_masters_to_config + vars: + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + serial: 1 + tasks: + - include: restart_hosts.yml + when: openshift.common.rolling_restart_mode == 'system' + - include: restart_services.yml + when: openshift.common.rolling_restart_mode == 'services' + diff --git a/playbooks/common/openshift-master/restart_hosts.yml b/playbooks/common/openshift-master/restart_hosts.yml new file mode 100644 index 000000000..b1c36718c --- /dev/null +++ b/playbooks/common/openshift-master/restart_hosts.yml @@ -0,0 +1,17 @@ +- name: Restart master system + # https://github.com/ansible/ansible/issues/10616 + shell: sleep 2 && shutdown -r now "OpenShift Ansible master rolling restart" + async: 1 + poll: 0 + ignore_errors: true + become: yes + +# Ensure the api_port is available. +- name: Wait for master API to come back online + become: no + local_action: + module: wait_for + host="{{ inventory_hostname }}" + state=started + delay=10 + port="{{ openshift.master.api_port }}" diff --git a/playbooks/common/openshift-master/restart_services.yml b/playbooks/common/openshift-master/restart_services.yml new file mode 100644 index 000000000..5e539cd65 --- /dev/null +++ b/playbooks/common/openshift-master/restart_services.yml @@ -0,0 +1,27 @@ +- name: Restart master + service: + name: "{{ openshift.common.service_type }}-master" + state: restarted + when: not openshift_master_ha | bool +- name: Restart master API + service: + name: "{{ openshift.common.service_type }}-master-api" + state: restarted + when: openshift_master_ha | bool and openshift.master.cluster_method != 'pacemaker' +- name: Wait for master API to come back online + become: no + local_action: + module: wait_for + host="{{ inventory_hostname }}" + state=started + delay=10 + port="{{ openshift.master.api_port }}" + when: openshift_master_ha | bool and openshift.master.cluster_method != 'pacemaker' +- name: Restart master controllers + service: + name: "{{ openshift.common.service_type }}-master-controllers" + state: restarted + # Ignore errrors since it is possible that type != simple for + # pre-3.1.1 installations. + ignore_errors: true + when: openshift_master_ha | bool and openshift.master.cluster_method != 'pacemaker' diff --git a/playbooks/common/openshift-master/scaleup.yml b/playbooks/common/openshift-master/scaleup.yml new file mode 100644 index 000000000..18e5c665f --- /dev/null +++ b/playbooks/common/openshift-master/scaleup.yml @@ -0,0 +1,67 @@ +--- +- include: ../openshift-cluster/evaluate_groups.yml + +- name: Gather facts + hosts: oo_etcd_to_config:oo_masters_to_config:oo_nodes_to_config + roles: + - openshift_facts + +- name: Update master count + hosts: oo_masters:!oo_masters_to_config + serial: 1 + roles: + - openshift_facts + post_tasks: + - openshift_facts: + role: master + local_facts: + ha: "{{ openshift_master_ha | default(groups.oo_masters | length > 1) }}" + master_count: "{{ openshift_master_count | default(groups.oo_masters | length) }}" + - name: Update master count + modify_yaml: + dest: "{{ openshift.common.config_base}}/master/master-config.yaml" + yaml_key: 'kubernetesMasterConfig.masterCount' + yaml_value: "{{ openshift.master.master_count }}" + notify: + - restart master api + - restart master controllers + handlers: + - name: restart master api + service: name={{ openshift.common.service_type }}-master-controllers state=restarted + notify: verify api server + - name: restart master controllers + service: name={{ openshift.common.service_type }}-master-controllers state=restarted + - name: verify api server + command: > + curl --silent --tlsv1.2 + {% if openshift.common.version_gte_3_2_or_1_2 | bool %} + --cacert {{ openshift.common.config_base }}/master/ca-bundle.crt + {% else %} + --cacert {{ openshift.common.config_base }}/master/ca.crt + {% endif %} + {{ openshift.master.api_url }}/healthz/ready + args: + # Disables the following warning: + # Consider using get_url or uri module rather than running curl + warn: no + register: api_available_output + until: api_available_output.stdout == 'ok' + retries: 120 + delay: 1 + changed_when: false + +- name: Configure docker hosts + hosts: oo_masters_to-config:oo_nodes_to_config + vars: + docker_additional_registries: "{{ lookup('oo_option', 'docker_additional_registries') | oo_split }}" + docker_insecure_registries: "{{ lookup('oo_option', 'docker_insecure_registries') | oo_split }}" + docker_blocked_registries: "{{ lookup('oo_option', 'docker_blocked_registries') | oo_split }}" + roles: + - openshift_facts + - openshift_docker + +- include: ../openshift-master/config.yml + +- include: ../openshift-loadbalancer/config.yml + +- include: ../openshift-node/config.yml diff --git a/playbooks/common/openshift-master/service.yml b/playbooks/common/openshift-master/service.yml index 5636ad156..5e5198335 100644 --- a/playbooks/common/openshift-master/service.yml +++ b/playbooks/common/openshift-master/service.yml @@ -2,17 +2,19 @@ - name: Populate g_service_masters host group if needed hosts: localhost gather_facts: no + connection: local + become: no tasks: - fail: msg="new_cluster_state is required to be injected in this playbook" when: new_cluster_state is not defined - name: Evaluate g_service_masters add_host: name={{ item }} groups=g_service_masters - with_items: oo_host_group_exp | default([]) + with_items: "{{ oo_host_group_exp | default([]) }}" -- name: Change openshift-master state on master instance(s) +- name: Change state on master instance(s) hosts: g_service_masters connection: ssh gather_facts: no tasks: - - service: name=openshift-master state="{{ new_cluster_state }}" + - service: name={{ openshift.common.service_type }}-master state="{{ new_cluster_state }}" diff --git a/playbooks/common/openshift-nfs/config.yml b/playbooks/common/openshift-nfs/config.yml new file mode 100644 index 000000000..000e46e80 --- /dev/null +++ b/playbooks/common/openshift-nfs/config.yml @@ -0,0 +1,6 @@ +--- +- name: Configure nfs + hosts: oo_nfs_to_config + roles: + - role: openshift_facts + - role: openshift_storage_nfs diff --git a/playbooks/common/openshift-nfs/filter_plugins b/playbooks/common/openshift-nfs/filter_plugins new file mode 120000 index 000000000..99a95e4ca --- /dev/null +++ b/playbooks/common/openshift-nfs/filter_plugins @@ -0,0 +1 @@ +../../../filter_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-nfs/lookup_plugins b/playbooks/common/openshift-nfs/lookup_plugins new file mode 120000 index 000000000..ac79701db --- /dev/null +++ b/playbooks/common/openshift-nfs/lookup_plugins @@ -0,0 +1 @@ +../../../lookup_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-nfs/roles b/playbooks/common/openshift-nfs/roles new file mode 120000 index 000000000..e2b799b9d --- /dev/null +++ b/playbooks/common/openshift-nfs/roles @@ -0,0 +1 @@ +../../../roles/
\ No newline at end of file diff --git a/playbooks/common/openshift-nfs/service.yml b/playbooks/common/openshift-nfs/service.yml new file mode 100644 index 000000000..8468014da --- /dev/null +++ b/playbooks/common/openshift-nfs/service.yml @@ -0,0 +1,18 @@ +--- +- name: Populate g_service_nfs host group if needed + hosts: localhost + gather_facts: no + tasks: + - fail: msg="new_cluster_state is required to be injected in this playbook" + when: new_cluster_state is not defined + + - name: Evaluate g_service_nfs + add_host: name={{ item }} groups=g_service_nfs + with_items: "{{ oo_host_group_exp | default([]) }}" + +- name: Change state on nfs instance(s) + hosts: g_service_nfs + connection: ssh + gather_facts: no + tasks: + - service: name=nfs-server state="{{ new_cluster_state }}" diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml index bd35008b8..4824eeef3 100644 --- a/playbooks/common/openshift-node/config.yml +++ b/playbooks/common/openshift-node/config.yml @@ -1,6 +1,12 @@ --- - name: Gather and set facts for node hosts hosts: oo_nodes_to_config + vars: + t_oo_option_node_debug_level: "{{ lookup('oo_option', 'openshift_node_debug_level') }}" + pre_tasks: + - set_fact: + openshift_node_debug_level: "{{ t_oo_option_node_debug_level }}" + when: openshift_node_debug_level is not defined and t_oo_option_node_debug_level != "" roles: - openshift_facts tasks: @@ -8,40 +14,16 @@ # configured, we need to make sure to set the node properties beforehand if # we do not want the defaults - openshift_facts: - role: "{{ item.role }}" - local_facts: "{{ item.local_facts }}" - with_items: - - role: common - local_facts: - hostname: "{{ openshift_hostname | default(None) }}" - public_hostname: "{{ openshift_public_hostname | default(None) }}" - deployment_type: "{{ openshift_deployment_type }}" - - role: node - local_facts: - labels: "{{ openshift_node_labels | default(None) }}" - annotations: "{{ openshift_node_annotations | default(None) }}" - - name: Check status of node certificates - stat: - path: "/etc/openshift/node/{{ item }}" - with_items: - - "system:node:{{ openshift.common.hostname }}.crt" - - "system:node:{{ openshift.common.hostname }}.key" - - "system:node:{{ openshift.common.hostname }}.kubeconfig" - - ca.crt - - server.key - - server.crt - register: stat_result - - set_fact: - certs_missing: "{{ stat_result.results | map(attribute='stat.exists') - | list | intersect([false])}}" - node_subdir: node-{{ openshift.common.hostname }} - config_dir: /etc/openshift/generated-configs/node-{{ openshift.common.hostname }} - node_cert_dir: /etc/openshift/node + role: node + local_facts: + labels: "{{ openshift_node_labels | default(None) }}" + annotations: "{{ openshift_node_annotations | default(None) }}" + schedulable: "{{ openshift_schedulable | default(openshift_scheduleable) | default(None) }}" - name: Create temp directory for syncing certs hosts: localhost connection: local - sudo: false + become: no gather_facts: no tasks: - name: Create local temp directory for syncing certs @@ -49,58 +31,114 @@ register: mktemp changed_when: False -- name: Create node certificates - hosts: oo_first_master +- name: Evaluate node groups + hosts: localhost + become: no + connection: local + tasks: + - name: Evaluate oo_containerized_master_nodes + add_host: + name: "{{ item }}" + groups: oo_containerized_master_nodes + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_become: "{{ g_sudo | default(omit) }}" + with_items: "{{ groups.oo_nodes_to_config | default([]) }}" + when: hostvars[item].openshift.common is defined and hostvars[item].openshift.common.is_containerized | bool and (item in groups.oo_nodes_to_config and item in groups.oo_masters_to_config) + +- name: Configure containerized nodes + hosts: oo_containerized_master_nodes + serial: 1 vars: - nodes_needing_certs: "{{ hostvars - | oo_select_keys(groups['oo_nodes_to_config'] - | default([])) - | oo_filter_list(filter_attr='certs_missing') }}" - sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}" + openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}" + openshift_node_first_master_ip: "{{ hostvars[groups.oo_first_master.0].openshift.common.ip }}" + openshift_docker_hosted_registry_network: "{{ hostvars[groups.oo_first_master.0].openshift.common.portal_net }}" + openshift_no_proxy_internal_hostnames: "{{ hostvars | oo_select_keys(groups['oo_nodes_to_config'] + | union(groups['oo_masters_to_config']) + | union(groups['oo_etcd_to_config'] | default([]))) + | oo_collect('openshift.common.hostname') | default([]) | join (',') + }}" + when: "{{ (openshift_http_proxy is defined or openshift_https_proxy is defined) and + openshift_generate_no_proxy_hosts | default(True) | bool }}" roles: - - openshift_node_certificates - post_tasks: - - name: Create a tarball of the node config directories - command: > - tar -czvf {{ item.config_dir }}.tgz - --transform 's|system:{{ item.node_subdir }}|node|' - -C {{ item.config_dir }} . - args: - creates: "{{ item.config_dir }}.tgz" - with_items: nodes_needing_certs + - role: openshift_common + - role: openshift_clock + - role: openshift_docker + - role: openshift_node_certificates + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + - role: openshift_cloud_provider + - role: openshift_node_dnsmasq + when: openshift.common.use_dnsmasq | bool + - role: os_firewall + os_firewall_allow: + - service: Kubernetes kubelet + port: 10250/tcp + - service: http + port: 80/tcp + - service: https + port: 443/tcp + - service: Openshift kubelet ReadOnlyPort + port: 10255/tcp + - service: Openshift kubelet ReadOnlyPort udp + port: 10255/udp + - service: OpenShift OVS sdn + port: 4789/udp + when: openshift.node.use_openshift_sdn | bool + - role: openshift_node - - name: Retrieve the node config tarballs from the master - fetch: - src: "{{ item.config_dir }}.tgz" - dest: "{{ sync_tmpdir }}/" - flat: yes - fail_on_missing: yes - validate_checksum: yes - with_items: nodes_needing_certs +- name: Configure nodes + hosts: oo_nodes_to_config:!oo_containerized_master_nodes + vars: + openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}" + openshift_node_first_master_ip: "{{ hostvars[groups.oo_first_master.0].openshift.common.ip }}" + openshift_docker_hosted_registry_network: "{{ hostvars[groups.oo_first_master.0].openshift.common.portal_net }}" + openshift_no_proxy_internal_hostnames: "{{ hostvars | oo_select_keys(groups['oo_nodes_to_config'] + | union(groups['oo_masters_to_config']) + | union(groups['oo_etcd_to_config'] | default([]))) + | oo_collect('openshift.common.hostname') | default([]) | join (',') + }}" + when: "{{ (openshift_http_proxy is defined or openshift_https_proxy is defined) and + openshift_generate_no_proxy_hosts | default(True) | bool }}" + roles: + - role: openshift_common + - role: openshift_clock + - role: openshift_docker + - role: openshift_node_certificates + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + - role: openshift_cloud_provider + - role: openshift_node_dnsmasq + when: openshift.common.use_dnsmasq | bool + - role: os_firewall + os_firewall_allow: + - service: Kubernetes kubelet + port: 10250/tcp + - service: http + port: 80/tcp + - service: https + port: 443/tcp + - service: Openshift kubelet ReadOnlyPort + port: 10255/tcp + - service: Openshift kubelet ReadOnlyPort udp + port: 10255/udp + - service: OpenShift OVS sdn + port: 4789/udp + when: openshift.node.use_openshift_sdn | bool + - role: openshift_node -- name: Configure node instances +- name: Additional node config hosts: oo_nodes_to_config vars: - sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}" openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}" - pre_tasks: - - name: Ensure certificate directory exists - file: - path: "{{ node_cert_dir }}" - state: directory - - # TODO: notify restart openshift-node - # possibly test service started time against certificate/config file - # timestamps in openshift-node to trigger notify - - name: Unarchive the tarball on the node - unarchive: - src: "{{ sync_tmpdir }}/{{ node_subdir }}.tgz" - dest: "{{ node_cert_dir }}" - when: certs_missing roles: - - openshift_node - - role: fluentd_node - when: openshift.common.use_fluentd | bool + - role: flannel + etcd_urls: "{{ hostvars[groups.oo_first_master.0].openshift.master.etcd_urls }}" + embedded_etcd: "{{ hostvars[groups.oo_first_master.0].openshift.master.embedded_etcd }}" + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_cert_subdir: "openshift-node-{{ openshift.common.hostname }}" + etcd_cert_config_dir: "{{ openshift.common.config_base }}/node" + when: openshift.common.use_flannel | bool + - role: nuage_node + when: openshift.common.use_nuage | bool + - role: nickhammond.logrotate tasks: - name: Create group for deployment type group_by: key=oo_nodes_deployment_type_{{ openshift.common.deployment_type }} @@ -109,36 +147,40 @@ - name: Delete temporary directory on localhost hosts: localhost connection: local - sudo: false + become: no gather_facts: no tasks: - file: name={{ mktemp.stdout }} state=absent changed_when: False -# Additional config for online type deployments -- name: Additional instance config - hosts: oo_nodes_deployment_type_online - gather_facts: no - roles: - - os_env_extras - - os_env_extras_node - -- name: Set scheduleability +- name: Set node schedulability hosts: oo_first_master vars: - openshift_nodes: "{{ hostvars - | oo_select_keys(groups['oo_nodes_to_config']) - | oo_collect('openshift.common.hostname') }}" - openshift_unscheduleable_nodes: "{{ hostvars - | oo_select_keys(groups['oo_nodes_to_config'] - | default([])) - | oo_collect('openshift.common.hostname', {'openshift_scheduleable': False}) }}" + openshift_nodes: "{{ groups.oo_nodes_to_config | default([]) }}" pre_tasks: - - set_fact: - openshift_scheduleable_nodes: "{{ hostvars - | oo_select_keys(groups['oo_nodes_to_config'] - | default([])) - | oo_collect('openshift.common.hostname') - | difference(openshift_unscheduleable_nodes) }}" + # Necessary because when you're on a node that's also a master the master will be + # restarted after the node restarts docker and it will take up to 60 seconds for + # systemd to start the master again + - name: Wait for master API to become available before proceeding + # Using curl here since the uri module requires python-httplib2 and + # wait_for port doesn't provide health information. + command: > + curl --silent --tlsv1.2 + {% if openshift.common.version_gte_3_2_or_1_2 | bool %} + --cacert {{ openshift.common.config_base }}/master/ca-bundle.crt + {% else %} + --cacert {{ openshift.common.config_base }}/master/ca.crt + {% endif %} + {{ openshift.master.api_url }}/healthz/ready + args: + # Disables the following warning: + # Consider using get_url or uri module rather than running curl + warn: no + register: api_available_output + until: api_available_output.stdout == 'ok' + retries: 120 + delay: 1 + changed_when: false + when: openshift.common.is_containerized | bool roles: - openshift_manage_node diff --git a/playbooks/common/openshift-node/scaleup.yml b/playbooks/common/openshift-node/scaleup.yml new file mode 100644 index 000000000..bb3b1e780 --- /dev/null +++ b/playbooks/common/openshift-node/scaleup.yml @@ -0,0 +1,30 @@ +--- +- include: ../openshift-cluster/evaluate_groups.yml + +- name: Gather facts + hosts: oo_etcd_to_config:oo_masters_to_config:oo_nodes_to_config + roles: + - openshift_facts + +- name: Gather and set facts for first master + hosts: oo_first_master + vars: + openshift_master_count: "{{ groups.oo_masters | length }}" + pre_tasks: + - set_fact: + openshift_master_default_subdomain: "{{ lookup('oo_option', 'openshift_master_default_subdomain') | default(None, true) }}" + when: openshift_master_default_subdomain is not defined + roles: + - openshift_master_facts + +- name: Configure docker hosts + hosts: oo_nodes_to_config + vars: + docker_additional_registries: "{{ lookup('oo_option', 'docker_additional_registries') | oo_split }}" + docker_insecure_registries: "{{ lookup('oo_option', 'docker_insecure_registries') | oo_split }}" + docker_blocked_registries: "{{ lookup('oo_option', 'docker_blocked_registries') | oo_split }}" + roles: + - openshift_facts + - openshift_docker + +- include: ../openshift-node/config.yml diff --git a/playbooks/common/openshift-node/service.yml b/playbooks/common/openshift-node/service.yml index f76df089f..33095c9fb 100644 --- a/playbooks/common/openshift-node/service.yml +++ b/playbooks/common/openshift-node/service.yml @@ -1,6 +1,8 @@ --- - name: Populate g_service_nodes host group if needed hosts: localhost + connection: local + become: no gather_facts: no tasks: - fail: msg="new_cluster_state is required to be injected in this playbook" @@ -8,11 +10,11 @@ - name: Evaluate g_service_nodes add_host: name={{ item }} groups=g_service_nodes - with_items: oo_host_group_exp | default([]) + with_items: "{{ oo_host_group_exp | default([]) }}" -- name: Change openshift-node state on node instance(s) +- name: Change state on node instance(s) hosts: g_service_nodes connection: ssh gather_facts: no tasks: - - service: name=openshift-node state="{{ new_cluster_state }}" + - service: name={{ service_type }}-node state="{{ new_cluster_state }}" |