diff options
Diffstat (limited to 'roles/openshift_master')
26 files changed, 516 insertions, 354 deletions
diff --git a/roles/openshift_master/README.md b/roles/openshift_master/README.md index fbf69c270..2dcc56e3f 100644 --- a/roles/openshift_master/README.md +++ b/roles/openshift_master/README.md @@ -1,4 +1,4 @@ -OpenShift/Atomic Enterprise Master +OpenShift Master ================================== Master service installation @@ -17,7 +17,6 @@ From this role: | Name | Default value | | |---------------------------------------------------|-----------------------|-------------------------------------------------------------------------------| -| openshift_master_debug_level | openshift_debug_level | Verbosity of the debug logs for master | | openshift_node_ips | [] | List of the openshift node ip addresses to pre-register when master starts up | | oreg_url | UNDEF | Default docker registry to use | | oreg_url_master | UNDEF | Default docker registry to use, specifically on the master | @@ -29,18 +28,10 @@ From this role: | openshift_master_public_console_url | UNDEF | | | openshift_master_saconfig_limit_secret_references | false | | -From openshift_common: - -| Name | Default Value | | -|-------------------------------|----------------|----------------------------------------| -| openshift_debug_level | 2 | Global openshift debug log verbosity | -| openshift_public_ip | UNDEF | Public IP address to use for this host | -| openshift_hostname | UNDEF | hostname to use for this instance | Dependencies ------------ -openshift_common Example Playbook ---------------- diff --git a/roles/openshift_master/defaults/main.yml b/roles/openshift_master/defaults/main.yml index 2d3ce5bcd..73e935d3f 100644 --- a/roles/openshift_master/defaults/main.yml +++ b/roles/openshift_master/defaults/main.yml @@ -1,4 +1,45 @@ --- +r_openshift_master_firewall_enabled: "{{ os_firewall_enabled | default(True) }}" +r_openshift_master_use_firewalld: "{{ os_firewall_use_firewalld | default(False) }}" + openshift_node_ips: [] r_openshift_master_clean_install: false r_openshift_master_etcd3_storage: false +r_openshift_master_os_firewall_enable: true +r_openshift_master_os_firewall_deny: [] +r_openshift_master_os_firewall_allow: +- service: api server https + port: "{{ openshift.master.api_port }}/tcp" +- service: api controllers https + port: "{{ openshift.master.controllers_port }}/tcp" +- service: skydns tcp + port: "{{ openshift.master.dns_port }}/tcp" +- service: skydns udp + port: "{{ openshift.master.dns_port }}/udp" +- service: etcd embedded + port: 4001/tcp + cond: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" + +# oreg_url is defined by user input +oreg_host: "{{ oreg_url.split('/')[0] if (oreg_url is defined and '.' in oreg_url.split('/')[0]) else '' }}" +oreg_auth_credentials_path: "{{ r_openshift_master_data_dir }}/.docker" +oreg_auth_credentials_replace: False +l_bind_docker_reg_auth: False + +# NOTE +# r_openshift_master_*_default may be defined external to this role. +# openshift_use_*, if defined, may affect other roles or play behavior. +r_openshift_master_use_openshift_sdn_default: "{{ openshift_use_openshift_sdn | default(True) }}" +r_openshift_master_use_openshift_sdn: "{{ r_openshift_master_use_openshift_sdn_default }}" + +r_openshift_master_use_nuage_default: "{{ openshift_use_nuage | default(False) }}" +r_openshift_master_use_nuage: "{{ r_openshift_master_use_nuage_default }}" + +r_openshift_master_use_contiv_default: "{{ openshift_use_contiv | default(False) }}" +r_openshift_master_use_contiv: "{{ r_openshift_master_use_contiv_default }}" + +r_openshift_master_data_dir_default: "{{ openshift_data_dir | default('/var/lib/origin') }}" +r_openshift_master_data_dir: "{{ r_openshift_master_data_dir_default }}" + +r_openshift_master_sdn_network_plugin_name_default: "{{ os_sdn_network_plugin_name | default('redhat/openshift-ovs-subnet') }}" +r_openshift_master_sdn_network_plugin_name: "{{ r_openshift_master_sdn_network_plugin_name_default }}" diff --git a/roles/openshift_master/files/atomic-openshift-master.service b/roles/openshift_master/files/atomic-openshift-master.service deleted file mode 100644 index 02af4dd16..000000000 --- a/roles/openshift_master/files/atomic-openshift-master.service +++ /dev/null @@ -1,23 +0,0 @@ -[Unit] -Description=Atomic OpenShift Master -Documentation=https://github.com/openshift/origin -After=network-online.target -After=etcd.service -Before=atomic-openshift-node.service -Requires=network-online.target - -[Service] -Type=notify -EnvironmentFile=/etc/sysconfig/atomic-openshift-master -Environment=GOTRACEBACK=crash -ExecStart=/usr/bin/openshift start master --config=${CONFIG_FILE} $OPTIONS -LimitNOFILE=131072 -LimitCORE=infinity -WorkingDirectory=/var/lib/origin/ -SyslogIdentifier=atomic-openshift-master -Restart=always -RestartSec=5s - -[Install] -WantedBy=multi-user.target -WantedBy=atomic-openshift-node.service diff --git a/roles/openshift_master/files/origin-master.service b/roles/openshift_master/files/origin-master.service deleted file mode 100644 index cf79dda02..000000000 --- a/roles/openshift_master/files/origin-master.service +++ /dev/null @@ -1,23 +0,0 @@ -[Unit] -Description=Origin Master Service -Documentation=https://github.com/openshift/origin -After=network-online.target -After=etcd.service -Before=origin-node.service -Requires=network-online.target - -[Service] -Type=notify -EnvironmentFile=/etc/sysconfig/origin-master -Environment=GOTRACEBACK=crash -ExecStart=/usr/bin/openshift start master --config=${CONFIG_FILE} $OPTIONS -LimitNOFILE=131072 -LimitCORE=infinity -WorkingDirectory=/var/lib/origin/ -SyslogIdentifier=origin-master -Restart=always -RestartSec=5s - -[Install] -WantedBy=multi-user.target -WantedBy=origin-node.service diff --git a/roles/openshift_master/handlers/main.yml b/roles/openshift_master/handlers/main.yml index 69c5a1663..d5094c2c9 100644 --- a/roles/openshift_master/handlers/main.yml +++ b/roles/openshift_master/handlers/main.yml @@ -1,17 +1,21 @@ --- -- name: restart master - systemd: name={{ openshift.common.service_type }}-master state=restarted - when: (openshift.master.ha is not defined or not openshift.master.ha | bool) and (not (master_service_status_changed | default(false) | bool)) - notify: Verify API Server - - name: restart master api - systemd: name={{ openshift.common.service_type }}-master-api state=restarted - when: (openshift.master.ha is defined and openshift.master.ha | bool) and (not (master_api_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native' - notify: Verify API Server + systemd: + name: "{{ openshift.common.service_type }}-master-api" + state: restarted + when: + - not (master_api_service_status_changed | default(false) | bool) + - openshift.master.cluster_method == 'native' + notify: + - Verify API Server - name: restart master controllers - systemd: name={{ openshift.common.service_type }}-master-controllers state=restarted - when: (openshift.master.ha is defined and openshift.master.ha | bool) and (not (master_controllers_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native' + systemd: + name: "{{ openshift.common.service_type }}-master-controllers" + state: restarted + when: + - not (master_controllers_service_status_changed | default(false) | bool) + - openshift.master.cluster_method == 'native' - name: Verify API Server # Using curl here since the uri module requires python-httplib2 and @@ -28,8 +32,8 @@ # Disables the following warning: # Consider using get_url or uri module rather than running curl warn: no - register: api_available_output - until: api_available_output.stdout == 'ok' + register: l_api_available_output + until: l_api_available_output.stdout == 'ok' retries: 120 delay: 1 changed_when: false diff --git a/roles/openshift_master/meta/main.yml b/roles/openshift_master/meta/main.yml index 907f25bc5..a657668a9 100644 --- a/roles/openshift_master/meta/main.yml +++ b/roles/openshift_master/meta/main.yml @@ -13,34 +13,4 @@ galaxy_info: - cloud dependencies: - role: lib_openshift -- role: openshift_master_facts -- role: openshift_hosted_facts -- role: openshift_master_certificates -- role: openshift_etcd_client_certificates - etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}" - etcd_cert_config_dir: "{{ openshift.common.config_base }}/master" - etcd_cert_prefix: "master.etcd-" - when: groups.oo_etcd_to_config | default([]) | length != 0 -- role: openshift_clock -- role: openshift_cloud_provider -- role: openshift_builddefaults -- role: openshift_buildoverrides -- role: os_firewall - os_firewall_allow: - - service: api server https - port: "{{ openshift.master.api_port }}/tcp" - - service: api controllers https - port: "{{ openshift.master.controllers_port }}/tcp" - - service: skydns tcp - port: "{{ openshift.master.dns_port }}/tcp" - - service: skydns udp - port: "{{ openshift.master.dns_port }}/udp" -- role: os_firewall - os_firewall_allow: - - service: etcd embedded - port: 4001/tcp - when: groups.oo_etcd_to_config | default([]) | length == 0 -- role: nickhammond.logrotate -- role: contiv - contiv_role: netmaster - when: openshift.common.use_contiv | bool +- role: lib_os_firewall diff --git a/roles/openshift_master/tasks/bootstrap.yml b/roles/openshift_master/tasks/bootstrap.yml new file mode 100644 index 000000000..0013f5289 --- /dev/null +++ b/roles/openshift_master/tasks/bootstrap.yml @@ -0,0 +1,28 @@ +--- + +- name: ensure the node-bootstrap service account exists + oc_serviceaccount: + name: node-bootstrapper + namespace: openshift-infra + state: present + run_once: true + +- name: grant node-bootstrapper the correct permissions to bootstrap + oc_adm_policy_user: + namespace: openshift-infra + user: system:serviceaccount:openshift-infra:node-bootstrapper + resource_kind: cluster-role + resource_name: system:node-bootstrapper + state: present + run_once: true + +# TODO: create a module for this command. +# oc_serviceaccounts_kubeconfig +- name: create service account kubeconfig with csr rights + command: "oc serviceaccounts create-kubeconfig node-bootstrapper -n openshift-infra" + register: kubeconfig_out + +- name: put service account kubeconfig into a file on disk for bootstrap + copy: + content: "{{ kubeconfig_out.stdout }}" + dest: "{{ openshift_master_config_dir }}/bootstrap.kubeconfig" diff --git a/roles/openshift_master/tasks/clean_systemd_units.yml b/roles/openshift_master/tasks/clean_systemd_units.yml new file mode 100644 index 000000000..e641f84d4 --- /dev/null +++ b/roles/openshift_master/tasks/clean_systemd_units.yml @@ -0,0 +1,9 @@ +--- + +- name: Disable master service + systemd: + name: "{{ openshift.common.service_type }}-master" + state: stopped + enabled: no + masked: yes + ignore_errors: true diff --git a/roles/openshift_master/tasks/files b/roles/openshift_master/tasks/files deleted file mode 120000 index feb122881..000000000 --- a/roles/openshift_master/tasks/files +++ /dev/null @@ -1 +0,0 @@ -../files
\ No newline at end of file diff --git a/roles/openshift_master/tasks/firewall.yml b/roles/openshift_master/tasks/firewall.yml new file mode 100644 index 000000000..38afb6764 --- /dev/null +++ b/roles/openshift_master/tasks/firewall.yml @@ -0,0 +1,44 @@ +--- +- when: r_openshift_master_firewall_enabled | bool and not r_openshift_master_use_firewalld | bool + block: + - name: Add iptables allow rules + os_firewall_manage_iptables: + name: "{{ item.service }}" + action: add + protocol: "{{ item.port.split('/')[1] }}" + port: "{{ item.port.split('/')[0] }}" + when: + - item.cond | default(True) + with_items: "{{ r_openshift_master_os_firewall_allow }}" + + - name: Remove iptables rules + os_firewall_manage_iptables: + name: "{{ item.service }}" + action: remove + protocol: "{{ item.port.split('/')[1] }}" + port: "{{ item.port.split('/')[0] }}" + when: + - item.cond | default(True) + with_items: "{{ r_openshift_master_os_firewall_deny }}" + +- when: r_openshift_master_firewall_enabled | bool and r_openshift_master_use_firewalld | bool + block: + - name: Add firewalld allow rules + firewalld: + port: "{{ item.port }}" + permanent: true + immediate: true + state: enabled + when: + - item.cond | default(True) + with_items: "{{ r_openshift_master_os_firewall_allow }}" + + - name: Remove firewalld allow rules + firewalld: + port: "{{ item.port }}" + permanent: true + immediate: true + state: disabled + when: + - item.cond | default(True) + with_items: "{{ r_openshift_master_os_firewall_deny }}" diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml index 9b7125240..82b4b420c 100644 --- a/roles/openshift_master/tasks/main.yml +++ b/roles/openshift_master/tasks/main.yml @@ -7,40 +7,60 @@ - fail: msg: > Invalid OAuth grant method: {{ openshift_master_oauth_grant_method }} - when: openshift_master_oauth_grant_method is defined and openshift_master_oauth_grant_method not in openshift_master_valid_grant_methods + when: + - openshift_master_oauth_grant_method is defined + - openshift_master_oauth_grant_method not in openshift_master_valid_grant_methods # HA Variable Validation - fail: msg: "openshift_master_cluster_method must be set to either 'native' or 'pacemaker' for multi-master installations" - when: openshift_master_ha | bool and ((openshift_master_cluster_method is not defined) or (openshift_master_cluster_method is defined and openshift_master_cluster_method not in ["native", "pacemaker"])) + when: + - openshift.master.ha | bool + - (openshift.master.cluster_method is not defined) or (openshift.master.cluster_method is defined and openshift.master.cluster_method not in ["native", "pacemaker"]) - fail: msg: "'native' high availability is not supported for the requested OpenShift version" - when: openshift_master_ha | bool and openshift_master_cluster_method == "native" and not openshift.common.version_gte_3_1_or_1_1 | bool + when: + - openshift.master.ha | bool + - openshift.master.cluster_method == "native" + - not openshift.common.version_gte_3_1_or_1_1 | bool - fail: msg: "openshift_master_cluster_password must be set for multi-master installations" - when: openshift_master_ha | bool and openshift_master_cluster_method == "pacemaker" and (openshift_master_cluster_password is not defined or not openshift_master_cluster_password) + when: + - openshift.master.ha | bool + - openshift.master.cluster_method == "pacemaker" + - openshift_master_cluster_password is not defined or not openshift_master_cluster_password - fail: msg: "Pacemaker based HA is not supported at this time when used with containerized installs" - when: openshift_master_ha | bool and openshift_master_cluster_method == "pacemaker" and openshift.common.is_containerized | bool + when: + - openshift.master.ha | bool + - openshift.master.cluster_method == "pacemaker" + - openshift.common.is_containerized | bool + +- name: Open up firewall ports + include: firewall.yml + static: yes - name: Install Master package package: name: "{{ openshift.common.service_type }}-master{{ openshift_pkg_version | default('') | oo_image_tag_to_rpm_version(include_dash=True) }}" state: present - when: not openshift.common.is_containerized | bool + when: + - not openshift.common.is_containerized | bool -- name: Create openshift.common.data_dir +- name: Create r_openshift_master_data_dir file: - path: "{{ openshift.common.data_dir }}" + path: "{{ r_openshift_master_data_dir }}" state: directory mode: 0755 owner: root group: root - when: openshift.common.is_containerized | bool + when: + - openshift.common.is_containerized | bool - name: Reload systemd units command: systemctl daemon-reload - when: openshift.common.is_containerized | bool and install_result | changed + when: + - openshift.common.is_containerized | bool - name: Re-gather package dependent master facts openshift_facts: @@ -57,9 +77,8 @@ args: creates: "{{ openshift_master_policy }}" notify: - - restart master - - restart master api - - restart master controllers + - restart master api + - restart master controllers - name: Create the scheduler config copy: @@ -67,21 +86,22 @@ dest: "{{ openshift_master_scheduler_conf }}" backup: true notify: - - restart master - - restart master api - - restart master controllers + - restart master api + - restart master controllers - name: Install httpd-tools if needed package: name=httpd-tools state=present - when: (item.kind == 'HTPasswdPasswordIdentityProvider') and - not openshift.common.is_atomic | bool + when: + - item.kind == 'HTPasswdPasswordIdentityProvider' + - not openshift.common.is_atomic | bool with_items: "{{ openshift.master.identity_providers }}" - name: Ensure htpasswd directory exists file: path: "{{ item.filename | dirname }}" state: directory - when: item.kind == 'HTPasswdPasswordIdentityProvider' + when: + - item.kind == 'HTPasswdPasswordIdentityProvider' with_items: "{{ openshift.master.identity_providers }}" - name: Create the htpasswd file if needed @@ -89,7 +109,9 @@ dest: "{{ item.filename }}" src: htpasswd.j2 backup: yes - when: item.kind == 'HTPasswdPasswordIdentityProvider' and openshift.master.manage_htpasswd | bool + when: + - item.kind == 'HTPasswdPasswordIdentityProvider' + - openshift.master.manage_htpasswd | bool with_items: "{{ openshift.master.identity_providers }}" - name: Ensure htpasswd file exists @@ -98,7 +120,8 @@ force: no content: "" mode: 0600 - when: item.kind == 'HTPasswdPasswordIdentityProvider' + when: + - item.kind == 'HTPasswdPasswordIdentityProvider' with_items: "{{ openshift.master.identity_providers }}" - name: Create the ldap ca file if needed @@ -107,7 +130,9 @@ content: "{{ openshift.master.ldap_ca }}" mode: 0600 backup: yes - when: openshift.master.ldap_ca is defined and item.kind == 'LDAPPasswordIdentityProvider' + when: + - openshift.master.ldap_ca is defined + - item.kind == 'LDAPPasswordIdentityProvider' with_items: "{{ openshift.master.identity_providers }}" - name: Create the openid ca file if needed @@ -116,7 +141,10 @@ content: "{{ openshift.master.openid_ca }}" mode: 0600 backup: yes - when: openshift.master.openid_ca is defined and item.kind == 'OpenIDIdentityProvider' and item.ca | default('') != '' + when: + - openshift.master.openid_ca is defined + - item.kind == 'OpenIDIdentityProvider' + - item.ca | default('') != '' with_items: "{{ openshift.master.identity_providers }}" - name: Create the request header ca file if needed @@ -125,27 +153,60 @@ content: "{{ openshift.master.request_header_ca }}" mode: 0600 backup: yes - when: openshift.master.request_header_ca is defined and item.kind == 'RequestHeaderIdentityProvider' and item.clientCA | default('') != '' + when: + - openshift.master.request_header_ca is defined + - item.kind == 'RequestHeaderIdentityProvider' + - item.clientCA | default('') != '' with_items: "{{ openshift.master.identity_providers }}" # This is an ugly hack to verify settings are in a file without modifying them with lineinfile. # The template file will stomp any other settings made. - block: - - name: check whether our docker-registry setting exists in the env file - command: "awk '/^OPENSHIFT_DEFAULT_REGISTRY=docker-registry.default.svc:5000/' /etc/sysconfig/{{ openshift.common.service_type }}-master" - ignore_errors: true - changed_when: false - register: already_set + - name: check whether our docker-registry setting exists in the env file + command: "awk '/^OPENSHIFT_DEFAULT_REGISTRY=docker-registry.default.svc:5000/' /etc/sysconfig/{{ openshift.common.service_type }}-master" + failed_when: false + changed_when: false + register: l_already_set + + - set_fact: + openshift_push_via_dns: "{{ openshift.common.version_gte_3_6 or (l_already_set.stdout is defined and l_already_set.stdout | match('OPENSHIFT_DEFAULT_REGISTRY=docker-registry.default.svc:5000')) }}" - - set_fact: - openshift_push_via_dns: "{{ (openshift_use_dnsmasq | default(true) and openshift.common.version_gte_3_6) or (already_set.stdout | match('OPENSHIFT_DEFAULT_REGISTRY=docker-registry.default.svc:5000')) }}" +- name: Set fact of all etcd host IPs + openshift_facts: + role: common + local_facts: + no_proxy_etcd_host_ips: "{{ openshift_no_proxy_etcd_host_ips }}" - name: Install the systemd units include: systemd_units.yml +- name: Checking for journald.conf + stat: path=/etc/systemd/journald.conf + register: journald_conf_file + +- name: Update journald setup + replace: + dest: /etc/systemd/journald.conf + regexp: '^(\#| )?{{ item.var }}=\s*.*?$' + replace: ' {{ item.var }}={{ item.val }}' + backup: yes + with_items: "{{ journald_vars_to_replace | default([]) }}" + when: journald_conf_file.stat.exists + register: journald_update + +# I need to restart journald immediatelly, otherwise it gets into way during +# further steps in ansible +- name: Restart journald + systemd: + name: systemd-journald + state: restarted + when: journald_update | changed + - name: Install Master system container include: system_container.yml - when: openshift.common.is_containerized | bool and openshift.common.is_master_system_container | bool + when: + - openshift.common.is_containerized | bool + - openshift.common.is_master_system_container | bool - name: Create session secrets file template: @@ -154,13 +215,14 @@ owner: root group: root mode: 0600 - when: openshift.master.session_auth_secrets is defined and openshift.master.session_encryption_secrets is defined + when: + - openshift.master.session_auth_secrets is defined + - openshift.master.session_encryption_secrets is defined notify: - - restart master - - restart master api + - restart master api - set_fact: - translated_identity_providers: "{{ openshift.master.identity_providers | translate_idps('v1', openshift.common.version, openshift.common.deployment_type) }}" + translated_identity_providers: "{{ openshift.master.identity_providers | translate_idps('v1') }}" # TODO: add the validate parameter when there is a validation command to run - name: Create master config @@ -172,89 +234,80 @@ group: root mode: 0600 notify: - - restart master - - restart master api - - restart master controllers + - restart master api + - restart master controllers + +- name: modify controller args + yedit: + src: /etc/origin/master/master-config.yaml + edits: + - key: kubernetesMasterConfig.controllerArguments.cluster-signing-cert-file + value: + - /etc/origin/master/ca.crt + - key: kubernetesMasterConfig.controllerArguments.cluster-signing-key-file + value: + - /etc/origin/master/ca.key + notify: + - restart master controllers + when: openshift_master_bootstrap_enabled | default(False) - include: set_loopback_context.yml - when: openshift.common.version_gte_3_2_or_1_2 - -# TODO: Master startup can fail when ec2 transparently reallocates the block -# storage, causing etcd writes to temporarily fail. Retry failures blindly just -# once to allow time for this transient condition to to resolve and for systemd -# to restart the master (which will eventually succeed). -# -# https://github.com/coreos/etcd/issues/3864 -# https://github.com/openshift/origin/issues/6065 -# https://github.com/openshift/origin/issues/6447 -- name: Start and enable master - systemd: - daemon_reload: yes - name: "{{ openshift.common.service_type }}-master" - enabled: yes - state: started - when: not openshift_master_ha | bool - register: start_result - until: not start_result | failed - retries: 1 - delay: 60 - notify: Verify API Server - -- name: Stop and disable non-HA master when running HA - systemd: - name: "{{ openshift.common.service_type }}-master" - enabled: no - state: stopped - when: openshift_master_ha | bool - register: task_result - failed_when: task_result|failed and 'could not' not in task_result.msg|lower - -- set_fact: - master_service_status_changed: "{{ start_result | changed }}" - when: not openshift_master_ha | bool - -- name: Mask master service - systemd: - name: "{{ openshift.common.service_type }}-master" - masked: yes - when: > - openshift_master_ha | bool and - openshift.master.cluster_method == 'native' and - not openshift.common.is_containerized | bool + when: + - openshift.common.version_gte_3_2_or_1_2 - name: Start and enable master api on first master systemd: name: "{{ openshift.common.service_type }}-master-api" enabled: yes state: started - when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' and inventory_hostname == openshift_master_hosts[0] - register: start_result - until: not start_result | failed + when: + - openshift.master.cluster_method == 'native' + - inventory_hostname == openshift_master_hosts[0] + register: l_start_result + until: not l_start_result | failed retries: 1 delay: 60 +- name: Dump logs from master-api if it failed + command: journalctl --no-pager -n 100 -u {{ openshift.common.service_type }}-master-api + when: + - l_start_result | failed + - set_fact: - master_api_service_status_changed: "{{ start_result | changed }}" - when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' and inventory_hostname == openshift_master_hosts[0] + master_api_service_status_changed: "{{ l_start_result | changed }}" + when: + - openshift.master.cluster_method == 'native' + - inventory_hostname == openshift_master_hosts[0] - pause: seconds: 15 - when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' + when: + - openshift.master.ha | bool + - openshift.master.cluster_method == 'native' - name: Start and enable master api all masters systemd: name: "{{ openshift.common.service_type }}-master-api" enabled: yes state: started - when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' and inventory_hostname != openshift_master_hosts[0] - register: start_result - until: not start_result | failed + when: + - openshift.master.cluster_method == 'native' + - inventory_hostname != openshift_master_hosts[0] + register: l_start_result + until: not l_start_result | failed retries: 1 delay: 60 +- name: Dump logs from master-api if it failed + command: journalctl --no-pager -n 100 -u {{ openshift.common.service_type }}-master-api + when: + - l_start_result | failed + - set_fact: - master_api_service_status_changed: "{{ start_result | changed }}" - when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' and inventory_hostname != openshift_master_hosts[0] + master_api_service_status_changed: "{{ l_start_result | changed }}" + when: + - openshift.master.cluster_method == 'native' + - inventory_hostname != openshift_master_hosts[0] # A separate wait is required here for native HA since notifies will # be resolved after all tasks in the role. @@ -269,59 +322,84 @@ --cacert {{ openshift.common.config_base }}/master/ca.crt {% endif %} {{ openshift.master.api_url }}/healthz/ready - register: api_available_output - until: api_available_output.stdout == 'ok' + register: l_api_available_output + until: l_api_available_output.stdout == 'ok' retries: 120 delay: 1 run_once: true changed_when: false - when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' and master_api_service_status_changed | bool + when: + - openshift.master.cluster_method == 'native' + - master_api_service_status_changed | bool - name: Start and enable master controller on first master systemd: name: "{{ openshift.common.service_type }}-master-controllers" enabled: yes state: started - when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' and inventory_hostname == openshift_master_hosts[0] - register: start_result - until: not start_result | failed + when: + - openshift.master.cluster_method == 'native' + - inventory_hostname == openshift_master_hosts[0] + register: l_start_result + until: not l_start_result | failed retries: 1 delay: 60 +- name: Dump logs from master-controllers if it failed + command: journalctl --no-pager -n 100 -u {{ openshift.common.service_type }}-master-controllers + when: + - l_start_result | failed + - name: Wait for master controller service to start on first master pause: seconds: 15 - when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' + when: + - openshift.master.cluster_method == 'native' - name: Start and enable master controller on all masters systemd: name: "{{ openshift.common.service_type }}-master-controllers" enabled: yes state: started - when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' and inventory_hostname != openshift_master_hosts[0] - register: start_result - until: not start_result | failed + when: + - openshift.master.cluster_method == 'native' + - inventory_hostname != openshift_master_hosts[0] + register: l_start_result + until: not l_start_result | failed retries: 1 delay: 60 +- name: Dump logs from master-controllers if it failed + command: journalctl --no-pager -n 100 -u {{ openshift.common.service_type }}-master-controllers + when: + - l_start_result | failed + - set_fact: - master_controllers_service_status_changed: "{{ start_result | changed }}" - when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' + master_controllers_service_status_changed: "{{ l_start_result | changed }}" + when: + - openshift.master.cluster_method == 'native' - name: Install cluster packages package: name=pcs state=present - when: openshift_master_ha | bool and openshift.master.cluster_method == 'pacemaker' - and not openshift.common.is_containerized | bool - register: install_result + when: + - openshift.master.cluster_method == 'pacemaker' + - not openshift.common.is_containerized | bool + register: l_install_result - name: Start and enable cluster service systemd: name: pcsd enabled: yes state: started - when: openshift_master_ha | bool and openshift.master.cluster_method == 'pacemaker' - and not openshift.common.is_containerized | bool + when: + - openshift.master.cluster_method == 'pacemaker' + - not openshift.common.is_containerized | bool - name: Set the cluster user password shell: echo {{ openshift_master_cluster_password | quote }} | passwd --stdin hacluster - when: install_result | changed + when: + - l_install_result | changed + +- name: node bootstrap settings + include: bootstrap.yml + when: openshift_master_bootstrap_enabled | default(False) diff --git a/roles/openshift_master/tasks/registry_auth.yml b/roles/openshift_master/tasks/registry_auth.yml new file mode 100644 index 000000000..2644f235e --- /dev/null +++ b/roles/openshift_master/tasks/registry_auth.yml @@ -0,0 +1,35 @@ +--- +# We need to setup some variables as this play might be called directly +# from outside of the role. +- set_fact: + oreg_auth_credentials_path: "{{ r_openshift_master_data_dir }}/.docker" + when: oreg_auth_credentials_path is not defined + +- set_fact: + oreg_host: "{{ oreg_url.split('/')[0] if (oreg_url is defined and '.' in oreg_url.split('/')[0]) else '' }}" + when: oreg_host is not defined + +- name: Check for credentials file for registry auth + stat: + path: "{{ oreg_auth_credentials_path }}" + when: oreg_auth_user is defined + register: master_oreg_auth_credentials_stat + +- name: Create credentials for registry auth + command: "docker --config={{ oreg_auth_credentials_path }} login -u {{ oreg_auth_user }} -p {{ oreg_auth_password }} {{ oreg_host }}" + when: + - oreg_auth_user is defined + - (not master_oreg_auth_credentials_stat.stat.exists or oreg_auth_credentials_replace) | bool + register: master_oreg_auth_credentials_create + notify: + - restart master api + - restart master controllers + +# Container images may need the registry credentials +- name: Setup ro mount of /root/.docker for containerized hosts + set_fact: + l_bind_docker_reg_auth: True + when: + - openshift.common.is_containerized | bool + - oreg_auth_user is defined + - (master_oreg_auth_credentials_stat.stat.exists or oreg_auth_credentials_replace or master_oreg_auth_credentials_create.changed) | bool diff --git a/roles/openshift_master/tasks/set_loopback_context.yml b/roles/openshift_master/tasks/set_loopback_context.yml index 9c3fb31dc..308b2f4cd 100644 --- a/roles/openshift_master/tasks/set_loopback_context.yml +++ b/roles/openshift_master/tasks/set_loopback_context.yml @@ -4,7 +4,7 @@ {{ openshift.common.client_binary }} config view --config={{ openshift_master_loopback_config }} changed_when: false - register: loopback_config + register: l_loopback_config - command: > {{ openshift.common.client_binary }} config set-cluster @@ -12,7 +12,8 @@ --embed-certs=true --server={{ openshift.master.loopback_api_url }} {{ openshift.master.loopback_cluster_name }} --config={{ openshift_master_loopback_config }} - when: loopback_context_string not in loopback_config.stdout + when: + - loopback_context_string not in l_loopback_config.stdout register: set_loopback_cluster - command: > @@ -21,11 +22,13 @@ --namespace=default --user={{ openshift.master.loopback_user }} {{ openshift.master.loopback_context_name }} --config={{ openshift_master_loopback_config }} - when: set_loopback_cluster | changed - register: set_loopback_context + when: + - set_loopback_cluster | changed + register: l_set_loopback_context - command: > {{ openshift.common.client_binary }} config use-context {{ openshift.master.loopback_context_name }} --config={{ openshift_master_loopback_config }} - when: set_loopback_context | changed + when: + - l_set_loopback_context | changed register: set_current_context diff --git a/roles/openshift_master/tasks/system_container.yml b/roles/openshift_master/tasks/system_container.yml index 8f77d40ce..91332acfb 100644 --- a/roles/openshift_master/tasks/system_container.yml +++ b/roles/openshift_master/tasks/system_container.yml @@ -1,40 +1,27 @@ --- - name: Pre-pull master system container image command: > - atomic pull --storage=ostree {{ openshift.common.system_images_registry }}/{{ openshift.master.master_system_image }}:{{ openshift_image_tag }} - register: pull_result - changed_when: "'Pulling layer' in pull_result.stdout" + atomic pull --storage=ostree {{ 'docker:' if openshift.common.system_images_registry == 'docker' else openshift.common.system_images_registry + '/' }}{{ openshift.master.master_system_image }}:{{ openshift_image_tag }} + register: l_pull_result + changed_when: "'Pulling layer' in l_pull_result.stdout" - name: Check Master system container package command: > atomic containers list --no-trunc -a -f container={{ openshift.common.service_type }}-master - register: result - -- name: Install or Update master system container - oc_atomic_container: - name: "{{ openshift.common.service_type }}-master" - image: "{{ openshift.common.system_images_registry }}/{{ openshift.master.master_system_image }}:{{ openshift_image_tag }}" - state: latest - when: - - not l_is_ha # HA - name: Install or Update HA api master system container oc_atomic_container: name: "{{ openshift.common.service_type }}-master-api" - image: "{{ openshift.common.system_images_registry }}/{{ openshift.master.master_system_image }}:{{ openshift_image_tag }}" + image: "{{ 'docker:' if openshift.common.system_images_registry == 'docker' else openshift.common.system_images_registry + '/' }}{{ openshift.master.master_system_image }}:{{ openshift_image_tag }}" state: latest values: - - COMMAND=api - when: - - l_is_ha + - COMMAND=api - name: Install or Update HA controller master system container oc_atomic_container: name: "{{ openshift.common.service_type }}-master-controllers" - image: "{{ openshift.common.system_images_registry }}/{{ openshift.master.master_system_image }}:{{ openshift_image_tag }}" + image: "{{ 'docker:' if openshift.common.system_images_registry == 'docker' else openshift.common.system_images_registry + '/' }}{{ openshift.master.master_system_image }}:{{ openshift_image_tag }}" state: latest values: - - COMMAND=controllers - when: - - l_is_ha + - COMMAND=controllers diff --git a/roles/openshift_master/tasks/systemd_units.yml b/roles/openshift_master/tasks/systemd_units.yml index dfc255b3d..8de62c59a 100644 --- a/roles/openshift_master/tasks/systemd_units.yml +++ b/roles/openshift_master/tasks/systemd_units.yml @@ -3,6 +3,25 @@ # playbooks. For that reason the ha_svc variables are use set_fact instead of # the vars directory on the role. +# This play may be consumed outside the role, we need to ensure that +# openshift_master_config_dir is set. +- name: Set openshift_master_config_dir if unset + set_fact: + openshift_master_config_dir: '/etc/origin/master' + when: openshift_master_config_dir is not defined + +# This play may be consumed outside the role, we need to ensure that +# r_openshift_master_data_dir is set. +- name: Set r_openshift_master_data_dir if unset + set_fact: + r_openshift_master_data_dir: "{{ openshift_data_dir | default('/var/lib/origin') }}" + when: r_openshift_master_data_dir is not defined + +- include: registry_auth.yml + +- name: Remove the legacy master service if it exists + include: clean_systemd_units.yml + - name: Init HA Service Info set_fact: containerized_svc_dir: "/usr/lib/systemd/system" @@ -12,65 +31,49 @@ set_fact: containerized_svc_dir: "/etc/systemd/system" ha_svc_template_path: "docker-cluster" - when: openshift.common.is_containerized | bool + when: + - openshift.common.is_containerized | bool # This is the image used for both HA and non-HA clusters: - name: Pre-pull master image command: > docker pull {{ openshift.master.master_image }}:{{ openshift_image_tag }} - register: pull_result - changed_when: "'Downloaded newer image' in pull_result.stdout" - when: openshift.common.is_containerized | bool and not openshift.common.is_master_system_container | bool - -# workaround for missing systemd unit files -- name: Create the systemd unit files - template: - src: "master_docker/master.docker.service.j2" - dest: "{{ containerized_svc_dir }}/{{ openshift.common.service_type }}-master.service" + register: l_pull_result + changed_when: "'Downloaded newer image' in l_pull_result.stdout" when: - - openshift.common.is_containerized | bool and (openshift.master.ha is not defined or not openshift.master.ha) | bool + - openshift.common.is_containerized | bool - not openshift.common.is_master_system_container | bool - register: create_master_unit_file - -- name: Install Master service file - copy: - dest: "/etc/systemd/system/{{ openshift.common.service_type }}-master.service" - src: "{{ openshift.common.service_type }}-master.service" - register: create_master_unit_file - when: - - not openshift.common.is_containerized | bool - - (openshift.master.ha is not defined or not openshift.master.ha) | bool - -- command: systemctl daemon-reload - when: create_master_unit_file | changed - name: Create the ha systemd unit files template: src: "{{ ha_svc_template_path }}/atomic-openshift-master-{{ item }}.service.j2" dest: "{{ containerized_svc_dir }}/{{ openshift.common.service_type }}-master-{{ item }}.service" when: - - openshift.master.ha is defined and openshift.master.ha | bool and openshift_master_cluster_method == "native" + - openshift.master.cluster_method == "native" - not openshift.common.is_master_system_container | bool with_items: - api - controllers - register: create_ha_unit_files + register: l_create_ha_unit_files - command: systemctl daemon-reload - when: create_ha_unit_files | changed + when: + - l_create_ha_unit_files | changed # end workaround for missing systemd unit files - name: Preserve Master API Proxy Config options command: grep PROXY /etc/sysconfig/{{ openshift.common.service_type }}-master-api - register: master_api_proxy - when: openshift.master.ha is defined and openshift.master.ha | bool and openshift_master_cluster_method == "native" + register: l_master_api_proxy + when: + - openshift.master.cluster_method == "native" failed_when: false changed_when: false - name: Preserve Master API AWS options command: grep AWS_ /etc/sysconfig/{{ openshift.common.service_type }}-master-api register: master_api_aws - when: openshift.master.ha is defined and openshift.master.ha | bool and openshift_master_cluster_method == "native" + when: + - openshift.master.cluster_method == "native" failed_when: false changed_when: false @@ -79,22 +82,27 @@ src: "{{ ha_svc_template_path }}/atomic-openshift-master-api.j2" dest: /etc/sysconfig/{{ openshift.common.service_type }}-master-api backup: true - when: openshift.master.ha is defined and openshift.master.ha | bool and openshift_master_cluster_method == "native" + when: + - openshift.master.cluster_method == "native" notify: - restart master api - name: Restore Master API Proxy Config Options - when: openshift.master.ha is defined and openshift.master.ha | bool and openshift_master_cluster_method == "native" - and master_api_proxy.rc == 0 and 'http_proxy' not in openshift.common and 'https_proxy' not in openshift.common + when: + - openshift.master.cluster_method == "native" + - l_master_api_proxy.rc == 0 + - "'http_proxy' not in openshift.common" + - "'https_proxy' not in openshift.common" lineinfile: dest: /etc/sysconfig/{{ openshift.common.service_type }}-master-api line: "{{ item }}" - with_items: "{{ master_api_proxy.stdout_lines | default([]) }}" + with_items: "{{ l_master_api_proxy.stdout_lines | default([]) }}" - name: Restore Master API AWS Options - when: openshift.master.ha is defined and openshift.master.ha | bool and openshift_master_cluster_method == "native" - and master_api_aws.rc == 0 and - not (openshift_cloudprovider_kind is defined and openshift_cloudprovider_kind == 'aws' and openshift_cloudprovider_aws_access_key is defined and openshift_cloudprovider_aws_secret_key is defined) + when: + - openshift.master.cluster_method == "native" + - master_api_aws.rc == 0 + - not (openshift_cloudprovider_kind is defined and openshift_cloudprovider_kind == 'aws' and openshift_cloudprovider_aws_access_key is defined and openshift_cloudprovider_aws_secret_key is defined) lineinfile: dest: /etc/sysconfig/{{ openshift.common.service_type }}-master-api line: "{{ item }}" @@ -104,14 +112,16 @@ - name: Preserve Master Controllers Proxy Config options command: grep PROXY /etc/sysconfig/{{ openshift.common.service_type }}-master-controllers register: master_controllers_proxy - when: openshift.master.ha is defined and openshift.master.ha | bool and openshift_master_cluster_method == "native" + when: + - openshift.master.cluster_method == "native" failed_when: false changed_when: false - name: Preserve Master Controllers AWS options command: grep AWS_ /etc/sysconfig/{{ openshift.common.service_type }}-master-controllers register: master_controllers_aws - when: openshift.master.ha is defined and openshift.master.ha | bool and openshift_master_cluster_method == "native" + when: + - openshift.master.cluster_method == "native" failed_when: false changed_when: false @@ -120,7 +130,8 @@ src: "{{ ha_svc_template_path }}/atomic-openshift-master-controllers.j2" dest: /etc/sysconfig/{{ openshift.common.service_type }}-master-controllers backup: true - when: openshift.master.ha is defined and openshift.master.ha | bool and openshift_master_cluster_method == "native" + when: + - openshift.master.cluster_method == "native" notify: - restart master controllers @@ -129,47 +140,18 @@ dest: /etc/sysconfig/{{ openshift.common.service_type }}-master-controllers line: "{{ item }}" with_items: "{{ master_controllers_proxy.stdout_lines | default([]) }}" - when: openshift.master.ha is defined and openshift.master.ha | bool and openshift_master_cluster_method == "native" - and master_controllers_proxy.rc == 0 and 'http_proxy' not in openshift.common and 'https_proxy' not in openshift.common + when: + - openshift.master.cluster_method == "native" + - master_controllers_proxy.rc == 0 + - "'http_proxy' not in openshift.common" + - "'https_proxy' not in openshift.common" - name: Restore Master Controllers AWS Options lineinfile: dest: /etc/sysconfig/{{ openshift.common.service_type }}-master-controllers line: "{{ item }}" with_items: "{{ master_controllers_aws.stdout_lines | default([]) }}" - when: openshift.master.ha is defined and openshift.master.ha | bool and openshift_master_cluster_method == "native" - and master_controllers_aws.rc == 0 and - not (openshift_cloudprovider_kind is defined and openshift_cloudprovider_kind == 'aws' and openshift_cloudprovider_aws_access_key is defined and openshift_cloudprovider_aws_secret_key is defined) - -- name: Install Master docker service file - template: - dest: "/etc/systemd/system/{{ openshift.common.service_type }}-master.service" - src: master_docker/master.docker.service.j2 - register: install_result - when: openshift.common.is_containerized | bool and openshift.master.ha is defined and not openshift.master.ha | bool and not openshift.common.is_master_system_container | bool - -- name: Preserve Master Proxy Config options - command: grep PROXY /etc/sysconfig/{{ openshift.common.service_type }}-master - register: master_proxy_result - failed_when: false - changed_when: false - -- set_fact: - master_proxy: "{{ master_proxy_result.stdout_lines | default([]) }}" - -- name: Preserve Master AWS options - command: grep AWS_ /etc/sysconfig/{{ openshift.common.service_type }}-master - register: master_aws_result - failed_when: false - changed_when: false - -- set_fact: - master_aws: "{{ master_aws_result.stdout_lines | default([]) }}" - -- name: Create the master service env file - template: - src: "atomic-openshift-master.j2" - dest: /etc/sysconfig/{{ openshift.common.service_type }}-master - backup: true - notify: - - restart master + when: + - openshift.master.cluster_method == "native" + - master_controllers_aws.rc == 0 + - not (openshift_cloudprovider_kind is defined and openshift_cloudprovider_kind == 'aws' and openshift_cloudprovider_aws_access_key is defined and openshift_cloudprovider_aws_secret_key is defined) diff --git a/roles/openshift_master/tasks/update_etcd_client_urls.yml b/roles/openshift_master/tasks/update_etcd_client_urls.yml new file mode 100644 index 000000000..1ab105808 --- /dev/null +++ b/roles/openshift_master/tasks/update_etcd_client_urls.yml @@ -0,0 +1,8 @@ +--- +- yedit: + src: "{{ openshift.common.config_base }}/master/master-config.yaml" + key: 'etcdClientInfo.urls' + value: "{{ openshift.master.etcd_urls }}" + notify: + - restart master api + - restart master controllers diff --git a/roles/openshift_master/templates/atomic-openshift-master.j2 b/roles/openshift_master/templates/atomic-openshift-master.j2 index 850fae0e4..b931f1414 100644 --- a/roles/openshift_master/templates/atomic-openshift-master.j2 +++ b/roles/openshift_master/templates/atomic-openshift-master.j2 @@ -1,6 +1,9 @@ OPTIONS=--loglevel={{ openshift.master.debug_level | default(2) }} CONFIG_FILE={{ openshift_master_config_file }} -{% if openshift_push_via_dns | default(false) %} +{# Preserve existing OPENSHIFT_DEFAULT_REGISTRY settings in scale up runs #} +{% if openshift_master_is_scaleup_host %} +{{ openshift_master_default_registry_value }} +{% elif openshift_push_via_dns | default(false) %} OPENSHIFT_DEFAULT_REGISTRY=docker-registry.default.svc:5000 {% endif %} {% if openshift.common.is_containerized | bool %} diff --git a/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-api.service.j2 b/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-api.service.j2 index e8f7c47b0..5d4a99c97 100644 --- a/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-api.service.j2 +++ b/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-api.service.j2 @@ -12,12 +12,22 @@ Requires={{ openshift.docker.service_name }}.service EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-master-api Environment=GOTRACEBACK=crash ExecStartPre=-/usr/bin/docker rm -f {{ openshift.common.service_type}}-master-api -ExecStart=/usr/bin/docker run --rm --privileged --net=host --name {{ openshift.common.service_type }}-master-api --env-file=/etc/sysconfig/{{ openshift.common.service_type }}-master-api -v {{ openshift.common.data_dir }}:{{ openshift.common.data_dir }} -v /var/log:/var/log -v /var/run/docker.sock:/var/run/docker.sock -v {{ openshift.common.config_base }}:{{ openshift.common.config_base }} {% if openshift_cloudprovider_kind | default('') != '' -%} -v {{ openshift.common.config_base }}/cloudprovider:{{ openshift.common.config_base}}/cloudprovider {% endif -%} -v /etc/pki:/etc/pki:ro {{ openshift.master.master_image }}:${IMAGE_VERSION} start master api --config=${CONFIG_FILE} $OPTIONS +ExecStart=/usr/bin/docker run --rm --privileged --net=host \ + --name {{ openshift.common.service_type }}-master-api \ + --env-file=/etc/sysconfig/{{ openshift.common.service_type }}-master-api \ + -v {{ r_openshift_master_data_dir }}:{{ r_openshift_master_data_dir }} \ + -v /var/log:/var/log -v /var/run/docker.sock:/var/run/docker.sock \ + -v {{ openshift.common.config_base }}:{{ openshift.common.config_base }} \ + {% if openshift_cloudprovider_kind | default('') != '' -%} -v {{ openshift.common.config_base }}/cloudprovider:{{ openshift.common.config_base}}/cloudprovider {% endif -%} \ + -v /etc/pki:/etc/pki:ro \ + {% if l_bind_docker_reg_auth | default(False) %} -v {{ oreg_auth_credentials_path }}:/root/.docker:ro{% endif %}\ + {{ openshift.master.master_image }}:${IMAGE_VERSION} start master api \ + --config=${CONFIG_FILE} $OPTIONS ExecStartPost=/usr/bin/sleep 10 ExecStop=/usr/bin/docker stop {{ openshift.common.service_type }}-master-api LimitNOFILE=131072 LimitCORE=infinity -WorkingDirectory={{ openshift.common.data_dir }} +WorkingDirectory={{ r_openshift_master_data_dir }} SyslogIdentifier={{ openshift.common.service_type }}-master-api Restart=always RestartSec=5s diff --git a/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-controllers.service.j2 b/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-controllers.service.j2 index 69db62f16..f93f3b565 100644 --- a/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-controllers.service.j2 +++ b/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-controllers.service.j2 @@ -11,12 +11,22 @@ PartOf={{ openshift.docker.service_name }}.service EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-master-controllers Environment=GOTRACEBACK=crash ExecStartPre=-/usr/bin/docker rm -f {{ openshift.common.service_type}}-master-controllers -ExecStart=/usr/bin/docker run --rm --privileged --net=host --name {{ openshift.common.service_type }}-master-controllers --env-file=/etc/sysconfig/{{ openshift.common.service_type }}-master-controllers -v {{ openshift.common.data_dir }}:{{ openshift.common.data_dir }} -v /var/run/docker.sock:/var/run/docker.sock -v {{ openshift.common.config_base }}:{{ openshift.common.config_base }} {% if openshift_cloudprovider_kind | default('') != '' -%} -v {{ openshift.common.config_base }}/cloudprovider:{{ openshift.common.config_base}}/cloudprovider {% endif -%} -v /etc/pki:/etc/pki:ro {{ openshift.master.master_image }}:${IMAGE_VERSION} start master controllers --config=${CONFIG_FILE} $OPTIONS +ExecStart=/usr/bin/docker run --rm --privileged --net=host \ + --name {{ openshift.common.service_type }}-master-controllers \ + --env-file=/etc/sysconfig/{{ openshift.common.service_type }}-master-controllers \ + -v {{ r_openshift_master_data_dir }}:{{ r_openshift_master_data_dir }} \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v {{ openshift.common.config_base }}:{{ openshift.common.config_base }} \ + {% if openshift_cloudprovider_kind | default('') != '' -%} -v {{ openshift.common.config_base }}/cloudprovider:{{ openshift.common.config_base}}/cloudprovider {% endif -%} \ + -v /etc/pki:/etc/pki:ro \ + {% if l_bind_docker_reg_auth | default(False) %} -v {{ oreg_auth_credentials_path }}:/root/.docker:ro{% endif %}\ + {{ openshift.master.master_image }}:${IMAGE_VERSION} start master controllers \ + --config=${CONFIG_FILE} $OPTIONS ExecStartPost=/usr/bin/sleep 10 ExecStop=/usr/bin/docker stop {{ openshift.common.service_type }}-master-controllers LimitNOFILE=131072 LimitCORE=infinity -WorkingDirectory={{ openshift.common.data_dir }} +WorkingDirectory={{ r_openshift_master_data_dir }} SyslogIdentifier={{ openshift.common.service_type }}-master-controllers Restart=always RestartSec=5s diff --git a/roles/openshift_master/templates/master.yaml.v1.j2 b/roles/openshift_master/templates/master.yaml.v1.j2 index af3ebc6d2..d045b402b 100644 --- a/roles/openshift_master/templates/master.yaml.v1.j2 +++ b/roles/openshift_master/templates/master.yaml.v1.j2 @@ -47,11 +47,10 @@ assetConfig: {% if openshift.master.audit_config | default(none) is not none and openshift.common.version_gte_3_2_or_1_2 | bool %} auditConfig:{{ openshift.master.audit_config | to_padded_yaml(level=1) }} {% endif %} -{% if openshift_master_ha | bool %} -controllerLeaseTTL: {{ openshift.master.controller_lease_ttl | default('30') }} -{% endif %} {% if openshift.common.version_gte_3_3_or_1_3 | bool %} controllerConfig: + election: + lockName: openshift-master-controllers serviceServingCert: signer: certFile: service-signer.crt @@ -107,7 +106,7 @@ etcdConfig: clientCA: ca.crt {% endif %} keyFile: etcd.server.key - storageDirectory: {{ openshift.common.data_dir }}/openshift.local.etcd + storageDirectory: {{ r_openshift_master_data_dir }}/openshift.local.etcd {% endif %} etcdStorageConfig: kubernetesStoragePrefix: kubernetes.io @@ -164,24 +163,24 @@ masterClients: externalKubernetesClientConnectionOverrides: acceptContentTypes: application/vnd.kubernetes.protobuf,application/json contentType: application/vnd.kubernetes.protobuf - burst: 400 - qps: 200 + burst: {{ openshift_master_external_ratelimit_burst | default(400) }} + qps: {{ openshift_master_external_ratelimit_qps | default(200) }} {% endif %} externalKubernetesKubeConfig: "" {% if openshift.common.version_gte_3_3_or_1_3 | bool %} openshiftLoopbackClientConnectionOverrides: acceptContentTypes: application/vnd.kubernetes.protobuf,application/json contentType: application/vnd.kubernetes.protobuf - burst: 600 - qps: 300 + burst: {{ openshift_master_loopback_ratelimit_burst | default(600) }} + qps: {{ openshift_master_loopback_ratelimit_qps | default(300) }} {% endif %} openshiftLoopbackKubeConfig: openshift-master.kubeconfig masterPublicURL: {{ openshift.master.public_api_url }} networkConfig: clusterNetworkCIDR: {{ openshift.master.sdn_cluster_network_cidr }} hostSubnetLength: {{ openshift.master.sdn_host_subnet_length }} -{% if openshift.common.use_openshift_sdn or openshift.common.use_nuage or openshift.common.use_contiv or openshift.common.sdn_network_plugin_name == 'cni' %} - networkPluginName: {{ openshift.common.sdn_network_plugin_name }} +{% if r_openshift_master_use_openshift_sdn or r_openshift_master_use_nuage or r_openshift_master_use_contiv or r_openshift_master_sdn_network_plugin_name == 'cni' %} + networkPluginName: {{ r_openshift_master_sdn_network_plugin_name_default }} {% endif %} # serviceNetworkCIDR must match kubernetesMasterConfig.servicesSubnet serviceNetworkCIDR: {{ openshift.common.portal_net }} diff --git a/roles/openshift_master/templates/master_docker/master.docker.service.j2 b/roles/openshift_master/templates/master_docker/master.docker.service.j2 deleted file mode 100644 index 31c1dfc33..000000000 --- a/roles/openshift_master/templates/master_docker/master.docker.service.j2 +++ /dev/null @@ -1,18 +0,0 @@ -[Unit] -After={{ openshift.docker.service_name }}.service -Requires={{ openshift.docker.service_name }}.service -PartOf={{ openshift.docker.service_name }}.service -After=etcd_container.service -Wants=etcd_container.service - -[Service] -EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-master -ExecStartPre=-/usr/bin/docker rm -f {{ openshift.common.service_type }}-master -ExecStart=/usr/bin/docker run --rm --privileged --net=host --name {{ openshift.common.service_type }}-master --env-file=/etc/sysconfig/{{ openshift.common.service_type }}-master -v {{ openshift.common.data_dir }}:{{ openshift.common.data_dir }} -v /var/log:/var/log -v /var/run/docker.sock:/var/run/docker.sock -v {{ openshift.common.config_base }}:{{ openshift.common.config_base }} {% if openshift_cloudprovider_kind | default('') != '' -%} -v {{ openshift.common.config_base }}/cloudprovider:{{ openshift.common.config_base}}/cloudprovider {% endif -%} -v /etc/pki:/etc/pki:ro {{ openshift.master.master_image }}:${IMAGE_VERSION} start master --config=${CONFIG_FILE} $OPTIONS -ExecStartPost=/usr/bin/sleep 10 -ExecStop=/usr/bin/docker stop {{ openshift.common.service_type }}-master -Restart=always -RestartSec=5s - -[Install] -WantedBy={{ openshift.docker.service_name }}.service diff --git a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2 b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2 index c05a27559..63eb3ea1b 100644 --- a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2 +++ b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2 @@ -1,6 +1,9 @@ OPTIONS=--loglevel={{ openshift.master.debug_level }} --listen={{ 'https' if openshift.master.api_use_ssl else 'http' }}://{{ openshift.master.bind_addr }}:{{ openshift.master.api_port }} --master={{ openshift.master.loopback_api_url }} CONFIG_FILE={{ openshift_master_config_file }} -{% if openshift_push_via_dns | default(false) %} +{# Preserve existing OPENSHIFT_DEFAULT_REGISTRY settings in scale up runs #} +{% if openshift_master_is_scaleup_host %} +{{ openshift_master_default_registry_value_api }} +{% elif openshift_push_via_dns | default(false) %} OPENSHIFT_DEFAULT_REGISTRY=docker-registry.default.svc:5000 {% endif %} {% if openshift.common.is_containerized | bool %} diff --git a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.service.j2 b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.service.j2 index 0e78d2d23..02bfd6f62 100644 --- a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.service.j2 +++ b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.service.j2 @@ -13,7 +13,7 @@ Environment=GOTRACEBACK=crash ExecStart=/usr/bin/openshift start master api --config=${CONFIG_FILE} $OPTIONS LimitNOFILE=131072 LimitCORE=infinity -WorkingDirectory={{ openshift.common.data_dir }} +WorkingDirectory={{ r_openshift_master_data_dir }} SyslogIdentifier=atomic-openshift-master-api Restart=always RestartSec=5s diff --git a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2 b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2 index a153fb33d..0adfd05b6 100644 --- a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2 +++ b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2 @@ -1,13 +1,16 @@ OPTIONS=--loglevel={{ openshift.master.debug_level }} --listen={{ 'https' if openshift.master.api_use_ssl else 'http' }}://{{ openshift.master.bind_addr }}:{{ openshift.master.controllers_port }} CONFIG_FILE={{ openshift_master_config_file }} -{% if openshift_push_via_dns | default(false) %} +{# Preserve existing OPENSHIFT_DEFAULT_REGISTRY settings in scale up runs #} +{% if openshift_master_is_scaleup_host %} +{{ openshift_master_default_registry_value_controllers }} +{% elif openshift_push_via_dns | default(false) %} OPENSHIFT_DEFAULT_REGISTRY=docker-registry.default.svc:5000 {% endif %} {% if openshift.common.is_containerized | bool %} IMAGE_VERSION={{ openshift_image_tag }} {% endif %} -{% if openshift_cloudprovider_kind | default('') == 'aws' and openshift_cloudprovider_aws_access_key is defined and openshift_cloudprovider_aws_access_key is defined %} +{% if openshift_cloudprovider_kind | default('') == 'aws' and openshift_cloudprovider_aws_access_key is defined and openshift_cloudprovider_aws_secret_key is defined %} AWS_ACCESS_KEY_ID={{ openshift_cloudprovider_aws_access_key }} AWS_SECRET_ACCESS_KEY={{ openshift_cloudprovider_aws_secret_key }} {% endif %} diff --git a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.service.j2 b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.service.j2 index 94928f88c..e284413f7 100644 --- a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.service.j2 +++ b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.service.j2 @@ -17,7 +17,7 @@ Environment=GOTRACEBACK=crash ExecStart=/usr/bin/openshift start master controllers --config=${CONFIG_FILE} $OPTIONS LimitNOFILE=131072 LimitCORE=infinity -WorkingDirectory={{ openshift.common.data_dir }} +WorkingDirectory={{ r_openshift_master_data_dir }} SyslogIdentifier={{ openshift.common.service_type }}-master-controllers Restart=always RestartSec=5s diff --git a/roles/openshift_master/vars/main.yml b/roles/openshift_master/vars/main.yml index c5ba20409..0c681c764 100644 --- a/roles/openshift_master/vars/main.yml +++ b/roles/openshift_master/vars/main.yml @@ -19,4 +19,23 @@ openshift_master_valid_grant_methods: - prompt - deny -l_is_ha: "{{ openshift.master.ha is defined and openshift.master.ha | bool }}" +openshift_master_is_scaleup_host: False + +# These defaults assume forcing journald persistence, fsync to disk once +# a second, rate-limiting to 10,000 logs a second, no forwarding to +# syslog or wall, using 8GB of disk space maximum, using 10MB journal +# files, keeping only a days worth of logs per journal file, and +# retaining journal files no longer than a month. +journald_vars_to_replace: +- { var: Storage, val: persistent } +- { var: Compress, val: yes } +- { var: SyncIntervalSec, val: 1s } +- { var: RateLimitInterval, val: 1s } +- { var: RateLimitBurst, val: 10000 } +- { var: SystemMaxUse, val: 8G } +- { var: SystemKeepFree, val: 20% } +- { var: SystemMaxFileSize, val: 10M } +- { var: MaxRetentionSec, val: 1month } +- { var: MaxFileSec, val: 1day } +- { var: ForwardToSyslog, val: no } +- { var: ForwardToWall, val: no } |