From 17e38ad8c62ac8f1c8f983e63181f69879a45d85 Mon Sep 17 00:00:00 2001 From: Jiri Mencak Date: Tue, 3 Oct 2017 19:57:01 +0200 Subject: Separate tuned daemon setup into a role. Currently, profiles for the tuned daemon are set only for OpenShift node(s). This excludes the OpenShift loadbalancer. As a result, ARP cache limits on loadbalancers are not raised. This causes problems with HA setups where loadbalancers serve 1k+ OpenShift nodes. This commit ensures the openshift-control-plane role is applied to loadbalancers, masters and OpenShift infra nodes. Regular OpenShift worker nodes get the openshift-node profile. Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1498213 --- playbooks/common/openshift-loadbalancer/config.yml | 1 + playbooks/common/openshift-master/config.yml | 1 + .../common/openshift-node/configure_nodes.yml | 1 + roles/openshift_node/tasks/config.yml | 4 --- roles/openshift_node/tasks/tuned.yml | 41 ---------------------- .../tuned/openshift-control-plane/tuned.conf | 25 ------------- .../templates/tuned/openshift-node/tuned.conf | 10 ------ .../templates/tuned/openshift/tuned.conf | 24 ------------- .../openshift_node/templates/tuned/recommend.conf | 8 ----- roles/tuned/defaults/main.yml | 3 ++ roles/tuned/meta/main.yml | 13 +++++++ roles/tuned/tasks/main.yml | 39 ++++++++++++++++++++ .../templates/openshift-control-plane/tuned.conf | 25 +++++++++++++ roles/tuned/templates/openshift-node/tuned.conf | 10 ++++++ roles/tuned/templates/openshift/tuned.conf | 24 +++++++++++++ roles/tuned/templates/recommend.conf | 11 ++++++ 16 files changed, 128 insertions(+), 112 deletions(-) delete mode 100644 roles/openshift_node/tasks/tuned.yml delete mode 100644 roles/openshift_node/templates/tuned/openshift-control-plane/tuned.conf delete mode 100644 roles/openshift_node/templates/tuned/openshift-node/tuned.conf delete mode 100644 roles/openshift_node/templates/tuned/openshift/tuned.conf delete mode 100644 roles/openshift_node/templates/tuned/recommend.conf create mode 100644 roles/tuned/defaults/main.yml create mode 100644 roles/tuned/meta/main.yml create mode 100644 roles/tuned/tasks/main.yml create mode 100644 roles/tuned/templates/openshift-control-plane/tuned.conf create mode 100644 roles/tuned/templates/openshift-node/tuned.conf create mode 100644 roles/tuned/templates/openshift/tuned.conf create mode 100644 roles/tuned/templates/recommend.conf diff --git a/playbooks/common/openshift-loadbalancer/config.yml b/playbooks/common/openshift-loadbalancer/config.yml index ecbb092bc..9e148e636 100644 --- a/playbooks/common/openshift-loadbalancer/config.yml +++ b/playbooks/common/openshift-loadbalancer/config.yml @@ -27,6 +27,7 @@ roles: - role: os_firewall - role: openshift_loadbalancer + - role: tuned - name: Load Balancer Install Checkpoint End hosts: localhost diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml index bc1fee982..04e2bdbbf 100644 --- a/playbooks/common/openshift-master/config.yml +++ b/playbooks/common/openshift-master/config.yml @@ -198,6 +198,7 @@ openshift_master_default_registry_value: "{{ hostvars[groups.oo_first_master.0].l_default_registry_value }}" openshift_master_default_registry_value_api: "{{ hostvars[groups.oo_first_master.0].l_default_registry_value_api }}" openshift_master_default_registry_value_controllers: "{{ hostvars[groups.oo_first_master.0].l_default_registry_value_controllers }}" + - role: tuned - role: nuage_ca when: openshift_use_nuage | default(false) | bool - role: nuage_common diff --git a/playbooks/common/openshift-node/configure_nodes.yml b/playbooks/common/openshift-node/configure_nodes.yml index c96e4921c..17259422d 100644 --- a/playbooks/common/openshift-node/configure_nodes.yml +++ b/playbooks/common/openshift-node/configure_nodes.yml @@ -13,4 +13,5 @@ roles: - role: os_firewall - role: openshift_node + - role: tuned - role: nickhammond.logrotate diff --git a/roles/openshift_node/tasks/config.yml b/roles/openshift_node/tasks/config.yml index e3898b520..e5fcaf9af 100644 --- a/roles/openshift_node/tasks/config.yml +++ b/roles/openshift_node/tasks/config.yml @@ -111,9 +111,5 @@ msg: Node failed to start please inspect the logs and try again when: node_start_result | failed -- name: Setup tuned - include: tuned.yml - static: yes - - set_fact: node_service_status_changed: "{{ node_start_result | changed }}" diff --git a/roles/openshift_node/tasks/tuned.yml b/roles/openshift_node/tasks/tuned.yml deleted file mode 100644 index 425bf6a26..000000000 --- a/roles/openshift_node/tasks/tuned.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -- name: Check for tuned package - command: rpm -q tuned - args: - warn: no - register: tuned_installed - changed_when: false - failed_when: false - -- name: Tuned service setup - block: - - name: Set tuned OpenShift variables - set_fact: - openshift_tuned_guest_profile: "{{ 'atomic-guest' if openshift.common.is_atomic else 'virtual-guest' }}" - tuned_etc_directory: '/etc/tuned' - tuned_templates_source: '../templates/tuned' - - - name: Ensure directory structure exists - file: - state: directory - dest: '{{ tuned_etc_directory }}/{{ item.path }}' - with_filetree: '{{ tuned_templates_source }}' - when: item.state == 'directory' - - - name: Ensure files are populated from templates - template: - src: '{{ item.src }}' - dest: '{{ tuned_etc_directory }}/{{ item.path }}' - with_filetree: '{{ tuned_templates_source }}' - when: item.state == 'file' - - - name: Make tuned use the recommended tuned profile on restart - file: path=/etc/tuned/active_profile state=absent - - - name: Restart tuned service - systemd: - state: restarted - daemon_reload: yes - name: tuned - - when: tuned_installed.rc == 0 | bool diff --git a/roles/openshift_node/templates/tuned/openshift-control-plane/tuned.conf b/roles/openshift_node/templates/tuned/openshift-control-plane/tuned.conf deleted file mode 100644 index f22f21065..000000000 --- a/roles/openshift_node/templates/tuned/openshift-control-plane/tuned.conf +++ /dev/null @@ -1,25 +0,0 @@ -# -# tuned configuration -# - -[main] -summary=Optimize systems running OpenShift control plane -include=openshift - -[sysctl] -# ktune sysctl settings, maximizing i/o throughput -# -# Minimal preemption granularity for CPU-bound tasks: -# (default: 1 msec# (1 + ilog(ncpus)), units: nanoseconds) -kernel.sched_min_granularity_ns=10000000 - -# The total time the scheduler will consider a migrated process -# "cache hot" and thus less likely to be re-migrated -# (system default is 500000, i.e. 0.5 ms) -kernel.sched_migration_cost_ns=5000000 - -# SCHED_OTHER wake-up granularity. -# -# Preemption granularity when tasks wake up. Lower the value to improve -# wake-up latency and throughput for latency critical tasks. -kernel.sched_wakeup_granularity_ns = 4000000 diff --git a/roles/openshift_node/templates/tuned/openshift-node/tuned.conf b/roles/openshift_node/templates/tuned/openshift-node/tuned.conf deleted file mode 100644 index 78c7d19c9..000000000 --- a/roles/openshift_node/templates/tuned/openshift-node/tuned.conf +++ /dev/null @@ -1,10 +0,0 @@ -# -# tuned configuration -# - -[main] -summary=Optimize systems running OpenShift nodes -include=openshift - -[sysctl] -net.ipv4.tcp_fastopen=3 diff --git a/roles/openshift_node/templates/tuned/openshift/tuned.conf b/roles/openshift_node/templates/tuned/openshift/tuned.conf deleted file mode 100644 index 68ac5dadb..000000000 --- a/roles/openshift_node/templates/tuned/openshift/tuned.conf +++ /dev/null @@ -1,24 +0,0 @@ -# -# tuned configuration -# - -[main] -summary=Optimize systems running OpenShift (parent profile) -include=${f:virt_check:{{ openshift_tuned_guest_profile }}:throughput-performance} - -[selinux] -avc_cache_threshold=65536 - -[net] -nf_conntrack_hashsize=131072 - -[sysctl] -kernel.pid_max=131072 -net.netfilter.nf_conntrack_max=1048576 -fs.inotify.max_user_watches=65536 -net.ipv4.neigh.default.gc_thresh1=8192 -net.ipv4.neigh.default.gc_thresh2=32768 -net.ipv4.neigh.default.gc_thresh3=65536 -net.ipv6.neigh.default.gc_thresh1=8192 -net.ipv6.neigh.default.gc_thresh2=32768 -net.ipv6.neigh.default.gc_thresh3=65536 diff --git a/roles/openshift_node/templates/tuned/recommend.conf b/roles/openshift_node/templates/tuned/recommend.conf deleted file mode 100644 index 5fa765798..000000000 --- a/roles/openshift_node/templates/tuned/recommend.conf +++ /dev/null @@ -1,8 +0,0 @@ -[openshift-node] -/etc/origin/node/node-config.yaml=.*region=primary - -[openshift-control-plane,master] -/etc/origin/master/master-config.yaml=.* - -[openshift-control-plane,node] -/etc/origin/node/node-config.yaml=.*region=infra diff --git a/roles/tuned/defaults/main.yml b/roles/tuned/defaults/main.yml new file mode 100644 index 000000000..418a4b521 --- /dev/null +++ b/roles/tuned/defaults/main.yml @@ -0,0 +1,3 @@ +--- +tuned_etc_directory: '/etc/tuned' +tuned_templates_source: '../templates' diff --git a/roles/tuned/meta/main.yml b/roles/tuned/meta/main.yml new file mode 100644 index 000000000..833d94c13 --- /dev/null +++ b/roles/tuned/meta/main.yml @@ -0,0 +1,13 @@ +--- +galaxy_info: + author: Jiri Mencak + description: Restart the tuned daemon if present and make it use the recommended profile + company: Red Hat, Inc. + license: Apache License, Version 2.0 + min_ansible_version: 2.3 + platforms: + - name: EL + versions: + - 7 + categories: + - cloud diff --git a/roles/tuned/tasks/main.yml b/roles/tuned/tasks/main.yml new file mode 100644 index 000000000..e95d274d5 --- /dev/null +++ b/roles/tuned/tasks/main.yml @@ -0,0 +1,39 @@ +--- +- name: Check for tuned package + command: rpm -q tuned + args: + warn: no + register: tuned_installed + changed_when: false + failed_when: false + +- name: Tuned service setup + block: + - name: Set tuned OpenShift variables + set_fact: + openshift_tuned_guest_profile: "{{ 'atomic-guest' if openshift.common.is_atomic else 'virtual-guest' }}" + + - name: Ensure directory structure exists + file: + state: directory + dest: '{{ tuned_etc_directory }}/{{ item.path }}' + with_filetree: '{{ tuned_templates_source }}' + when: item.state == 'directory' + + - name: Ensure files are populated from templates + template: + src: '{{ item.src }}' + dest: '{{ tuned_etc_directory }}/{{ item.path }}' + with_filetree: '{{ tuned_templates_source }}' + when: item.state == 'file' + + - name: Make tuned use the recommended tuned profile on restart + file: path=/etc/tuned/active_profile state=absent + + - name: Restart tuned service + systemd: + state: restarted + daemon_reload: yes + name: tuned + + when: tuned_installed.rc == 0 | bool diff --git a/roles/tuned/templates/openshift-control-plane/tuned.conf b/roles/tuned/templates/openshift-control-plane/tuned.conf new file mode 100644 index 000000000..f22f21065 --- /dev/null +++ b/roles/tuned/templates/openshift-control-plane/tuned.conf @@ -0,0 +1,25 @@ +# +# tuned configuration +# + +[main] +summary=Optimize systems running OpenShift control plane +include=openshift + +[sysctl] +# ktune sysctl settings, maximizing i/o throughput +# +# Minimal preemption granularity for CPU-bound tasks: +# (default: 1 msec# (1 + ilog(ncpus)), units: nanoseconds) +kernel.sched_min_granularity_ns=10000000 + +# The total time the scheduler will consider a migrated process +# "cache hot" and thus less likely to be re-migrated +# (system default is 500000, i.e. 0.5 ms) +kernel.sched_migration_cost_ns=5000000 + +# SCHED_OTHER wake-up granularity. +# +# Preemption granularity when tasks wake up. Lower the value to improve +# wake-up latency and throughput for latency critical tasks. +kernel.sched_wakeup_granularity_ns = 4000000 diff --git a/roles/tuned/templates/openshift-node/tuned.conf b/roles/tuned/templates/openshift-node/tuned.conf new file mode 100644 index 000000000..78c7d19c9 --- /dev/null +++ b/roles/tuned/templates/openshift-node/tuned.conf @@ -0,0 +1,10 @@ +# +# tuned configuration +# + +[main] +summary=Optimize systems running OpenShift nodes +include=openshift + +[sysctl] +net.ipv4.tcp_fastopen=3 diff --git a/roles/tuned/templates/openshift/tuned.conf b/roles/tuned/templates/openshift/tuned.conf new file mode 100644 index 000000000..68ac5dadb --- /dev/null +++ b/roles/tuned/templates/openshift/tuned.conf @@ -0,0 +1,24 @@ +# +# tuned configuration +# + +[main] +summary=Optimize systems running OpenShift (parent profile) +include=${f:virt_check:{{ openshift_tuned_guest_profile }}:throughput-performance} + +[selinux] +avc_cache_threshold=65536 + +[net] +nf_conntrack_hashsize=131072 + +[sysctl] +kernel.pid_max=131072 +net.netfilter.nf_conntrack_max=1048576 +fs.inotify.max_user_watches=65536 +net.ipv4.neigh.default.gc_thresh1=8192 +net.ipv4.neigh.default.gc_thresh2=32768 +net.ipv4.neigh.default.gc_thresh3=65536 +net.ipv6.neigh.default.gc_thresh1=8192 +net.ipv6.neigh.default.gc_thresh2=32768 +net.ipv6.neigh.default.gc_thresh3=65536 diff --git a/roles/tuned/templates/recommend.conf b/roles/tuned/templates/recommend.conf new file mode 100644 index 000000000..086e5673d --- /dev/null +++ b/roles/tuned/templates/recommend.conf @@ -0,0 +1,11 @@ +[openshift-control-plane,master] +/etc/origin/master/master-config.yaml=.* + +[openshift-control-plane,node] +/etc/origin/node/node-config.yaml=.*region=infra + +[openshift-control-plane,lb] +/etc/haproxy/haproxy.cfg=.* + +[openshift-node] +/etc/origin/node/node-config.yaml=.* -- cgit v1.2.3