From 49d8b7e373f00dfcce2940cdd22d4d60471f84c4 Mon Sep 17 00:00:00 2001 From: Noriko Hosoi Date: Wed, 24 May 2017 09:50:07 -0700 Subject: Introducing fluentd/mux buffer_queue_limit, buffer_size_limit, cpu_limit, and memory_limit. Default values: openshift_logging_fluentd_cpu_limit: 100m openshift_logging_fluentd_memory_limit: 512Mi openshift_logging_fluentd_buffer_queue_limit: 1024 openshift_logging_fluentd_buffer_size_limit: 16m openshift_logging_fluentd_buffer_size_limit: 1m openshift_logging_mux_cpu_limit: 500m openshift_logging_mux_memory_limit: 1Gi openshift_logging_mux_buffer_queue_limit: 1024 openshift_logging_mux_buffer_size_limit: 1m --- roles/openshift_logging/defaults/main.yml | 6 ++++-- roles/openshift_logging_fluentd/templates/fluentd.j2 | 4 ++++ roles/openshift_logging_mux/defaults/main.yml | 6 ++++-- roles/openshift_logging_mux/templates/mux.j2 | 4 ++++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/roles/openshift_logging/defaults/main.yml b/roles/openshift_logging/defaults/main.yml index 573cbdd09..d7c83d8e8 100644 --- a/roles/openshift_logging/defaults/main.yml +++ b/roles/openshift_logging/defaults/main.yml @@ -69,13 +69,15 @@ openshift_logging_kibana_ops_key: "" openshift_logging_kibana_ops_ca: "" openshift_logging_fluentd_nodeselector: "{{ openshift_hosted_logging_fluentd_nodeselector_label | default('logging-infra-fluentd=true') | map_from_pairs }}" -openshift_logging_fluentd_cpu_limit: 100m -openshift_logging_fluentd_memory_limit: 512Mi +openshift_logging_fluentd_cpu_limit: "{{ openshift_hosted_logging_fluentd_cpu_limit | default('100m') }}" +openshift_logging_fluentd_memory_limit: "{{ openshift_hosted_logging_fluentd_memory_limit | default('512Mi') }}" openshift_logging_fluentd_es_copy: false openshift_logging_fluentd_use_journal: "{{ openshift_hosted_logging_use_journal if openshift_hosted_logging_use_journal is defined else (docker_log_driver == 'journald') | ternary(True, False) if docker_log_driver is defined else (openshift.docker.log_driver == 'journald') | ternary(True, False) if openshift.docker.log_driver is defined else openshift.docker.options | search('--log-driver=journald') if openshift.docker.options is defined else default(omit) }}" openshift_logging_fluentd_journal_source: "{{ openshift_hosted_logging_journal_source | default('') }}" openshift_logging_fluentd_journal_read_from_head: "{{ openshift_hosted_logging_journal_read_from_head | default('') }}" openshift_logging_fluentd_hosts: ['--all'] +openshift_logging_fluentd_buffer_queue_limit: "{{ openshift_hosted_logging_fluentd_buffer_queue_limit | default('1024') }}" +openshift_logging_fluentd_buffer_size_limit: "{{ openshift_hosted_logging_fluentd_buffer_size_limit | default('1m') }}" openshift_logging_es_host: logging-es openshift_logging_es_port: 9200 diff --git a/roles/openshift_logging_fluentd/templates/fluentd.j2 b/roles/openshift_logging_fluentd/templates/fluentd.j2 index e185938e3..bfaef938f 100644 --- a/roles/openshift_logging_fluentd/templates/fluentd.j2 +++ b/roles/openshift_logging_fluentd/templates/fluentd.j2 @@ -93,6 +93,10 @@ spec: value: "{{ openshift_logging_fluentd_journal_source | default('') }}" - name: "JOURNAL_READ_FROM_HEAD" value: "{{ openshift_logging_fluentd_journal_read_from_head | lower }}" + - name: "BUFFER_QUEUE_LIMIT" + value: "{{ openshift_logging_fluentd_buffer_queue_limit }}" + - name: "BUFFER_SIZE_LIMIT" + value: "{{ openshift_logging_fluentd_buffer_size_limit }}" volumes: - name: runlogjournal hostPath: diff --git a/roles/openshift_logging_mux/defaults/main.yml b/roles/openshift_logging_mux/defaults/main.yml index 10fa4372c..3c706b1ca 100644 --- a/roles/openshift_logging_mux/defaults/main.yml +++ b/roles/openshift_logging_mux/defaults/main.yml @@ -9,8 +9,10 @@ openshift_logging_mux_namespace: logging ### Common settings openshift_logging_mux_nodeselector: "{{ openshift_hosted_logging_mux_nodeselector_label | default('') | map_from_pairs }}" -openshift_logging_mux_cpu_limit: 500m -openshift_logging_mux_memory_limit: 1Gi +openshift_logging_mux_cpu_limit: "{{ openshift_hosted_logging_mux_cpu_limit | default('500m') }}" +openshift_logging_mux_memory_limit: "{{ openshift_hosted_logging_mux_memory_limit | default('1Gi') }}" +openshift_logging_mux_buffer_queue_limit: "{{ openshift_hosted_logging_mux_buffer_queue_limit | default('1024') }}" +openshift_logging_mux_buffer_size_limit: "{{ openshift_hosted_logging_mux_buffer_size_limit | default('1m') }}" openshift_logging_mux_replicas: 1 diff --git a/roles/openshift_logging_mux/templates/mux.j2 b/roles/openshift_logging_mux/templates/mux.j2 index 770a2bfbd..e51cfcbc6 100644 --- a/roles/openshift_logging_mux/templates/mux.j2 +++ b/roles/openshift_logging_mux/templates/mux.j2 @@ -103,6 +103,10 @@ spec: value: "true" - name: MUX_ALLOW_EXTERNAL value: "{{ openshift_logging_mux_allow_external | default('false') }}" + - name: "BUFFER_QUEUE_LIMIT" + value: "{{ openshift_logging_mux_buffer_queue_limit }}" + - name: "BUFFER_SIZE_LIMIT" + value: "{{ openshift_logging_mux_buffer_size_limit }}" volumes: - name: config configMap: -- cgit v1.2.3 From 33f3a95da29ba2441911e761fbf8a5a2a553b9b9 Mon Sep 17 00:00:00 2001 From: Noriko Hosoi Date: Fri, 2 Jun 2017 17:06:30 -0700 Subject: Adding environment variables FLUENTD_CPU_LIMIT, FLUENTD_MEMORY_LIMIT, MUX_CPU_LIMIT, MUX_MEMORY_LIMIT. --- roles/openshift_logging_fluentd/templates/fluentd.j2 | 4 ++++ roles/openshift_logging_mux/templates/mux.j2 | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/roles/openshift_logging_fluentd/templates/fluentd.j2 b/roles/openshift_logging_fluentd/templates/fluentd.j2 index bfaef938f..a5695ee26 100644 --- a/roles/openshift_logging_fluentd/templates/fluentd.j2 +++ b/roles/openshift_logging_fluentd/templates/fluentd.j2 @@ -97,6 +97,10 @@ spec: value: "{{ openshift_logging_fluentd_buffer_queue_limit }}" - name: "BUFFER_SIZE_LIMIT" value: "{{ openshift_logging_fluentd_buffer_size_limit }}" + - name: "FLUENTD_CPU_LIMIT" + value: "{{ openshift_logging_fluentd_cpu_limit }}" + - name: "FLUENTD_MEMORY_LIMIT" + value: "{{ openshift_logging_fluentd_memory_limit }}" volumes: - name: runlogjournal hostPath: diff --git a/roles/openshift_logging_mux/templates/mux.j2 b/roles/openshift_logging_mux/templates/mux.j2 index e51cfcbc6..a00f7c939 100644 --- a/roles/openshift_logging_mux/templates/mux.j2 +++ b/roles/openshift_logging_mux/templates/mux.j2 @@ -107,6 +107,10 @@ spec: value: "{{ openshift_logging_mux_buffer_queue_limit }}" - name: "BUFFER_SIZE_LIMIT" value: "{{ openshift_logging_mux_buffer_size_limit }}" + - name: "MUX_CPU_LIMIT" + value: "{{ openshift_logging_mux_cpu_limit }}" + - name: "MUX_MEMORY_LIMIT" + value: "{{ openshift_logging_mux_memory_limit }}" volumes: - name: config configMap: -- cgit v1.2.3 From e1b3fe7618186f84136675e24e6c3ec7877ea944 Mon Sep 17 00:00:00 2001 From: Noriko Hosoi Date: Sun, 4 Jun 2017 20:07:23 -0700 Subject: Adding the defaults for openshift_logging_fluentd_{cpu,memory}_limit to roles/openshift_logging_fluentd/defaults/main.yml. Adding the description for openshift_hosted_logging_fluentd_{cpu,memory}_limit to README.md. Replaced openshift_hosted_logging_mux_.* with openshift_logging_mux_.*. --- roles/openshift_hosted_logging/README.md | 2 ++ roles/openshift_logging_fluentd/defaults/main.yml | 4 ++-- roles/openshift_logging_mux/defaults/main.yml | 8 ++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/roles/openshift_hosted_logging/README.md b/roles/openshift_hosted_logging/README.md index 12ffe777d..ad1e9d92b 100644 --- a/roles/openshift_hosted_logging/README.md +++ b/roles/openshift_hosted_logging/README.md @@ -25,6 +25,8 @@ - openshift_hosted_logging_use_journal: If this is unset or empty, logging will try to figure out from docker which log driver it is using (json-file or journald). You can set this param to "true" or "false" to force logging to use journal or not (but make sure you are sure which one docker is using). - openshift_hosted_logging_journal_source: By default, if this param is unset or empty, logging will use `/var/log/journal` if it exists, or `/run/log/journal` if not. You can use this param to force logging to use a different location. - openshift_hosted_logging_journal_read_from_head: Set to `true` to have fluentd read from the beginning of the journal, to get historical log data. Default is `false`. *WARNING* Using `true` may take several minutes or even hours, depending on the size of the journal, until any new records show up in Elasticsearch, and will cause fluentd to consume a lot of CPU and RAM resources. +- openshift_hosted_logging_fluentd_cpu_limit: The CPU limit for Fluentd pods. Defaults to "100m". +- openshift_hosted_logging_fluentd_memory_limit: The memory limit for Fluentd pods. Defaults to '512Mi'. When `openshift_hosted_logging_enable_ops_cluster` is `True`, there are some additional vars. These work the same as above for their non-ops counterparts, diff --git a/roles/openshift_logging_fluentd/defaults/main.yml b/roles/openshift_logging_fluentd/defaults/main.yml index 228196d74..41481f87e 100644 --- a/roles/openshift_logging_fluentd/defaults/main.yml +++ b/roles/openshift_logging_fluentd/defaults/main.yml @@ -8,8 +8,8 @@ openshift_logging_fluentd_namespace: logging ### Common settings openshift_logging_fluentd_nodeselector: "{{ openshift_hosted_logging_fluentd_nodeselector_label | default('logging-infra-fluentd=true') | map_from_pairs }}" -openshift_logging_fluentd_cpu_limit: 100m -openshift_logging_fluentd_memory_limit: 512Mi +openshift_logging_fluentd_cpu_limit: "{{ openshift_hosted_logging_fluentd_cpu_limit | default('100m') }}" +openshift_logging_fluentd_memory_limit: "{{ openshift_hosted_logging_fluentd_memory_limit | default('512Mi') }}" openshift_logging_fluentd_hosts: ['--all'] # float time in seconds to wait between node labelling diff --git a/roles/openshift_logging_mux/defaults/main.yml b/roles/openshift_logging_mux/defaults/main.yml index 3c706b1ca..52d1c5320 100644 --- a/roles/openshift_logging_mux/defaults/main.yml +++ b/roles/openshift_logging_mux/defaults/main.yml @@ -9,10 +9,10 @@ openshift_logging_mux_namespace: logging ### Common settings openshift_logging_mux_nodeselector: "{{ openshift_hosted_logging_mux_nodeselector_label | default('') | map_from_pairs }}" -openshift_logging_mux_cpu_limit: "{{ openshift_hosted_logging_mux_cpu_limit | default('500m') }}" -openshift_logging_mux_memory_limit: "{{ openshift_hosted_logging_mux_memory_limit | default('1Gi') }}" -openshift_logging_mux_buffer_queue_limit: "{{ openshift_hosted_logging_mux_buffer_queue_limit | default('1024') }}" -openshift_logging_mux_buffer_size_limit: "{{ openshift_hosted_logging_mux_buffer_size_limit | default('1m') }}" +openshift_logging_mux_cpu_limit: "{{ openshift_logging_mux_cpu_limit | default('500m') }}" +openshift_logging_mux_memory_limit: "{{ openshift_logging_mux_memory_limit | default('1Gi') }}" +openshift_logging_mux_buffer_queue_limit: "{{ openshift_logging_mux_buffer_queue_limit | default('1024') }}" +openshift_logging_mux_buffer_size_limit: "{{ openshift_logging_mux_buffer_size_limit | default('1m') }}" openshift_logging_mux_replicas: 1 -- cgit v1.2.3 From 1bbac1e2e1b3ecc0b5ff4a43cfcdd770c3060a51 Mon Sep 17 00:00:00 2001 From: Noriko Hosoi Date: Mon, 5 Jun 2017 13:46:33 -0700 Subject: Reverted most of this pr 4356 except: adding openshift_logging_fluentd_buffer_queue_limit: 1024 openshift_logging_fluentd_buffer_size_limit: 1m openshift_logging_mux_buffer_queue_limit: 1024 openshift_logging_mux_buffer_size_limit: 1m and setting the matched environment variables. --- roles/openshift_hosted_logging/README.md | 2 -- roles/openshift_logging/README.md | 5 +++++ roles/openshift_logging/defaults/main.yml | 8 ++++---- roles/openshift_logging_fluentd/defaults/main.yml | 4 ++-- roles/openshift_logging_mux/defaults/main.yml | 8 ++++---- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/roles/openshift_hosted_logging/README.md b/roles/openshift_hosted_logging/README.md index ad1e9d92b..12ffe777d 100644 --- a/roles/openshift_hosted_logging/README.md +++ b/roles/openshift_hosted_logging/README.md @@ -25,8 +25,6 @@ - openshift_hosted_logging_use_journal: If this is unset or empty, logging will try to figure out from docker which log driver it is using (json-file or journald). You can set this param to "true" or "false" to force logging to use journal or not (but make sure you are sure which one docker is using). - openshift_hosted_logging_journal_source: By default, if this param is unset or empty, logging will use `/var/log/journal` if it exists, or `/run/log/journal` if not. You can use this param to force logging to use a different location. - openshift_hosted_logging_journal_read_from_head: Set to `true` to have fluentd read from the beginning of the journal, to get historical log data. Default is `false`. *WARNING* Using `true` may take several minutes or even hours, depending on the size of the journal, until any new records show up in Elasticsearch, and will cause fluentd to consume a lot of CPU and RAM resources. -- openshift_hosted_logging_fluentd_cpu_limit: The CPU limit for Fluentd pods. Defaults to "100m". -- openshift_hosted_logging_fluentd_memory_limit: The memory limit for Fluentd pods. Defaults to '512Mi'. When `openshift_hosted_logging_enable_ops_cluster` is `True`, there are some additional vars. These work the same as above for their non-ops counterparts, diff --git a/roles/openshift_logging/README.md b/roles/openshift_logging/README.md index 0c60ef6fd..dd0f22d4b 100644 --- a/roles/openshift_logging/README.md +++ b/roles/openshift_logging/README.md @@ -55,6 +55,9 @@ When both `openshift_logging_install_logging` and `openshift_logging_upgrade_log - `openshift_logging_fluentd_use_journal`: NOTE: Fluentd will attempt to detect whether or not Docker is using the journald log driver when using the default of empty. - `openshift_logging_fluentd_journal_read_from_head`: If empty, Fluentd will use its internal default, which is false. - `openshift_logging_fluentd_hosts`: List of nodes that should be labeled for Fluentd to be deployed to. Defaults to ['--all']. +- `openshift_logging_fluentd_buffer_queue_limit`: Buffer queue limit for Fluentd. Defaults to 1024. +- `openshift_logging_fluentd_buffer_size_limit`: Buffer chunk limit for Fluentd. Defaults to 1m. + - `openshift_logging_es_host`: The name of the ES service Fluentd should send logs to. Defaults to 'logging-es'. - `openshift_logging_es_port`: The port for the ES service Fluentd should sent its logs to. Defaults to '9200'. @@ -155,3 +158,5 @@ Elasticsearch OPS too, if using an OPS cluster: - `openshift_logging_mux_namespaces`: Default `[]` - additional namespaces to create for _external_ mux clients to associate with their logs - users will need to set this +- `openshift_logging_mux_buffer_queue_limit`: Default `[1024]` - Buffer queue limit for Mux. +- `openshift_logging_mux_buffer_size_limit`: Default `[1m]` - Buffer chunk limit for Mux. diff --git a/roles/openshift_logging/defaults/main.yml b/roles/openshift_logging/defaults/main.yml index d7c83d8e8..95a86354e 100644 --- a/roles/openshift_logging/defaults/main.yml +++ b/roles/openshift_logging/defaults/main.yml @@ -69,15 +69,15 @@ openshift_logging_kibana_ops_key: "" openshift_logging_kibana_ops_ca: "" openshift_logging_fluentd_nodeselector: "{{ openshift_hosted_logging_fluentd_nodeselector_label | default('logging-infra-fluentd=true') | map_from_pairs }}" -openshift_logging_fluentd_cpu_limit: "{{ openshift_hosted_logging_fluentd_cpu_limit | default('100m') }}" -openshift_logging_fluentd_memory_limit: "{{ openshift_hosted_logging_fluentd_memory_limit | default('512Mi') }}" +openshift_logging_fluentd_cpu_limit: 100m +openshift_logging_fluentd_memory_limit: 512Mi openshift_logging_fluentd_es_copy: false openshift_logging_fluentd_use_journal: "{{ openshift_hosted_logging_use_journal if openshift_hosted_logging_use_journal is defined else (docker_log_driver == 'journald') | ternary(True, False) if docker_log_driver is defined else (openshift.docker.log_driver == 'journald') | ternary(True, False) if openshift.docker.log_driver is defined else openshift.docker.options | search('--log-driver=journald') if openshift.docker.options is defined else default(omit) }}" openshift_logging_fluentd_journal_source: "{{ openshift_hosted_logging_journal_source | default('') }}" openshift_logging_fluentd_journal_read_from_head: "{{ openshift_hosted_logging_journal_read_from_head | default('') }}" openshift_logging_fluentd_hosts: ['--all'] -openshift_logging_fluentd_buffer_queue_limit: "{{ openshift_hosted_logging_fluentd_buffer_queue_limit | default('1024') }}" -openshift_logging_fluentd_buffer_size_limit: "{{ openshift_hosted_logging_fluentd_buffer_size_limit | default('1m') }}" +openshift_logging_fluentd_buffer_queue_limit: 1024 +openshift_logging_fluentd_buffer_size_limit: 1m openshift_logging_es_host: logging-es openshift_logging_es_port: 9200 diff --git a/roles/openshift_logging_fluentd/defaults/main.yml b/roles/openshift_logging_fluentd/defaults/main.yml index 41481f87e..228196d74 100644 --- a/roles/openshift_logging_fluentd/defaults/main.yml +++ b/roles/openshift_logging_fluentd/defaults/main.yml @@ -8,8 +8,8 @@ openshift_logging_fluentd_namespace: logging ### Common settings openshift_logging_fluentd_nodeselector: "{{ openshift_hosted_logging_fluentd_nodeselector_label | default('logging-infra-fluentd=true') | map_from_pairs }}" -openshift_logging_fluentd_cpu_limit: "{{ openshift_hosted_logging_fluentd_cpu_limit | default('100m') }}" -openshift_logging_fluentd_memory_limit: "{{ openshift_hosted_logging_fluentd_memory_limit | default('512Mi') }}" +openshift_logging_fluentd_cpu_limit: 100m +openshift_logging_fluentd_memory_limit: 512Mi openshift_logging_fluentd_hosts: ['--all'] # float time in seconds to wait between node labelling diff --git a/roles/openshift_logging_mux/defaults/main.yml b/roles/openshift_logging_mux/defaults/main.yml index 52d1c5320..77e47d38c 100644 --- a/roles/openshift_logging_mux/defaults/main.yml +++ b/roles/openshift_logging_mux/defaults/main.yml @@ -9,10 +9,10 @@ openshift_logging_mux_namespace: logging ### Common settings openshift_logging_mux_nodeselector: "{{ openshift_hosted_logging_mux_nodeselector_label | default('') | map_from_pairs }}" -openshift_logging_mux_cpu_limit: "{{ openshift_logging_mux_cpu_limit | default('500m') }}" -openshift_logging_mux_memory_limit: "{{ openshift_logging_mux_memory_limit | default('1Gi') }}" -openshift_logging_mux_buffer_queue_limit: "{{ openshift_logging_mux_buffer_queue_limit | default('1024') }}" -openshift_logging_mux_buffer_size_limit: "{{ openshift_logging_mux_buffer_size_limit | default('1m') }}" +openshift_logging_mux_cpu_limit: 500m +openshift_logging_mux_memory_limit: 2Gi +openshift_logging_mux_buffer_queue_limit: 1024 +openshift_logging_mux_buffer_size_limit: 1m openshift_logging_mux_replicas: 1 -- cgit v1.2.3