diff options
Diffstat (limited to 'roles/openshift_logging')
| -rw-r--r-- | roles/openshift_logging/README.md | 77 | ||||
| -rw-r--r-- | roles/openshift_logging/filter_plugins/openshift_logging.py | 27 | ||||
| -rw-r--r-- | roles/openshift_logging/filter_plugins/test | 15 | ||||
| -rw-r--r-- | roles/openshift_logging/handlers/main.yml | 11 | ||||
| -rw-r--r-- | roles/openshift_logging/tasks/install_logging.yaml | 41 | ||||
| -rw-r--r-- | roles/openshift_logging/tasks/main.yaml | 11 | ||||
| -rw-r--r-- | roles/openshift_logging/templates/jks_pod.j2 | 2 | ||||
| -rw-r--r-- | roles/openshift_logging/vars/default_images.yml | 3 | ||||
| -rw-r--r-- | roles/openshift_logging/vars/openshift-enterprise.yml | 3 |
9 files changed, 155 insertions, 35 deletions
diff --git a/roles/openshift_logging/README.md b/roles/openshift_logging/README.md index 280d7d24c..6c5bb8693 100644 --- a/roles/openshift_logging/README.md +++ b/roles/openshift_logging/README.md @@ -225,3 +225,80 @@ The corresponding openshift\_logging\_mux\_* parameters are below. - `openshift_logging_mux_remote_syslog_tag_key`: If string specified, use this field from the record to set the key field on the syslog message - `openshift_logging_mux_remote_syslog_use_record`: Set `true` to use the severity and facility from the record, defaults to `false` - `openshift_logging_mux_remote_syslog_payload_key`: If string is specified, use this field from the record as the payload on the syslog message + +Image update procedure +---------------------- +An upgrade of the logging stack from older version to newer is an automated process and should be performed by calling appropriate ansible playbook and setting required ansible variables in your inventory as documented in https://docs.openshift.org/. + +Following text describes manual update of the logging images without version upgrade. To determine the current version of images being used you can. +``` +oc describe pod | grep 'Image ID:' +``` +This will get the repo digest that can later be compared to the inspected image details. + +A way to determine when was your image last updated: +``` +$ docker images +REPOSITORY TAG IMAGE ID CREATED SIZE +<registry>/openshift3/logging-fluentd v3.7 ff2e249fc45a About an hour ago 235.2 MB + +$ docker inspect ff2e249fc45a +[ + { + . . . + "RepoDigests": [ + "<registry>/openshift3/logging-fluentd@sha256:4346f0aa9694f32735115705ad324803b1a6ff08343c3288f7a62c3a5cb70495" + ], + . . . + "Config": { + . . . + "Labels": { + . . . + "build-date": "2017-10-12T14:38:22.414827", + . . . + "release": "0.143.3.0", + . . . + "url": "https://access.redhat.com/containers/#/registry.access.redhat.com/openshift3/logging-fluentd/images/v3.7.0-0.143.3.0", + . . . + "version": "v3.7.0" + } + }, + . . . +``` + +Pull a new image to see if registry has any newer images with the same tag: +``` +$ docker pull <registry>/openshift3/logging-fluentd:v3.7 +``` + +If there was an update, you need to run the `docker pull` on each node. + +It is recommended that you now rerun the `openshift_logging` playbook to ensure that any necessary config changes are also picked up. + +To manually redeploy your pod you can do the following: +- for a DC you can do: +``` +oc rollout latest <dc_name> +``` + +- for a RC you can scale down and scale back up +``` +oc scale --replicas=0 <rc_name> + +... wait for scale down + +oc scale --replicas=<original_replica_count> <rc_name> +``` + +- for a DS you can delete the pod or unlabel and relabel your node +``` +oc delete pod --selector=<ds_selector> +``` + +Changelog +--------- +Tue Oct 26, 2017 +- Make CPU request equal limit if limit is greater then request + +Tue Oct 10, 2017 +- Default imagePullPolicy changed from Always to IfNotPresent diff --git a/roles/openshift_logging/filter_plugins/openshift_logging.py b/roles/openshift_logging/filter_plugins/openshift_logging.py index 959573635..e1a5ea726 100644 --- a/roles/openshift_logging/filter_plugins/openshift_logging.py +++ b/roles/openshift_logging/filter_plugins/openshift_logging.py @@ -3,6 +3,7 @@ ''' import random +import re def es_storage(os_logging_facts, dc_name, pvc_claim, root='elasticsearch'): @@ -17,6 +18,31 @@ def es_storage(os_logging_facts, dc_name, pvc_claim, root='elasticsearch'): return dict(kind='emptydir') +def min_cpu(left, right): + '''Return the minimum cpu value of the two values given''' + message = "Unable to evaluate {} cpu value is specified correctly '{}'. Exp whole, decimal or int followed by M" + pattern = re.compile(r"^(\d*\.?\d*)([Mm])?$") + millis_per_core = 1000 + if not right: + return left + m_left = pattern.match(left) + if not m_left: + raise RuntimeError(message.format("left", left)) + m_right = pattern.match(right) + if not m_right: + raise RuntimeError(message.format("right", right)) + left_value = float(m_left.group(1)) + right_value = float(m_right.group(1)) + if m_left.group(2) not in ["M", "m"]: + left_value = left_value * millis_per_core + if m_right.group(2) not in ["M", "m"]: + right_value = right_value * millis_per_core + response = left + if left_value != min(left_value, right_value): + response = right + return response + + def walk(source, path, default, delimiter='.'): '''Walk the sourch hash given the path and return the value or default if not found''' if not isinstance(source, dict): @@ -87,6 +113,7 @@ class FilterModule(object): 'random_word': random_word, 'entry_from_named_pair': entry_from_named_pair, 'map_from_pairs': map_from_pairs, + 'min_cpu': min_cpu, 'es_storage': es_storage, 'serviceaccount_name': serviceaccount_name, 'serviceaccount_namespace': serviceaccount_namespace, diff --git a/roles/openshift_logging/filter_plugins/test b/roles/openshift_logging/filter_plugins/test index 3ad956cca..bac25c012 100644 --- a/roles/openshift_logging/filter_plugins/test +++ b/roles/openshift_logging/filter_plugins/test @@ -1,7 +1,22 @@ import unittest from openshift_logging import walk +from openshift_logging import min_cpu class TestFilterMethods(unittest.TestCase): + + + def test_min_cpu_for_none(self): + source = "1000M" + self.assertEquals(min_cpu(source, None), "1000M") + + def test_min_cpu_for_millis(self): + source = "1" + self.assertEquals(min_cpu(source, "0.1"), "0.1") + + + def test_min_cpu_for_whole(self): + source = "120M" + self.assertEquals(min_cpu(source, "2"), "120M") def test_walk_find_key(self): diff --git a/roles/openshift_logging/handlers/main.yml b/roles/openshift_logging/handlers/main.yml index ce7688581..074b72942 100644 --- a/roles/openshift_logging/handlers/main.yml +++ b/roles/openshift_logging/handlers/main.yml @@ -4,8 +4,13 @@ when: (not (master_api_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native' notify: Verify API Server +# We retry the controllers because the API may not be 100% initialized yet. - name: restart master controllers - systemd: name={{ openshift.common.service_type }}-master-controllers state=restarted + command: "systemctl restart {{ openshift.common.service_type }}-master-controllers" + retries: 3 + delay: 5 + register: result + until: result.rc == 0 when: (not (master_controllers_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native' - name: Verify API Server @@ -13,11 +18,7 @@ # wait_for port doesn't provide health information. command: > curl --silent --tlsv1.2 - {% if openshift.common.version_gte_3_2_or_1_2 | bool %} --cacert {{ openshift.common.config_base }}/master/ca-bundle.crt - {% else %} - --cacert {{ openshift.common.config_base }}/master/ca.crt - {% endif %} {{ openshift.master.api_url }}/healthz/ready args: # Disables the following warning: diff --git a/roles/openshift_logging/tasks/install_logging.yaml b/roles/openshift_logging/tasks/install_logging.yaml index 76627acf2..cec295d65 100644 --- a/roles/openshift_logging/tasks/install_logging.yaml +++ b/roles/openshift_logging/tasks/install_logging.yaml @@ -36,6 +36,14 @@ - openshift_logging_label_key != "" - openshift_logging_label_value is defined +- name: Annotate Logging Project to allow overcommit + oc_edit: + kind: ns + name: "{{ openshift_logging_namespace }}" + separator: '#' + content: + metadata#annotations#quota.openshift.io/cluster-resource-override-enabled: "false" + - name: Create logging cert directory file: path: "{{ openshift.common.config_base }}/logging" @@ -69,21 +77,23 @@ vars: generated_certs_dir: "{{openshift.common.config_base}}/logging" openshift_logging_elasticsearch_namespace: "{{ openshift_logging_namespace }}" - openshift_logging_elasticsearch_deployment_name: "{{ item.0.name }}" - openshift_logging_elasticsearch_pvc_name: "{{ openshift_logging_es_pvc_prefix ~ '-' ~ item.2 if item.1 is none else item.1 }}" + openshift_logging_elasticsearch_deployment_name: "{{ outer_item.0.name }}" + openshift_logging_elasticsearch_pvc_name: "{{ outer_item.0.volumes['elasticsearch-storage'].persistentVolumeClaim.claimName if outer_item.0.volumes['elasticsearch-storage'].persistentVolumeClaim is defined else openshift_logging_es_pvc_prefix ~ '-' ~ outer_item.2 if outer_item.1 is none else outer_item.1 }}" openshift_logging_elasticsearch_replica_count: "{{ openshift_logging_es_cluster_size | int }}" openshift_logging_elasticsearch_storage_type: "{{ elasticsearch_storage_type }}" openshift_logging_elasticsearch_pvc_pv_selector: "{{ openshift_logging_es_pv_selector }}" - openshift_logging_elasticsearch_nodeselector: "{{ openshift_logging_es_nodeselector if item.0.nodeSelector | default(None) is none else item.0.nodeSelector }}" - openshift_logging_elasticsearch_storage_group: "{{ [openshift_logging_es_storage_group] if item.0.storageGroups | default([]) | length == 0 else item.0.storageGroups }}" - _es_containers: "{{item.0.containers}}" + openshift_logging_elasticsearch_nodeselector: "{{ openshift_logging_es_nodeselector if outer_item.0.nodeSelector | default(None) is none else outer_item.0.nodeSelector }}" + openshift_logging_elasticsearch_storage_group: "{{ [openshift_logging_es_storage_group] if outer_item.0.storageGroups | default([]) | length == 0 else outer_item.0.storageGroups }}" + _es_containers: "{{ outer_item.0.containers}}" _es_configmap: "{{ openshift_logging_facts | walk('elasticsearch#configmaps#logging-elasticsearch#elasticsearch.yml', '{}', delimiter='#') | from_yaml }}" with_together: - "{{ openshift_logging_facts.elasticsearch.deploymentconfigs.values() }}" - "{{ openshift_logging_facts.elasticsearch.pvcs }}" - "{{ es_indices }}" + loop_control: + loop_var: outer_item when: - openshift_logging_facts.elasticsearch.deploymentconfigs.keys() | count > 0 @@ -93,13 +103,15 @@ vars: generated_certs_dir: "{{openshift.common.config_base}}/logging" openshift_logging_elasticsearch_namespace: "{{ openshift_logging_namespace }}" - openshift_logging_elasticsearch_pvc_name: "{{ openshift_logging_es_pvc_prefix }}-{{ item | int + openshift_logging_facts.elasticsearch.deploymentconfigs | count - 1 }}" + openshift_logging_elasticsearch_pvc_name: "{{ openshift_logging_es_pvc_prefix }}-{{ outer_item | int + openshift_logging_facts.elasticsearch.deploymentconfigs | count - 1 }}" openshift_logging_elasticsearch_replica_count: "{{ openshift_logging_es_cluster_size | int }}" openshift_logging_elasticsearch_storage_type: "{{ elasticsearch_storage_type }}" openshift_logging_elasticsearch_pvc_pv_selector: "{{ openshift_logging_es_pv_selector }}" with_sequence: count={{ openshift_logging_es_cluster_size | int - openshift_logging_facts.elasticsearch.deploymentconfigs.keys() | count }} + loop_control: + loop_var: outer_item - set_fact: es_ops_indices={{ es_ops_indices | default([]) + [item | int - 1] }} with_sequence: count={{ openshift_logging_facts.elasticsearch_ops.deploymentconfigs.keys() | count }} @@ -123,8 +135,8 @@ vars: generated_certs_dir: "{{openshift.common.config_base}}/logging" openshift_logging_elasticsearch_namespace: "{{ openshift_logging_namespace }}" - openshift_logging_elasticsearch_deployment_name: "{{ item.0.name }}" - openshift_logging_elasticsearch_pvc_name: "{{ openshift_logging_es_ops_pvc_prefix ~ '-' ~ item.2 if item.1 is none else item.1 }}" + openshift_logging_elasticsearch_deployment_name: "{{ outer_item.0.name }}" + openshift_logging_elasticsearch_pvc_name: "{{ outer_item.0.volumes['elasticsearch-storage'].persistentVolumeClaim.claimName if outer_item.0.volumes['elasticsearch-storage'].persistentVolumeClaim is defined else openshift_logging_es_ops_pvc_prefix ~ '-' ~ outer_item.2 if outer_item.1 is none else outer_item.1 }}" openshift_logging_elasticsearch_ops_deployment: true openshift_logging_elasticsearch_replica_count: "{{ openshift_logging_es_ops_cluster_size | int }}" @@ -135,8 +147,8 @@ openshift_logging_elasticsearch_memory_limit: "{{ openshift_logging_es_ops_memory_limit }}" openshift_logging_elasticsearch_cpu_limit: "{{ openshift_logging_es_ops_cpu_limit }}" openshift_logging_elasticsearch_cpu_request: "{{ openshift_logging_es_ops_cpu_request }}" - openshift_logging_elasticsearch_nodeselector: "{{ openshift_logging_es_ops_nodeselector if item.0.nodeSelector | default(None) is none else item.0.nodeSelector }}" - openshift_logging_elasticsearch_storage_group: "{{ [openshift_logging_es_ops_storage_group] if item.0.storageGroups | default([]) | length == 0 else item.0.storageGroups }}" + openshift_logging_elasticsearch_nodeselector: "{{ openshift_logging_es_ops_nodeselector if outer_item.0.nodeSelector | default(None) is none else outer_item.0.nodeSelector }}" + openshift_logging_elasticsearch_storage_group: "{{ [openshift_logging_es_ops_storage_group] if outer_item.0.storageGroups | default([]) | length == 0 else outer_item.0.storageGroups }}" openshift_logging_es_key: "{{ openshift_logging_es_ops_key }}" openshift_logging_es_cert: "{{ openshift_logging_es_ops_cert }}" openshift_logging_es_ca_ext: "{{ openshift_logging_es_ops_ca_ext }}" @@ -145,13 +157,16 @@ openshift_logging_es_allow_external: "{{ openshift_logging_es_ops_allow_external }}" openshift_logging_es_number_of_shards: "{{ openshift_logging_es_ops_number_of_shards | default(None) }}" openshift_logging_es_number_of_replicas: "{{ openshift_logging_es_ops_number_of_replicas | default(None) }}" - _es_containers: "{{item.0.containers}}" + _es_containers: "{{ outer_item.0.containers}}" _es_configmap: "{{ openshift_logging_facts | walk('elasticsearch_ops#configmaps#logging-elasticsearch-ops#elasticsearch.yml', '{}', delimiter='#') | from_yaml }}" with_together: - "{{ openshift_logging_facts.elasticsearch_ops.deploymentconfigs.values() }}" - "{{ openshift_logging_facts.elasticsearch_ops.pvcs }}" - "{{ es_ops_indices }}" + loop_control: + loop_var: outer_item + when: - openshift_logging_use_ops | bool - openshift_logging_facts.elasticsearch_ops.deploymentconfigs.keys() | count > 0 @@ -162,7 +177,7 @@ vars: generated_certs_dir: "{{openshift.common.config_base}}/logging" openshift_logging_elasticsearch_namespace: "{{ openshift_logging_namespace }}" - openshift_logging_elasticsearch_pvc_name: "{{ openshift_logging_es_ops_pvc_prefix }}-{{ item | int + openshift_logging_facts.elasticsearch_ops.deploymentconfigs | count - 1 }}" + openshift_logging_elasticsearch_pvc_name: "{{ openshift_logging_es_ops_pvc_prefix }}-{{ outer_item | int + openshift_logging_facts.elasticsearch_ops.deploymentconfigs | count - 1 }}" openshift_logging_elasticsearch_ops_deployment: true openshift_logging_elasticsearch_replica_count: "{{ openshift_logging_es_ops_cluster_size | int }}" @@ -182,6 +197,8 @@ openshift_logging_es_allow_external: "{{ openshift_logging_es_ops_allow_external }}" with_sequence: count={{ openshift_logging_es_ops_cluster_size | int - openshift_logging_facts.elasticsearch_ops.deploymentconfigs.keys() | count }} + loop_control: + loop_var: outer_item when: - openshift_logging_use_ops | bool diff --git a/roles/openshift_logging/tasks/main.yaml b/roles/openshift_logging/tasks/main.yaml index 15f6a23e6..7f8e88036 100644 --- a/roles/openshift_logging/tasks/main.yaml +++ b/roles/openshift_logging/tasks/main.yaml @@ -3,17 +3,6 @@ msg: Only one Fluentd nodeselector key pair should be provided when: openshift_logging_fluentd_nodeselector.keys() | count > 1 -- name: Set default image variables based on deployment_type - include_vars: "{{ item }}" - with_first_found: - - "{{ openshift_deployment_type | default(deployment_type) }}.yml" - - "default_images.yml" - -- name: Set logging image facts - set_fact: - openshift_logging_image_prefix: "{{ openshift_logging_image_prefix | default(__openshift_logging_image_prefix) }}" - openshift_logging_image_version: "{{ openshift_logging_image_version | default(__openshift_logging_image_version) }}" - - name: Create temp directory for doing work in command: mktemp -d /tmp/openshift-logging-ansible-XXXXXX register: mktemp diff --git a/roles/openshift_logging/templates/jks_pod.j2 b/roles/openshift_logging/templates/jks_pod.j2 index 8b1c74211..e4110b7b3 100644 --- a/roles/openshift_logging/templates/jks_pod.j2 +++ b/roles/openshift_logging/templates/jks_pod.j2 @@ -8,7 +8,7 @@ spec: containers: - name: jks-cert-gen image: {{openshift_logging_image_prefix}}logging-deployer:{{openshift_logging_image_version}} - imagePullPolicy: Always + imagePullPolicy: IfNotPresent command: ["sh", "{{generated_certs_dir}}/generate-jks.sh"] securityContext: privileged: true diff --git a/roles/openshift_logging/vars/default_images.yml b/roles/openshift_logging/vars/default_images.yml deleted file mode 100644 index 1a77808f6..000000000 --- a/roles/openshift_logging/vars/default_images.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -__openshift_logging_image_prefix: "{{ openshift_hosted_logging_deployer_prefix | default('docker.io/openshift/origin-') }}" -__openshift_logging_image_version: "{{ openshift_hosted_logging_deployer_version | default('latest') }}" diff --git a/roles/openshift_logging/vars/openshift-enterprise.yml b/roles/openshift_logging/vars/openshift-enterprise.yml deleted file mode 100644 index f60fa8d7d..000000000 --- a/roles/openshift_logging/vars/openshift-enterprise.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -__openshift_logging_image_prefix: "{{ openshift_hosted_logging_deployer_prefix | default('registry.access.redhat.com/openshift3/') }}" -__openshift_logging_image_version: "{{ openshift_hosted_logging_deployer_version | default ('v3.7') }}" |