summaryrefslogtreecommitdiffstats
path: root/roles/openshift_logging_elasticsearch
diff options
context:
space:
mode:
authorMatt Bruzek <mbruzek@gmail.com>2018-01-18 15:27:13 -0600
committerMatt Bruzek <mbruzek@gmail.com>2018-01-18 15:27:13 -0600
commitcb581bfb67a53f887c4705d45fc7b0024a6816f9 (patch)
tree9c351ddd9282f5d3d37c1189af0ac2ad444c0125 /roles/openshift_logging_elasticsearch
parentc7a1c448cbd64de98e1f097d14b58ee9f6ccf511 (diff)
parent1a2a895356df638756d2117e3d324710167737db (diff)
downloadopenshift-cb581bfb67a53f887c4705d45fc7b0024a6816f9.tar.gz
openshift-cb581bfb67a53f887c4705d45fc7b0024a6816f9.tar.bz2
openshift-cb581bfb67a53f887c4705d45fc7b0024a6816f9.tar.xz
openshift-cb581bfb67a53f887c4705d45fc7b0024a6816f9.zip
Merge branch 'master' into mbruzek-openshift-openstack
Diffstat (limited to 'roles/openshift_logging_elasticsearch')
-rw-r--r--roles/openshift_logging_elasticsearch/meta/main.yaml1
-rw-r--r--roles/openshift_logging_elasticsearch/tasks/determine_version.yaml2
-rw-r--r--roles/openshift_logging_elasticsearch/tasks/get_es_version.yml42
-rw-r--r--roles/openshift_logging_elasticsearch/tasks/main.yaml54
-rw-r--r--roles/openshift_logging_elasticsearch/tasks/restart_cluster.yml62
-rw-r--r--roles/openshift_logging_elasticsearch/tasks/restart_es_node.yml2
-rw-r--r--roles/openshift_logging_elasticsearch/templates/es.j22
-rw-r--r--roles/openshift_logging_elasticsearch/vars/main.yml6
8 files changed, 144 insertions, 27 deletions
diff --git a/roles/openshift_logging_elasticsearch/meta/main.yaml b/roles/openshift_logging_elasticsearch/meta/main.yaml
index 6a9a6539c..e93d6b73e 100644
--- a/roles/openshift_logging_elasticsearch/meta/main.yaml
+++ b/roles/openshift_logging_elasticsearch/meta/main.yaml
@@ -14,3 +14,4 @@ galaxy_info:
dependencies:
- role: lib_openshift
- role: openshift_facts
+- role: lib_utils
diff --git a/roles/openshift_logging_elasticsearch/tasks/determine_version.yaml b/roles/openshift_logging_elasticsearch/tasks/determine_version.yaml
index c53a06019..c55e7c5ea 100644
--- a/roles/openshift_logging_elasticsearch/tasks/determine_version.yaml
+++ b/roles/openshift_logging_elasticsearch/tasks/determine_version.yaml
@@ -15,3 +15,5 @@
- fail:
msg: Invalid version specified for Elasticsearch
when: es_version not in __allowed_es_versions
+
+- include_tasks: get_es_version.yml
diff --git a/roles/openshift_logging_elasticsearch/tasks/get_es_version.yml b/roles/openshift_logging_elasticsearch/tasks/get_es_version.yml
new file mode 100644
index 000000000..16de6f252
--- /dev/null
+++ b/roles/openshift_logging_elasticsearch/tasks/get_es_version.yml
@@ -0,0 +1,42 @@
+---
+- command: >
+ oc get pod -l component=es,provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[?(@.status.phase==\"Running\")].metadata.name}
+ register: _cluster_pods
+
+- name: "Getting ES version for logging-es cluster"
+ command: >
+ oc exec {{ _cluster_pods.stdout.split(' ')[0] }} -c elasticsearch -n {{ openshift_logging_elasticsearch_namespace }} -- {{ __es_local_curl }} -XGET 'https://localhost:9200/'
+ register: _curl_output
+ when: _cluster_pods.stdout_lines | count > 0
+
+- command: >
+ oc get pod -l component=es-ops,provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[?(@.status.phase==\"Running\")].metadata.name}
+ register: _ops_cluster_pods
+
+- name: "Getting ES version for logging-es-ops cluster"
+ command: >
+ oc exec {{ _ops_cluster_pods.stdout.split(' ')[0] }} -c elasticsearch -n {{ openshift_logging_elasticsearch_namespace }} -- {{ __es_local_curl }} -XGET 'https://localhost:9200/'
+ register: _ops_curl_output
+ when: _ops_cluster_pods.stdout_lines | count > 0
+
+- set_fact:
+ _es_output: "{{ _curl_output.stdout | from_json }}"
+ when: _curl_output.stdout is defined
+
+- set_fact:
+ _es_ops_output: "{{ _ops_curl_output.stdout | from_json }}"
+ when: _ops_curl_output.stdout is defined
+
+- set_fact:
+ _es_installed_version: "{{ _es_output.version.number }}"
+ when:
+ - _es_output is defined
+ - _es_output.version is defined
+ - _es_output.version.number is defined
+
+- set_fact:
+ _es_ops_installed_version: "{{ _es_ops_output.version.number }}"
+ when:
+ - _es_ops_output is defined
+ - _es_ops_output.version is defined
+ - _es_ops_output.version.number is defined
diff --git a/roles/openshift_logging_elasticsearch/tasks/main.yaml b/roles/openshift_logging_elasticsearch/tasks/main.yaml
index 5fe683ae5..ff5ad1045 100644
--- a/roles/openshift_logging_elasticsearch/tasks/main.yaml
+++ b/roles/openshift_logging_elasticsearch/tasks/main.yaml
@@ -15,10 +15,10 @@
elasticsearch_name: "{{ 'logging-elasticsearch' ~ ( (openshift_logging_elasticsearch_ops_deployment | default(false) | bool) | ternary('-ops', '')) }}"
es_component: "{{ 'es' ~ ( (openshift_logging_elasticsearch_ops_deployment | default(false) | bool) | ternary('-ops', '') ) }}"
-- name: Set default image variables based on deployment_type
+- name: Set default image variables based on openshift_deployment_type
include_vars: "{{ var_file_name }}"
with_first_found:
- - "{{ openshift_deployment_type | default(deployment_type) }}.yml"
+ - "{{ openshift_deployment_type }}.yml"
- "default_images.yml"
loop_control:
loop_var: var_file_name
@@ -32,6 +32,18 @@
- include_tasks: determine_version.yaml
+- set_fact:
+ full_restart_cluster: True
+ when:
+ - _es_installed_version is defined
+ - _es_installed_version.split('.')[0] | int < __es_version.split('.')[0] | int
+
+- set_fact:
+ full_restart_cluster: True
+ when:
+ - _es_ops_installed_version is defined
+ - _es_ops_installed_version.split('.')[0] | int < __es_version.split('.')[0] | int
+
# allow passing in a tempdir
- name: Create temp directory for doing work in
command: mktemp -d /tmp/openshift-logging-ansible-XXXXXX
@@ -111,7 +123,7 @@
- name: Create logging-metrics-reader-role
command: >
- {{ openshift.common.client_binary }}
+ {{ openshift_client_binary }}
--config={{ openshift.common.config_base }}/master/admin.kubeconfig
-n "{{ openshift_logging_elasticsearch_namespace }}"
create -f "{{mktemp.stdout}}/templates/logging-metrics-role.yml"
@@ -168,33 +180,33 @@
when: es_logging_contents is undefined
changed_when: no
-- set_fact:
- __es_num_of_shards: "{{ _es_configmap | default({}) | walk('index.number_of_shards', '1') }}"
- __es_num_of_replicas: "{{ _es_configmap | default({}) | walk('index.number_of_replicas', '0') }}"
-
- template:
src: elasticsearch.yml.j2
dest: "{{ tempdir }}/elasticsearch.yml"
vars:
allow_cluster_reader: "{{ openshift_logging_elasticsearch_ops_allow_cluster_reader | lower | default('false') }}"
- es_number_of_shards: "{{ openshift_logging_es_number_of_shards | default(None) or __es_num_of_shards }}"
- es_number_of_replicas: "{{ openshift_logging_es_number_of_replicas | default(None) or __es_num_of_replicas }}"
+ es_number_of_shards: "{{ openshift_logging_es_number_of_shards | default(1) }}"
+ es_number_of_replicas: "{{ openshift_logging_es_number_of_replicas| default(0) }}"
es_kibana_index_mode: "{{ openshift_logging_elasticsearch_kibana_index_mode | default('unique') }}"
when: es_config_contents is undefined
changed_when: no
-- copy:
- content: "{{ es_logging_contents }}"
- dest: "{{ tempdir }}/elasticsearch-logging.yml"
- when: es_logging_contents is defined
- changed_when: no
-
-- copy:
- content: "{{ es_config_contents }}"
- dest: "{{ tempdir }}/elasticsearch.yml"
- when: es_config_contents is defined
- changed_when: no
+# create diff between current configmap files and our current files
+# NOTE: include_role must be used instead of import_role because
+# this task file is looped over from another role.
+- include_role:
+ name: openshift_logging
+ tasks_from: patch_configmap_files.yaml
+ vars:
+ configmap_name: "logging-elasticsearch"
+ configmap_namespace: "logging"
+ configmap_file_names:
+ - current_file: "elasticsearch.yml"
+ new_file: "{{ tempdir }}/elasticsearch.yml"
+ protected_lines: ["number_of_shards", "number_of_replicas"]
+ - current_file: "logging.yml"
+ new_file: "{{ tempdir }}/elasticsearch-logging.yml"
- name: Set ES configmap
oc_configmap:
@@ -352,7 +364,7 @@
delete_after: true
- set_fact:
- es_deploy_name: "logging-{{ es_component }}-{{ openshift_logging_elasticsearch_deployment_type }}-{{ 8 | oo_random_word('abcdefghijklmnopqrstuvwxyz0123456789') }}"
+ es_deploy_name: "logging-{{ es_component }}-{{ openshift_logging_elasticsearch_deployment_type }}-{{ 8 | lib_utils_oo_random_word('abcdefghijklmnopqrstuvwxyz0123456789') }}"
when: openshift_logging_elasticsearch_deployment_name == ""
- set_fact:
diff --git a/roles/openshift_logging_elasticsearch/tasks/restart_cluster.yml b/roles/openshift_logging_elasticsearch/tasks/restart_cluster.yml
index 4a32453e3..6bce13d1d 100644
--- a/roles/openshift_logging_elasticsearch/tasks/restart_cluster.yml
+++ b/roles/openshift_logging_elasticsearch/tasks/restart_cluster.yml
@@ -1,7 +1,25 @@
---
+# Disable external communication for {{ _cluster_component }}
+- name: Disable external communication for logging-{{ _cluster_component }}
+ oc_service:
+ state: present
+ name: "logging-{{ _cluster_component }}"
+ namespace: "{{ openshift_logging_elasticsearch_namespace }}"
+ selector:
+ component: "{{ _cluster_component }}"
+ provider: openshift
+ connection: blocked
+ labels:
+ logging-infra: 'support'
+ ports:
+ - port: 9200
+ targetPort: "restapi"
+ when:
+ - full_restart_cluster | bool
+
## get all pods for the cluster
- command: >
- oc get pod -l component={{ _cluster_component }},provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[*].metadata.name}
+ oc get pod -l component={{ _cluster_component }},provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[?(@.status.phase==\"Running\")].metadata.name}
register: _cluster_pods
- name: "Disable shard balancing for logging-{{ _cluster_component }} cluster"
@@ -11,21 +29,42 @@
changed_when: "'\"acknowledged\":true' in _disable_output.stdout"
when: _cluster_pods.stdout_lines | count > 0
+# Flush ES
+- name: "Flushing for logging-{{ _cluster_component }} cluster"
+ command: >
+ oc exec {{ _cluster_pods.stdout.split(' ')[0] }} -c elasticsearch -n {{ openshift_logging_elasticsearch_namespace }} -- {{ __es_local_curl }} -XPUT 'https://localhost:9200/_flush/synced'
+ register: _flush_output
+ changed_when: "'\"acknowledged\":true' in _flush_output.stdout"
+ when:
+ - _cluster_pods.stdout_lines | count > 0
+ - full_restart_cluster | bool
+
- command: >
oc get dc -l component={{ _cluster_component }},provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[*].metadata.name}
register: _cluster_dcs
+## restart all dcs for full restart
+- name: "Restart ES node {{ _es_node }}"
+ include_tasks: restart_es_node.yml
+ with_items: "{{ _cluster_dcs }}"
+ loop_control:
+ loop_var: _es_node
+ when:
+ - full_restart_cluster | bool
+
## restart the node if it's dc is in the list of nodes to restart?
- name: "Restart ES node {{ _es_node }}"
include_tasks: restart_es_node.yml
with_items: "{{ _restart_logging_nodes }}"
loop_control:
loop_var: _es_node
- when: _es_node in _cluster_dcs.stdout
+ when:
+ - not full_restart_cluster | bool
+ - _es_node in _cluster_dcs.stdout
## we may need a new first pod to run against -- fetch them all again
- command: >
- oc get pod -l component={{ _cluster_component }},provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[*].metadata.name}
+ oc get pod -l component={{ _cluster_component }},provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[?(@.status.phase==\"Running\")].metadata.name}
register: _cluster_pods
- name: "Enable shard balancing for logging-{{ _cluster_component }} cluster"
@@ -33,3 +72,20 @@
oc exec {{ _cluster_pods.stdout.split(' ')[0] }} -c elasticsearch -n {{ openshift_logging_elasticsearch_namespace }} -- {{ __es_local_curl }} -XPUT 'https://localhost:9200/_cluster/settings' -d '{ "transient": { "cluster.routing.allocation.enable" : "all" } }'
register: _enable_output
changed_when: "'\"acknowledged\":true' in _enable_output.stdout"
+
+# Reenable external communication for {{ _cluster_component }}
+- name: Reenable external communication for logging-{{ _cluster_component }}
+ oc_service:
+ state: present
+ name: "logging-{{ _cluster_component }}"
+ namespace: "{{ openshift_logging_elasticsearch_namespace }}"
+ selector:
+ component: "{{ _cluster_component }}"
+ provider: openshift
+ labels:
+ logging-infra: 'support'
+ ports:
+ - port: 9200
+ targetPort: "restapi"
+ when:
+ - full_restart_cluster | bool
diff --git a/roles/openshift_logging_elasticsearch/tasks/restart_es_node.yml b/roles/openshift_logging_elasticsearch/tasks/restart_es_node.yml
index b07b232ce..6d0df40c8 100644
--- a/roles/openshift_logging_elasticsearch/tasks/restart_es_node.yml
+++ b/roles/openshift_logging_elasticsearch/tasks/restart_es_node.yml
@@ -14,6 +14,8 @@
- _dc_output.results.results[0].status is defined
- _dc_output.results.results[0].status.readyReplicas is defined
- _dc_output.results.results[0].status.readyReplicas > 0
+ - _dc_output.results.results[0].status.updatedReplicas is defined
+ - _dc_output.results.results[0].status.updatedReplicas > 0
retries: 60
delay: 30
diff --git a/roles/openshift_logging_elasticsearch/templates/es.j2 b/roles/openshift_logging_elasticsearch/templates/es.j2
index cf6ee36bb..4b189f255 100644
--- a/roles/openshift_logging_elasticsearch/templates/es.j2
+++ b/roles/openshift_logging_elasticsearch/templates/es.j2
@@ -50,7 +50,7 @@ spec:
- -provider=openshift
- -client-id={{openshift_logging_elasticsearch_prometheus_sa}}
- -client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token
- - -cookie-secret={{ 16 | oo_random_word | b64encode }}
+ - -cookie-secret={{ 16 | lib_utils_oo_random_word | b64encode }}
- -upstream=https://localhost:9200
- '-openshift-sar={"namespace": "{{ openshift_logging_elasticsearch_namespace}}", "verb": "view", "resource": "prometheus", "group": "metrics.openshift.io"}'
- '-openshift-delegate-urls={"/": {"resource": "prometheus", "verb": "view", "group": "metrics.openshift.io", "namespace": "{{ openshift_logging_elasticsearch_namespace}}"}}'
diff --git a/roles/openshift_logging_elasticsearch/vars/main.yml b/roles/openshift_logging_elasticsearch/vars/main.yml
index c8e995146..122231031 100644
--- a/roles/openshift_logging_elasticsearch/vars/main.yml
+++ b/roles/openshift_logging_elasticsearch/vars/main.yml
@@ -1,9 +1,10 @@
---
-__latest_es_version: "3_6"
-__allowed_es_versions: ["3_5", "3_6", "3_7"]
+__latest_es_version: "3_9"
+__allowed_es_versions: ["3_5", "3_6", "3_7", "3_8", "3_9"]
__allowed_es_types: ["data-master", "data-client", "master", "client"]
__es_log_appenders: ['file', 'console']
__kibana_index_modes: ["unique", "shared_ops"]
+__es_version: "2.4.4"
__es_local_curl: "curl -s --cacert /etc/elasticsearch/secret/admin-ca --cert /etc/elasticsearch/secret/admin-cert --key /etc/elasticsearch/secret/admin-key"
@@ -14,3 +15,4 @@ es_min_masters_default: "{{ (openshift_logging_elasticsearch_replica_count | int
es_min_masters: "{{ (openshift_logging_elasticsearch_replica_count == 1) | ternary(1, es_min_masters_default) }}"
es_recover_after_nodes: "{{ openshift_logging_elasticsearch_replica_count | int }}"
es_recover_expected_nodes: "{{ openshift_logging_elasticsearch_replica_count | int }}"
+full_restart_cluster: False