initial support for v1.3 with logging v1.3

This also includes some fixes to make deploying logging
more idempotent, such as ignoring failures if trying to
create objects that already exist

3 files changed, 122 insertions, 33 deletions

diff --git a/roles/openshift_hosted_logging/README.md b/roles/openshift_hosted_logging/README.md
index b3f363571..03db1c4b1 100644
--- a/roles/openshift_hosted_logging/README.md
+++ b/roles/openshift_hosted_logging/README.md
@@ -5,6 +5,35 @@
 - openshift_hosted_logging_master_public_url: https://localhost:8443
 
 ###Optional vars:
+- openshift_hosted_logging_image_prefix: logging image prefix.  No default.  Use this to specify an alternate image repository e.g. my.private.repo:5000/private_openshift/
+- target_registry: DEPRECATED - use openshift_hosted_logging_image_prefix instead
+- openshift_hosted_logging_image_version: logging image version suffix.  Defaults to the current version of the deployed software.
 - openshift_hosted_logging_secret_vars: (defaults to nothing=/dev/null) kibana.crt=/etc/origin/master/ca.crt kibana.key=/etc/origin/master/ca.key ca.crt=/etc/origin/master/ca.crt ca.key=/etc/origin/master/ca.key
 - openshift_hosted_logging_fluentd_replicas: (defaults to 1) 3
 - openshift_hosted_logging_cleanup: (defaults to no) Set this to 'yes' in order to cleanup logging components instead of deploying.
+- openshift_hosted_logging_elasticsearch_instance_ram: Amount of RAM to reserve per ElasticSearch instance (e.g. 1024M, 2G). Defaults to 8GiB; must be at least 512M (Ref.: [ElasticSearch documentation](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html\#\_memory).
+- openshift_hosted_logging_elasticsearch_pvc_size: Size of the PersistentVolumeClaim to create per ElasticSearch ops instance, e.g. 100G. If empty, no PVCs will be created and emptyDir volumes are used instead.
+- openshift_hosted_logging_elasticsearch_pvc_prefix: Prefix for the names of PersistentVolumeClaims to be created; a number will be appended per instance. If they don't already exist, they will be created with size `openshift_hosted_logging_elasticsearch_pvc_size`.
+- openshift_hosted_logging_elasticsearch_pvc_dynamic: Set to `true` to have created PersistentVolumeClaims annotated such that their backing storage can be dynamically provisioned (if that is available for your cluster).
+- openshift_hosted_logging_elasticsearch_storage_group: Number of a supplemental group ID for access to Elasticsearch storage volumes; backing volumes should allow access by this group ID (defaults to 65534).
+- openshift_hosted_logging_elasticsearch_nodeselector: Specify the nodeSelector that Elasticsearch should be use (label=value)
+- openshift_hosted_logging_fluentd_nodeselector: The nodeSelector to use for the Fluentd DaemonSet. Defaults to "logging-infra-fluentd=true".
+- openshift_hosted_logging_kibana_nodeselector: Specify the nodeSelector that Kibana should be use (label=value)
+- openshift_hosted_logging_curator_nodeselector: Specify the nodeSelector that Curator should be use (label=value)
+- openshift_hosted_logging_enable_ops_cluster: If "true", configure a second ES cluster and Kibana for ops logs.
+- openshift_hosted_logging_use_journal: If this is unset or empty, logging will try to figure out from docker which log driver it is using (json-file or journald).  You can set this param to "true" or "false" to force logging to use journal or not (but make sure you are sure which one docker is using).
+- openshift_hosted_logging_journal_source: By default, if this param is unset or empty, logging will use `/var/log/journal` if it exists, or `/run/log/journal` if not.  You can use this param to force logging to use a different location.
+- openshift_hosted_logging_journal_read_from_head: Set to `true` to have fluentd read from the beginning of the journal, to get historical log data.  Default is `false`.  *WARNING* Using `true` may take several minutes or even hours, depending on the size of the journal, until any new records show up in Elasticsearch, and will cause fluentd to consume a lot of CPU and RAM resources.
+
+When `openshift_hosted_logging_enable_ops_cluster` is `True`, there are some
+additional vars.  These work the same as above for their non-ops counterparts,
+but apply to the OPS cluster instance:
+- openshift_hosted_logging_ops_hostname: kibana-ops.example.com
+- openshift_hosted_logging_elasticsearch_ops_cluster_size
+- openshift_hosted_logging_elasticsearch_ops_instance_ram
+- openshift_hosted_logging_elasticsearch_ops_pvc_size
+- openshift_hosted_logging_elasticsearch_ops_pvc_prefix
+- openshift_hosted_logging_elasticsearch_ops_pvc_dynamic
+- openshift_hosted_logging_elasticsearch_ops_nodeselector
+- openshift_hosted_logging_kibana_ops_nodeselector
+- openshift_hosted_logging_curator_ops_nodeselector
diff --git a/roles/openshift_hosted_logging/tasks/deploy_logging.yaml b/roles/openshift_hosted_logging/tasks/deploy_logging.yaml
index f19ce3e2b..082bb6ea2 100644
--- a/roles/openshift_hosted_logging/tasks/deploy_logging.yaml
+++ b/roles/openshift_hosted_logging/tasks/deploy_logging.yaml
@@ -1,4 +1,7 @@
 ---
+  - debug: msg="WARNING target_registry is deprecated, use openshift_hosted_logging_image_prefix instead"
+    when: target_registry is defined and target_registry
+
   - fail: msg="This role requires the following vars to be defined. openshift_hosted_logging_master_public_url, openshift_hosted_logging_hostname, openshift_hosted_logging_elasticsearch_cluster_size"
     when: "openshift_hosted_logging_hostname is not defined or
           openshift_hosted_logging_elasticsearch_cluster_size is not defined or
@@ -35,21 +38,20 @@
     register: secret_output
     failed_when: "secret_output.rc == 1 and 'exists' not in secret_output.stderr"
 
-  - name: "Copy serviceAccount file"
-    copy:
-      dest: /tmp/logging-deployer-sa.yaml
-      src: "{{role_path}}/files/logging-deployer-sa.yaml"
-      force: yes
-
-  - name: "Create logging-deployer service account"
+  - name: "Create templates for logging accounts and the deployer"
     command: >
-      {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig create -f  /tmp/logging-deployer-sa.yaml
-    register: deployer_output
-    failed_when: "deployer_output.rc == 1 and 'exists' not in deployer_output.stderr"
+      {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig create -f /usr/share/openshift/examples/infrastructure-templates/enterprise/logging-deployer.yaml
+    register: template_output
+    failed_when: "template_output.rc == 1 and 'exists' not in template_output.stderr"
+
+  - name: "Process the logging accounts template"
+    shell:  "{{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig process logging-deployer-account-template |  {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig create -f -"
+    register: process_deployer_accounts
+    failed_when: process_deployer_accounts.rc == 1 and 'already exists' not in process_deployer_accounts.stderr
 
   - name: "Set permissions for logging-deployer service account"
     command: >
-      {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig policy add-role-to-user edit system:serviceaccount:logging:logging-deployer
+      {{ openshift.common.admin_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig policy add-cluster-role-to-user oauth-editor system:serviceaccount:logging:logging-deployer
     register: permiss_output
     failed_when: "permiss_output.rc == 1 and 'exists' not in permiss_output.stderr"
 
@@ -65,14 +67,14 @@
     register: fluentd2_output
     failed_when: "fluentd2_output.rc == 1 and 'exists' not in fluentd2_output.stderr"
 
-  - name: "Create deployer template"
+  - name: "Create ConfigMap for deployer parameters"
     command: >
-      {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig create -f /usr/share/openshift/examples/infrastructure-templates/enterprise/logging-deployer.yaml
-    register: template_output
-    failed_when: "template_output.rc == 1 and 'exists' not in template_output.stderr"
+      {{ openshift.common.client_binary}} --config={{ mktemp.stdout }}/admin.kubeconfig create configmap logging-deployer {{ deployer_cmap_params }}
+    register: deployer_configmap_output
+    failed_when: "deployer_configmap_output.rc == 1 and 'exists' not in deployer_configmap_output.stderr"
 
   - name: "Process the deployer template"
-    shell:  "{{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig process logging-deployer-template -v {{ oc_process_values }} |  {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig create -f -"
+    shell:  "{{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig new-app logging-deployer-template {{ oc_new_app_values }}"
     register: process_deployer
     failed_when: process_deployer.rc == 1 and 'already exists' not in process_deployer.stderr
 
@@ -83,39 +85,71 @@
     retries: 15
     delay: 10
 
-  - name: "Process support template"
-    shell:  "{{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig process logging-support-template |  {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig create -f -"
-    register: process_support
-    failed_when: process_support.rc == 1 and 'already exists' not in process_support.stderr
+  - name: "Process imagestream template"
+    shell:  "{{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig new-app logging-imagestream-template {{ oc_new_app_values }}"
+    when: tr_or_ohlip is defined and insecure_registry is defined and insecure_registry
+    register: process_is
+    failed_when: process_is.rc == 1 and 'already exists' not in process_is.stderr
 
   - name: "Set insecured registry"
     command:  "{{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig annotate is --all  openshift.io/image.insecureRepository=true --overwrite"
-    when: "target_registry is defined and insecure_registry == 'true'"
+    when: tr_or_ohlip is defined and insecure_registry is defined and insecure_registry
 
   - name: "Wait for imagestreams to become available"
     shell:  "{{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get is | grep logging-fluentd"
+    when: tr_or_ohlip is defined and insecure_registry is defined and insecure_registry
     register: result
     until: result.rc == 0
     failed_when: result.rc == 1 and 'not found' not in result.stderr
     retries: 20
     delay: 10
 
-  - name: "Wait for replication controllers to become available"
-    shell:  "{{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get rc | grep logging-fluentd-1"
+  - name: "Wait for component pods to be running"
+    shell:  "{{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get pods -l component={{ item }} | grep Running"
+    with_items:
+      - es
+      - kibana
+      - curator
     register: result
     until: result.rc == 0
-    failed_when: result.rc == 1 and 'not found' not in result.stderr
+    failed_when: result.rc == 1 or 'Error' in result.stderr
+    retries: 20
+    delay: 10
+
+  - name: "Wait for ops component pods to be running"
+    shell:  "{{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get pods -l component={{ item }} | grep Running"
+    with_items:
+      - es-ops
+      - kibana-ops
+      - curator-ops
+    when: openshift_hosted_logging_enable_ops_cluster is defined and openshift_hosted_logging_enable_ops_cluster
+    register: result
+    until: result.rc == 0
+    failed_when: result.rc == 1 or 'Error' in result.stderr
     retries: 20
     delay: 10
 
+  - name: "Wait for fluentd DaemonSet to exist"
+    shell:  "{{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get daemonset logging-fluentd"
+    register: result
+    until: result.rc == 0
+    failed_when: result.rc == 1 or 'Error' in result.stderr
+    retries: 20
+    delay: 10
 
-  - name: "Scale fluentd deployment config"
-    command: >
-      {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig scale dc/logging-fluentd --replicas={{ fluentd_replicas | default('1') }}
+  - name: "Deploy fluentd by labeling the node"
+    shell:  "{{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig label node {{ openshift_hostname }} {{ openshift_hosted_logging_fluentd_nodeselector if openshift_hosted_logging_fluentd_nodeselector is defined else 'logging-infra-fluentd=true' }}"
 
+  - name: "Wait for fluentd to be running"
+    shell:  "{{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get pods -l component=fluentd | grep Running"
+    register: result
+    until: result.rc == 0
+    failed_when: result.rc == 1 or 'Error' in result.stderr
+    retries: 20
+    delay: 10
 
   - debug:
-      msg: "Logging components deployed. Note persistant volume for elasticsearch must be setup manually"
+      msg: "Logging components deployed. Note persistent volume for elasticsearch must be setup manually"
 
   - name: Delete temp directory
     file:
diff --git a/roles/openshift_hosted_logging/vars/main.yaml b/roles/openshift_hosted_logging/vars/main.yaml
index 88d112209..2ef3164e1 100644
--- a/roles/openshift_hosted_logging/vars/main.yaml
+++ b/roles/openshift_hosted_logging/vars/main.yaml
@@ -1,6 +1,32 @@
-kh_kv: "KIBANA_HOSTNAME={{ openshift_hosted_logging_hostname | quote }}"
-es_cs_kv: "ES_CLUSTER_SIZE={{ openshift_hosted_logging_elasticsearch_cluster_size | string | quote }}"
-pmu_kv: "PUBLIC_MASTER_URL={{ openshift_hosted_logging_master_public_url | quote }}"
-ip_kv: "{{ 'IMAGE_PREFIX=' ~ target_registry | quote if target_registry is defined else '' }}"
-oc_process_values: "{{ kh_kv }},{{ es_cs_kv }},{{ pmu_kv }},{{ ip_kv }}"
+tr_or_ohlip: "{{ openshift_hosted_logging_image_prefix or target_registry or none }}"
+ip_kv: "{{ '-p IMAGE_PREFIX=' ~ tr_or_ohlip | quote if tr_or_ohlip is defined else '' }}"
+iv_kv: "{{ '-p IMAGE_VERSION=' ~ openshift_hosted_logging_image_version | quote if openshift_hosted_logging_image_version is defined else '' }}"
+oc_new_app_values: "{{ ip_kv }} {{ iv_kv }}"
 openshift_master_config_dir: "{{ openshift.common.config_base }}/master"
+kh_cmap_param: "{{ '--from-literal kibana-hostname=' ~ openshift_hosted_logging_hostname | quote if openshift_hosted_logging_hostname is defined else '' }}"
+kh_ops_cmap_param: "{{ '--from-literal kibana-ops-hostname=' ~ openshift_hosted_logging_ops_hostname | quote if openshift_hosted_logging_ops_hostname is defined else '' }}"
+pmu_cmap_param: "{{ '--from-literal public-master-url=' ~ openshift_hosted_logging_master_public_url | quote if openshift_hosted_logging_master_public_url is defined else '' }}"
+es_cs_cmap_param: "{{ '--from-literal es-cluster-size=' ~ openshift_hosted_logging_elasticsearch_cluster_size | string | quote if openshift_hosted_logging_elasticsearch_cluster_size is defined else '' }}"
+es_ops_cs_cmap_param: "{{ '--from-literal es-ops-cluster-size=' ~ openshift_hosted_logging_elasticsearch_ops_cluster_size | string | quote if openshift_hosted_logging_elasticsearch_ops_cluster_size is defined else '' }}"
+es_ir_cmap_param: "{{ '--from-literal es-instance-ram=' ~ openshift_hosted_logging_elasticsearch_instance_ram | quote if openshift_hosted_logging_elasticsearch_instance_ram is defined else '' }}"
+es_ops_ir_cmap_param: "{{ '--from-literal es-ops-instance-ram=' ~ openshift_hosted_logging_elasticsearch_ops_instance_ram | quote if openshift_hosted_logging_elasticsearch_ops_instance_ram is defined else '' }}"
+es_pvcs_cmap_param: "{{ '--from-literal es-pvc-size=' ~ openshift_hosted_logging_elasticsearch_pvc_size | quote if openshift_hosted_logging_elasticsearch_pvc_size is defined else '' }}"
+es_ops_pvcs_cmap_param: "{{ '--from-literal es-ops-pvc-size=' ~ openshift_hosted_logging_elasticsearch_ops_pvc_size | quote if openshift_hosted_logging_elasticsearch_ops_pvc_size is defined else '' }}"
+es_pvcp_cmap_param: "{{ '--from-literal es-pvc-prefix=' ~ openshift_hosted_logging_elasticsearch_pvc_prefix | quote if openshift_hosted_logging_elasticsearch_pvc_prefix is defined else '' }}"
+es_ops_pvcp_cmap_param: "{{ '--from-literal es-ops-pvc-prefix=' ~ openshift_hosted_logging_elasticsearch_ops_pvc_prefix | quote if openshift_hosted_logging_elasticsearch_ops_pvc_prefix is defined else '' }}"
+es_pvcd_cmap_param: "{{ '--from-literal es-pvc-dynamic=' ~ openshift_hosted_logging_elasticsearch_pvc_dynamic | quote if openshift_hosted_logging_elasticsearch_pvc_dynamic is defined else '' }}"
+es_ops_pvcd_cmap_param: "{{ '--from-literal es-ops-pvc-dynamic=' ~ openshift_hosted_logging_elasticsearch_ops_pvc_dynamic | quote if openshift_hosted_logging_elasticsearch_ops_pvc_dynamic is defined else '' }}"
+es_sg_cmap_param: "{{ '--from-literal storage-group=' ~ openshift_hosted_logging_elasticsearch_storage_group | string | quote if openshift_hosted_logging_elasticsearch_storage_group is defined else '' }}"
+es_ns_cmap_param: "{{ '--from-literal es-nodeselector=' ~ openshift_hosted_logging_elasticsearch_nodeselector | quote if openshift_hosted_logging_elasticsearch_nodeselector is defined else '' }}"
+es_ops_ns_cmap_param: "{{ '--from-literal es-ops-nodeselector=' ~ openshift_hosted_logging_elasticsearch_ops_nodeselector | quote if openshift_hosted_logging_elasticsearch_ops_nodeselector is defined else '' }}"
+fd_ns_cmap_param: "{{ '--from-literal fluentd-nodeselector=' ~ openshift_hosted_logging_fluentd_nodeselector | quote if openshift_hosted_logging_fluentd_nodeselector is defined else 'logging-infra-fluentd=true' }}"
+kb_ns_cmap_param: "{{ '--from-literal kibana-nodeselector=' ~ openshift_hosted_logging_kibana_nodeselector | quote if openshift_hosted_logging_kibana_nodeselector is defined else '' }}"
+kb_ops_ns_cmap_param: "{{ '--from-literal kibana-ops-nodeselector=' ~ openshift_hosted_logging_kibana_ops_nodeselector | quote if openshift_hosted_logging_kibana_ops_nodeselector is defined else '' }}"
+cr_ns_cmap_param: "{{ '--from-literal curator-nodeselector=' ~ openshift_hosted_logging_curator_nodeselector | quote if openshift_hosted_logging_curator_nodeselector is defined else '' }}"
+cr_ops_ns_cmap_param: "{{ '--from-literal curator-ops-nodeselector=' ~ openshift_hosted_logging_curator_ops_nodeselector | quote if openshift_hosted_logging_curator_ops_nodeselector is defined else '' }}"
+ops_cmap_param: "{{ '--from-literal enable-ops-cluster=' ~ openshift_hosted_logging_enable_ops_cluster | string | lower | quote if openshift_hosted_logging_enable_ops_cluster is defined else '' }}"
+use_journal_cmap_param: "{{ '--from-literal use-journal=' ~ openshift_hosted_logging_use_journal | string | lower | quote if openshift_hosted_logging_use_journal is defined else '' }}"
+journal_source_cmap_param: "{{ '--from-literal journal-source=' ~ openshift_hosted_logging_journal_source | quote if openshift_hosted_logging_journal_source is defined else '' }}"
+journal_read_from_head_cmap_param: "{{ '--from-literal journal-read-from-head=' ~ openshift_hosted_logging_journal_read_from_head | string | lower | quote if openshift_hosted_logging_journal_read_from_head is defined else '' }}"
+ips_cmap_param: "{{ '--from-literal image-pull-secret=' ~ openshift_hosted_logging_image_pull_secret | quote if openshift_hosted_logging_image_pull_secret is defined else '' }}"
+deployer_cmap_params: "{{ kh_cmap_param }} {{ kh_ops_cmap_param }} {{ pmu_cmap_param }} {{ es_cs_cmap_param }} {{ es_ir_cmap_param }} {{ es_pvcs_cmap_param }} {{ es_pvcp_cmap_param }} {{ es_pvcd_cmap_param }} {{ es_ops_cs_cmap_param }} {{ es_ops_ir_cmap_param }} {{ es_ops_pvcs_cmap_param }} {{ es_ops_pvcp_cmap_param }} {{ es_ops_pvcd_cmap_param }} {{ es_sg_cmap_param }} {{ es_ns_cmap_param }} {{ es_ops_ns_cmap_param }} {{ fd_ns_cmap_param }} {{ kb_ns_cmap_param }} {{ kb_ops_ns_cmap_param }} {{ cr_ns_cmap_param }} {{ cr_ops_ns_cmap_param }} {{ ops_cmap_param }} {{ use_journal_cmap_param }} {{ journal_source_cmap_param }} {{ journal_read_from_head_cmap_param }} {{ ips_cmap_param }}"