diff options
| -rw-r--r-- | roles/openshift_metrics/handlers/main.yml | 31 | ||||
| -rw-r--r-- | roles/openshift_metrics/tasks/install.yml | 114 | ||||
| -rw-r--r-- | roles/openshift_metrics/tasks/main.yaml | 112 | ||||
| -rw-r--r-- | roles/openshift_metrics/vars/main.yaml | 2 |
4 files changed, 175 insertions, 84 deletions
diff --git a/roles/openshift_metrics/handlers/main.yml b/roles/openshift_metrics/handlers/main.yml new file mode 100644 index 000000000..edb7369de --- /dev/null +++ b/roles/openshift_metrics/handlers/main.yml @@ -0,0 +1,31 @@ +--- +- name: restart master + service: name={{ openshift.common.service_type }}-master state=restarted + when: (openshift.master.ha is not defined or not openshift.master.ha | bool) and (not (master_service_status_changed | default(false) | bool)) + notify: Verify API Server + +- name: restart master api + service: name={{ openshift.common.service_type }}-master-api state=restarted + when: (openshift.master.ha is defined and openshift.master.ha | bool) and (not (master_api_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native' + notify: Verify API Server + +- name: restart master controllers + service: name={{ openshift.common.service_type }}-master-controllers state=restarted + when: (openshift.master.ha is defined and openshift.master.ha | bool) and (not (master_controllers_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native' + +- name: Verify API Server + # Using curl here since the uri module requires python-httplib2 and + # wait_for port doesn't provide health information. + command: > + curl --silent + {% if openshift.common.version_gte_3_2_or_1_2 | bool %} + --cacert {{ openshift.common.config_base }}/master/ca-bundle.crt + {% else %} + --cacert {{ openshift.common.config_base }}/master/ca.crt + {% endif %} + {{ openshift.master.api_url }}/healthz/ready + register: api_available_output + until: api_available_output.stdout == 'ok' + retries: 120 + delay: 1 + changed_when: false diff --git a/roles/openshift_metrics/tasks/install.yml b/roles/openshift_metrics/tasks/install.yml new file mode 100644 index 000000000..1f30e292b --- /dev/null +++ b/roles/openshift_metrics/tasks/install.yml @@ -0,0 +1,114 @@ +--- + +- name: Test if metrics-deployer service account exists + command: > + {{ openshift.common.client_binary }} + --config={{ openshift_metrics_kubeconfig }} + --namespace=openshift-infra + get serviceaccount metrics-deployer -o json + register: serviceaccount + changed_when: false + failed_when: false + +- name: Create metrics-deployer Service Account + shell: > + echo {{ metrics_deployer_sa | to_json | quote }} | + {{ openshift.common.client_binary }} + --config={{ openshift_metrics_kubeconfig }} + --namespace openshift-infra + create -f - + when: serviceaccount.rc == 1 + +- name: Test edit permissions + command: > + {{ openshift.common.client_binary }} + --config={{ openshift_metrics_kubeconfig }} + --namespace openshift-infra + get rolebindings -o jsonpath='{.items[?(@.metadata.name == "edit")].userNames}' + register: edit_rolebindings + changed_when: false + +- name: Add edit permission to the openshift-infra project to metrics-deployer SA + command: > + {{ openshift.common.admin_binary }} + --config={{ openshift_metrics_kubeconfig }} + --namespace openshift-infra + policy add-role-to-user edit + system:serviceaccount:openshift-infra:metrics-deployer + when: "'system:serviceaccount:openshift-infra:metrics-deployer' not in edit_rolebindings.stdout" + +- name: Test cluster-reader permissions + command: > + {{ openshift.common.client_binary }} + --config={{ openshift_metrics_kubeconfig }} + --namespace openshift-infra + get clusterrolebindings -o jsonpath='{.items[?(@.metadata.name == "cluster-reader")].userNames}' + register: cluster_reader_clusterrolebindings + changed_when: false + +- name: Add cluster-reader permission to the openshift-infra project to heapster SA + command: > + {{ openshift.common.admin_binary }} + --config={{ openshift_metrics_kubeconfig }} + --namespace openshift-infra + policy add-cluster-role-to-user cluster-reader + system:serviceaccount:openshift-infra:heapster + when: "'system:serviceaccount:openshift-infra:heapster' not in cluster_reader_clusterrolebindings.stdout" + +- name: Create metrics-deployer secret + command: > + {{ openshift.common.client_binary }} + --config={{ openshift_metrics_kubeconfig }} + --namespace openshift-infra + secrets new metrics-deployer nothing=/dev/null + register: metrics_deployer_secret + changed_when: metrics_deployer_secret.rc == 0 + failed_when: "metrics_deployer_secret.rc == 1 and 'already exists' not in metrics_deployer_secret.stderr" + +# TODO: extend this to allow user passed in certs or generating cert with +# OpenShift CA +- name: Build metrics deployer command + set_fact: + deployer_cmd: "{{ openshift.common.client_binary }} process -f \ + {{ metrics_template_dir }}/metrics-deployer.yaml -v \ + HAWKULAR_METRICS_HOSTNAME={{ metrics_hostname }},USE_PERSISTENT_STORAGE={{metrics_persistence | string | lower }},METRIC_DURATION={{ openshift.hosted.metrics.duration }},METRIC_RESOLUTION={{ openshift.hosted.metrics.resolution }},IMAGE_PREFIX={{ openshift.hosted.metrics.deployer_prefix }},IMAGE_VERSION={{ openshift.hosted.metrics.deployer_version }},MODE={{ deployment_mode }} \ + | {{ openshift.common.client_binary }} --namespace openshift-infra \ + --config={{ openshift_metrics_kubeconfig }} \ + create -f -" + +- name: Deploy Metrics + shell: "{{ deployer_cmd }}" + register: deploy_metrics + failed_when: "'already exists' not in deploy_metrics.stderr and deploy_metrics.rc != 0" + changed_when: deploy_metrics.rc == 0 + +- set_fact: + deployer_pod: "{{ deploy_metrics.stdout[1:2] }}" + +# TODO: re-enable this once the metrics deployer validation issue is fixed +# when using dynamically provisioned volumes +- name: "Wait for image pull and deployer pod" + shell: > + {{ openshift.common.client_binary }} + --namespace openshift-infra + --config={{ openshift_metrics_kubeconfig }} + get {{ deploy_metrics.stdout }} + register: deploy_result + until: "{{ 'Completed' in deploy_result.stdout }}" + failed_when: "{{ 'Completed' not in deploy_result.stdout }}" + retries: 60 + delay: 10 + +- name: Configure master for metrics + modify_yaml: + dest: "{{ openshift.common.config_base }}/master/master-config.yaml" + yaml_key: assetConfig.metricsPublicURL + yaml_value: "https://{{ metrics_hostname }}/hawkular/metrics" + notify: restart master + +- name: Store metrics public_url + openshift_facts: + role: master + local_facts: + metrics_public_url: "https://{{ metrics_hostname }}/hawkular/metrics" + when: deploy_result | changed diff --git a/roles/openshift_metrics/tasks/main.yaml b/roles/openshift_metrics/tasks/main.yaml index d7d984118..b20cd4b0e 100644 --- a/roles/openshift_metrics/tasks/main.yaml +++ b/roles/openshift_metrics/tasks/main.yaml @@ -8,7 +8,8 @@ register: mktemp changed_when: False -- set_fact: +- name: Record kubeconfig tmp dir + set_fact: openshift_metrics_kubeconfig: "{{ mktemp.stdout }}/admin.kubeconfig" - name: Copy the admin client config(s) @@ -32,108 +33,51 @@ | default('hawkular-metrics.' ~ (openshift.master.default_subdomain | default(openshift_master_default_subdomain ))) | oo_hostname_from_url }}" - metrics_persistence: "{{ openshift.hosted.metrics.storage_kind | default(none) is not none }}" + metrics_persistence: True + #"{{ openshift.hosted.metrics.storage_kind | default(none) is not none }}" metrics_dynamic_vol: "{{ openshift.hosted.metrics.storage_kind | default(none) == 'dynamic' }}" metrics_template_dir: "/usr/share/openshift/examples/infrastructure-templates/{{ 'origin' if deployment_type == 'origin' else 'enterprise' }}" cassandra_nodes: "{{ ',CASSANDRA_NODES=' ~ openshift.hosted.metrics.cassandra_nodes if 'cassandra' in openshift.hosted.metrics else '' }}" cassandra_pv_size: "{{ ',CASSANDRA_PV_SIZE=' ~ openshift.hosted.metrics.storage_volume_size if openshift.hosted.metrics.storage_volume_size | default(none) is not none else '' }}" -- name: Test if metrics-deployer service account exists - command: > - {{ openshift.common.client_binary }} - --config={{ openshift_metrics_kubeconfig }} - --namespace=openshift-infra - get serviceaccount metrics-deployer -o json - register: serviceaccount - changed_when: false - failed_when: false -- name: Create metrics-deployer Service Account +- name: Check for existing metrics pods shell: > - echo {{ metrics_deployer_sa | to_json | quote }} | - {{ openshift.common.client_binary }} - --config={{ openshift_metrics_kubeconfig }} - --namespace openshift-infra - create -f - - when: serviceaccount.rc == 1 - -- name: Test edit permissions - command: > {{ openshift.common.client_binary }} --config={{ openshift_metrics_kubeconfig }} --namespace openshift-infra - get rolebindings -o jsonpath='{.items[?(@.metadata.name == "edit")].userNames}' - register: edit_rolebindings + get pods -l {{ item }} | grep -q Running + register: metrics_pods_status + with_items: + - metrics-infra=hawkular-metrics + - metrics-infra=heapster + - metrics-infra=hawkular-cassandra + failed_when: false changed_when: false -- name: Add edit permission to the openshift-infra project to metrics-deployer SA - command: > - {{ openshift.common.admin_binary }} - --config={{ openshift_metrics_kubeconfig }} - --namespace openshift-infra - policy add-role-to-user edit - system:serviceaccount:openshift-infra:metrics-deployer - when: "'system:serviceaccount:openshift-infra:metrics-deployer' not in edit_rolebindings.stdout" - -- name: Test cluster-reader permissions - command: > +- name: Check for previous deployer + shell: > {{ openshift.common.client_binary }} --config={{ openshift_metrics_kubeconfig }} --namespace openshift-infra - get clusterrolebindings -o jsonpath='{.items[?(@.metadata.name == "cluster-reader")].userNames}' - register: cluster_reader_clusterrolebindings + get pods -l metrics-infra=deployer --sort-by='{.metadata.creationTimestamp}' | tail -1 | grep metrics-deployer- + register: metrics_deployer_status + failed_when: false changed_when: false -- name: Add cluster-reader permission to the openshift-infra project to heapster SA - command: > - {{ openshift.common.admin_binary }} - --config={{ openshift_metrics_kubeconfig }} - --namespace openshift-infra - policy add-cluster-role-to-user cluster-reader - system:serviceaccount:openshift-infra:heapster - when: "'system:serviceaccount:openshift-infra:heapster' not in cluster_reader_clusterrolebindings.stdout" - -# TODO: extend this to allow user passed in certs or generating cert with -# OpenShift CA -- name: Create metrics-deployer secret - command: > - {{ openshift.common.client_binary }} - --config={{ openshift_metrics_kubeconfig }} - --namespace openshift-infra - secrets new metrics-deployer nothing=/dev/null - register: metrics_deployer_secret - changed_when: metrics_deployer_secret.rc == 0 - failed_when: "metrics_deployer_secret.rc == 1 and 'already exists' not in metrics_deployer_secret.stderr" +- name: Record current deployment status + set_fact: + greenfield: "{{ not metrics_deployer_status.rc == 0 }}" + failed_error: "{{ True if 'Error' in metrics_deployer_status.stdout else False }}" + metrics_running: "{{ metrics_pods_status.results | oo_collect(attribute='rc') == [0,0,0] }}" -- debug: var=openshift.hosted.metrics.deployer_prefix -- debug: var=openshift.hosted.metrics.deployer_version +- name: Set deployment mode + set_fact: + deployment_mode: "{{ 'refresh' if (failed_error | bool or metrics_upgrade | bool) else 'deploy' }}" - -- name: Deploy Metrics - shell: > - {{ openshift.common.client_binary }} process -f - {{ metrics_template_dir }}/metrics-deployer.yaml -v - HAWKULAR_METRICS_HOSTNAME={{ metrics_hostname }},USE_PERSISTENT_STORAGE={{ - metrics_persistence | string | lower }},METRIC_DURATION={{ openshift.hosted.metrics.duration }},METRIC_RESOLUTION={{ openshift.hosted.metrics.resolution }},IMAGE_PREFIX={{ openshift.hosted.metrics.deployer_prefix }},IMAGE_VERSION={{ openshift.hosted.metrics.deployer_version }} - | {{ openshift.common.client_binary }} --namespace openshift-infra - --config={{ openshift_metrics_kubeconfig }} - create -f - - register: deploy_metrics - failed_when: "'already exists' not in deploy_metrics.stderr and deploy_metrics.rc != 0" - changed_when: deploy_metrics.rc == 0 - -# TODO: re-enable this once the metrics deployer validation issue is fixed -# when using dynamically provisioned volumes -- name: "Wait for image pull and deployer pod" - shell: > - {{ openshift.common.client_binary }} - --namespace openshift-infra - --config={{ openshift_metrics_kubeconfig }} - get pods | grep metrics-deployer.*Completed - register: result - until: result.rc == 0 - retries: 60 - delay: 10 +# TODO: handle non greenfield deployments in the future +- include: install.yml + when: greenfield - name: Delete temp directory file: diff --git a/roles/openshift_metrics/vars/main.yaml b/roles/openshift_metrics/vars/main.yaml index bd542382f..0331bcb89 100644 --- a/roles/openshift_metrics/vars/main.yaml +++ b/roles/openshift_metrics/vars/main.yaml @@ -16,3 +16,5 @@ hawkular_tmp_conf: /tmp/hawkular_admin.kubeconfig hawkular_persistence: "{% if openshift.hosted.metrics.storage.kind != None %}true{% else %}false{% endif %}" hawkular_type: "{{ 'origin' if deployment_type == 'origin' else 'enterprise' }}" + +metrics_upgrade: openshift.hosted.metrics.upgrade | default(False) |