From 04c1500801f4d88635001bda1e4f73473fe8e33a Mon Sep 17 00:00:00 2001 From: Jeff Cantrill Date: Tue, 29 Nov 2016 16:31:13 -0500 Subject: =?UTF-8?q?Bruno=20Barcarol=20Guimar=C3=A3es=20work=20to=20move=20?= =?UTF-8?q?metrics=20to=20ansible=20from=20deployer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- roles/openshift_metrics/README.md | 86 ++++++++ roles/openshift_metrics/defaults/main.yaml | 17 ++ roles/openshift_metrics/meta/main.yaml | 2 + roles/openshift_metrics/tasks/cleanup.yaml | 14 ++ .../tasks/generate_certificates.yaml | 233 +++++++++++++++++++++ .../tasks/generate_rolebindings.yaml | 30 +++ .../tasks/generate_serviceaccounts.yaml | 25 +++ .../openshift_metrics/tasks/generate_services.yaml | 43 ++++ .../openshift_metrics/tasks/install_hawkular.yaml | 57 +++++ .../openshift_metrics/tasks/install_heapster.yaml | 3 + roles/openshift_metrics/tasks/install_metrics.yaml | 17 ++ roles/openshift_metrics/tasks/main.yaml | 24 +++ .../openshift_metrics/tasks/setup_certificate.yaml | 50 +++++ .../templates/hawkular_cassandra_rc.j2 | 94 +++++++++ .../templates/hawkular_metrics_rc.j2 | 88 ++++++++ roles/openshift_metrics/templates/heapster.j2 | 66 ++++++ roles/openshift_metrics/templates/pvc.j2 | 27 +++ roles/openshift_metrics/templates/rolebinding.j2 | 23 ++ roles/openshift_metrics/templates/route.j2 | 23 ++ roles/openshift_metrics/templates/secret.j2 | 12 ++ roles/openshift_metrics/templates/service.j2 | 32 +++ .../openshift_metrics/templates/serviceaccount.j2 | 16 ++ roles/openshift_metrics/vars/main.yaml | 4 + 23 files changed, 986 insertions(+) create mode 100644 roles/openshift_metrics/README.md create mode 100644 roles/openshift_metrics/defaults/main.yaml create mode 100644 roles/openshift_metrics/meta/main.yaml create mode 100644 roles/openshift_metrics/tasks/cleanup.yaml create mode 100644 roles/openshift_metrics/tasks/generate_certificates.yaml create mode 100644 roles/openshift_metrics/tasks/generate_rolebindings.yaml create mode 100644 roles/openshift_metrics/tasks/generate_serviceaccounts.yaml create mode 100644 roles/openshift_metrics/tasks/generate_services.yaml create mode 100644 roles/openshift_metrics/tasks/install_hawkular.yaml create mode 100644 roles/openshift_metrics/tasks/install_heapster.yaml create mode 100644 roles/openshift_metrics/tasks/install_metrics.yaml create mode 100644 roles/openshift_metrics/tasks/main.yaml create mode 100644 roles/openshift_metrics/tasks/setup_certificate.yaml create mode 100644 roles/openshift_metrics/templates/hawkular_cassandra_rc.j2 create mode 100644 roles/openshift_metrics/templates/hawkular_metrics_rc.j2 create mode 100644 roles/openshift_metrics/templates/heapster.j2 create mode 100644 roles/openshift_metrics/templates/pvc.j2 create mode 100644 roles/openshift_metrics/templates/rolebinding.j2 create mode 100644 roles/openshift_metrics/templates/route.j2 create mode 100644 roles/openshift_metrics/templates/secret.j2 create mode 100644 roles/openshift_metrics/templates/service.j2 create mode 100644 roles/openshift_metrics/templates/serviceaccount.j2 create mode 100644 roles/openshift_metrics/vars/main.yaml diff --git a/roles/openshift_metrics/README.md b/roles/openshift_metrics/README.md new file mode 100644 index 000000000..ac5353886 --- /dev/null +++ b/roles/openshift_metrics/README.md @@ -0,0 +1,86 @@ +OpenShift Metrics with Hawkular +==================== + +OpenShift Metrics Installation + +Requirements +------------ + +The following variables need to be set and will be validated: + +- `metrics_hostname`: hostname used on the hawkular metrics route. + +- `metrics_project`: project (i.e. namespace) where the components will be + deployed. + + +Role Variables +-------------- + +For default values, see [`defaults/main.yaml`](defaults/main.yaml). + +- `image_prefix`: Specify prefix for metrics components; e.g for + "openshift/origin-metrics-deployer:v1.1", set prefix "openshift/origin-". + +- `image_version`: Specify version for metrics components; e.g. for + "openshift/origin-metrics-deployer:v1.1", set version "v1.1". + +- `master_url`: Internal URL for the master, for authentication retrieval. + +- `hawkular_user_write_access`: If user accounts should be able to write + metrics. Defaults to 'false' so that only Heapster can write metrics and not + individual users. It is recommended to disable user write access, if enabled + any user will be able to write metrics to the system which can affect + performance and use Cassandra disk usage to unpredictably increase. + +- `hawkular_cassandra_nodes`: The number of Cassandra Nodes to deploy for the + initial cluster. + +- `hawkular_cassandra_storage_type`: Use `emptydir` for ephemeral storage (for + testing), `pv` to use persistent volumes (which need to be created before the + installation) or `dynamic` for dynamic persistent volumes. + +- `hawkular_cassandra_pv_prefix`: The name of persistent volume claims created + for cassandra will be this with a serial number appended to the end, starting + from 1. + +- `hawkular_cassandra_pv_size`: The persistent volume size for each of the + Cassandra nodes. + +- `heapster_standalone`: Deploy only heapster, without the Hawkular Metrics and + Cassandra components. + +- `heapster_allowed_users`: A comma-separated list of CN to accept. By + default, this is set to allow the OpenShift service proxy to connect. If you + override this, make sure to add `system:master-proxy` to the list in order to + allow horizontal pod autoscaling to function properly. + +- `metrics_duration`: How many days metrics should be stored for. + +- `metrics_resolution`: How often metrics should be gathered. + + +Dependencies +------------ +openshift_facts + + +Example Playbook +---------------- + +``` +- name: Configure openshift-metrics + hosts: oo_first_master + roles: + - role: openshift_metrics +``` + +License +------- + +Apache License, Version 2.0 + +Author Information +------------------ + +Jose David Martín (j.david.nieto@gmail.com) diff --git a/roles/openshift_metrics/defaults/main.yaml b/roles/openshift_metrics/defaults/main.yaml new file mode 100644 index 000000000..cb4fbdee2 --- /dev/null +++ b/roles/openshift_metrics/defaults/main.yaml @@ -0,0 +1,17 @@ +--- +image_prefix: docker.io/openshift/origin- +image_version: latest +master_url: https://kubernetes.default.svc.cluster.local + +hawkular_user_write_access: False +hawkular_cassandra_nodes: 1 +hawkular_cassandra_storage_type: emptydir +hawkular_cassandra_pv_prefix: metrics-cassandra +hawkular_cassandra_pv_size: 10Gi + +heapster_standalone: False +heapster_allowed_users: system:master-proxy + +metrics_duration: 7 +metrics_resolution: 15s +metrics_node_id: nodename diff --git a/roles/openshift_metrics/meta/main.yaml b/roles/openshift_metrics/meta/main.yaml new file mode 100644 index 000000000..a8fbeff02 --- /dev/null +++ b/roles/openshift_metrics/meta/main.yaml @@ -0,0 +1,2 @@ +dependencies: +- { role: openshift_facts } diff --git a/roles/openshift_metrics/tasks/cleanup.yaml b/roles/openshift_metrics/tasks/cleanup.yaml new file mode 100644 index 000000000..a61fed7b4 --- /dev/null +++ b/roles/openshift_metrics/tasks/cleanup.yaml @@ -0,0 +1,14 @@ +--- +- name: remove metrics components + command: > + {{ openshift.common.client_binary }} -n '{{ metrics_project }}' + delete --selector=metrics-infra + all,sa,secrets,templates,routes,pvc,rolebindings,clusterrolebindings + register: delete_metrics + changed_when: "delete_metrics.stdout != 'No resources found'" +- name: remove rolebindings + command: > + {{ openshift.common.client_binary }} -n {{ metrics_project }} + delete --ignore-not-found + rolebinding/hawkular-view + clusterrolebinding/heapster-cluster-reader diff --git a/roles/openshift_metrics/tasks/generate_certificates.yaml b/roles/openshift_metrics/tasks/generate_certificates.yaml new file mode 100644 index 000000000..b1ecf46b9 --- /dev/null +++ b/roles/openshift_metrics/tasks/generate_certificates.yaml @@ -0,0 +1,233 @@ +--- +# TODO idempotency? +# TODO support providing custom certificates +- name: create certificate output directory + file: + path: "{{ mktemp.stdout }}/certs" + state: directory + mode: 0700 +- name: generate ca certificate chain + shell: > + {{ openshift.common.admin_binary }} ca create-signer-cert + --key='{{ mktemp.stdout }}/certs/ca.key' + --cert='{{ mktemp.stdout }}/certs/ca.crt' + --serial='{{ mktemp.stdout }}/certs/ca.serial.txt' + --name="metrics-signer@$(date +%s)" +- name: generate heapster key/cert + command: > + {{ openshift.common.admin_binary }} ca create-server-cert + --key='{{ mktemp.stdout }}/certs/heapster.key' + --cert='{{ mktemp.stdout }}/certs/heapster.cert' + --hostnames=heapster + --signer-cert='{{ mktemp.stdout }}/certs/ca.crt' + --signer-key='{{ mktemp.stdout }}/certs/ca.key' + --signer-serial='{{ mktemp.stdout }}/certs/ca.serial.txt' +# TODO maybe there's an easier way to get the service accounts' ca crt? +- name: get heapster service account secrets + shell: > + {{ openshift.common.client_binary }} -n '{{ metrics_project }}' + get serviceaccount/default + --template '{{ '{{range .secrets}}{{println .name}}{{end}}' }}' + | grep ^default-token- + register: sa_secret +- name: get heapster service account ca + command: > + {{ openshift.common.client_binary }} -n '{{ metrics_project }}' + get 'secret/{{ sa_secret.stdout }}' + --template '{{ '{{index .data "ca.crt"}}' }}' + register: sa_secret +- name: read files for the heapster secret + command: base64 --wrap 0 "{{ mktemp.stdout }}/certs/heapster.{{ item }}" + register: heapster_secret + with_items: + - cert + - key +- name: generate heapster secret template + template: + src: secret.j2 + dest: "{{ mktemp.stdout }}/templates/heapster_secrets.yaml" + vars: + name: heapster-secrets + labels: + metrics-infra: heapster + data: + heapster.cert: "{{ heapster_secret.results[0].stdout }}" + heapster.key: "{{ heapster_secret.results[1].stdout }}" + heapster.client-ca: "{{ sa_secret.stdout }}" + heapster.allowed-users: "{{ heapster_allowed_users|b64encode }}" +- name: generate hawkular-metrics certificates + include: setup_certificate.yaml + vars: + component: hawkular-metrics + hostnames: "hawkular-metrics,{{ hawkular_metrics_hostname }}" +- name: generate hawkular-cassandra certificates + include: setup_certificate.yaml + vars: + component: hawkular-cassandra + hostnames: hawkular-cassandra +# TODO keytool as dependency? move key/trust store generation to containers? +- name: import the hawkular metrics cert into the cassandra truststore + shell: > + keytool -noprompt -import -v -trustcacerts + -alias hawkular-metrics + -file '{{ mktemp.stdout|quote }}/certs/hawkular-metrics.cert' + -keystore '{{ mktemp.stdout|quote }}/certs/hawkular-cassandra.truststore' + -storepass + "$(< "{{ mktemp.stdout|quote }}/certs/hawkular-cassandra-truststore.pwd")" +- name: import the hawkular cassandra cert into the hawkular metrics truststore + shell: > + keytool -noprompt -import -v -trustcacerts + -alias hawkular-cassandra + -file '{{ mktemp.stdout }}/certs/hawkular-cassandra.cert' + -keystore '{{ mktemp.stdout }}/certs/hawkular-metrics.truststore' + -storepass + "$(< "{{ mktemp.stdout|quote }}/certs/hawkular-metrics-truststore.pwd")" +- name: import the hawkular cassandra cert into the cassandra truststore + shell: > + keytool -noprompt -import -v -trustcacerts + -alias hawkular-cassandra + -file '{{ mktemp.stdout }}/certs/hawkular-cassandra.cert' + -keystore '{{ mktemp.stdout }}/certs/hawkular-cassandra.truststore' + -storepass + "$(< "{{ mktemp.stdout|quote }}/certs/hawkular-cassandra-truststore.pwd")" +- name: import the ca certificate into the cassandra truststore + shell: > + keytool -noprompt -import -v -trustcacerts + -alias '{{ item }}' + -file '{{ mktemp.stdout }}/certs/ca.crt' + -keystore '{{ mktemp.stdout }}/certs/hawkular-cassandra.truststore' + -storepass + "$(< "{{ mktemp.stdout|quote }}/certs/hawkular-cassandra-truststore.pwd")" + with_items: + - ca + - metricca + - cassandraca +- name: import the ca certificate into the hawkular metrics truststore + shell: > + keytool -noprompt -import -v -trustcacerts + -alias '{{ item }}' + -file '{{ mktemp.stdout }}/certs/ca.crt' + -keystore '{{ mktemp.stdout }}/certs/hawkular-metrics.truststore' + -storepass + "$(< "{{ mktemp.stdout|quote }}/certs/hawkular-metrics-truststore.pwd")" + with_items: + - ca + - metricca + - cassandraca +- name: generate password for htpasswd file for hawkular metrics + shell: cat /dev/urandom | tr -dc _A-Z-a-z-0-9 | head -c15 + register: hawkular_metrics_password +- name: generate password for hawkular metrics jgroups + shell: cat /dev/urandom | tr -dc _A-Z-a-z-0-9 | head -c15 + register: hawkular_metrics_jgroups_password +- name: generate htpasswd file for hawkular metrics + shell: > + htpasswd -cb + "{{ mktemp.stdout|quote }}/certs/hawkular-metrics.htpasswd" hawkular + '{{ hawkular_metrics_password.stdout }}' +- name: generate the jgroups keystore + command: > + keytool -genseckey -alias hawkular + -keypass {{ hawkular_metrics_jgroups_password.stdout }} + -storepass {{ hawkular_metrics_jgroups_password.stdout }} + -keyalg Blowfish -keysize 56 -storetype JCEKS + -keystore {{ mktemp.stdout }}/certs/hawkular-jgroups.keystore +- name: read files for the hawkular-metrics secret + command: > + base64 --wrap 0 "{{ mktemp.stdout }}/certs/{{ item }}" + register: hawkular_metrics_secret + with_items: + - hawkular-metrics.keystore + - hawkular-metrics-keystore.pwd + - hawkular-metrics.truststore + - hawkular-metrics-truststore.pwd + - hawkular-metrics.htpasswd + - hawkular-metrics.cert + - ca.crt + - hawkular-cassandra.keystore + - hawkular-cassandra-keystore.pwd + - hawkular-cassandra.truststore + - hawkular-cassandra-truststore.pwd + - hawkular-cassandra.pem + - hawkular-cassandra.cert + - hawkular-jgroups.keystore +- name: generate hawkular-metrics-secrets secret template + template: + src: secret.j2 + dest: "{{ mktemp.stdout }}/templates/hawkular_metrics_secrets.yaml" + vars: + name: hawkular-metrics-secrets + labels: + metrics-infra: hawkular-metrics + data: + hawkular-metrics.keystore: > + "{{ hawkular_metrics_secret.results[0].stdout }}" + hawkular-metrics.keystore.password: > + "{{ hawkular_metrics_secret.results[1].stdout }}" + hawkular-metrics.truststore: > + "{{ hawkular_metrics_secret.results[2].stdout }}" + hawkular-metrics.truststore.password: > + "{{ hawkular_metrics_secret.results[3].stdout }}" + hawkular-metrics.keystore.alias: "{{ 'hawkular-metrics'|b64encode }}" + hawkular-metrics.htpasswd.file: > + "{{ hawkular_metrics_secret.results[4].stdout }}" + hawkular-metrics.jgroups.keystore.password: > + "{{ hawkular_metrics_jgroups_password.stdout|b64encode }}" + hawkular-metrics.jgroups.keystore: > + "{{ hawkular_metrics_secret.results[13].stdout }}" + hawkular-metrics.jgroups.alias: "{{ 'hawkular'|b64encode }}" +- name: generate hawkular-metrics-certificate secret template + template: + src: secret.j2 + dest: "{{ mktemp.stdout }}/templates/hawkular_metrics_certificate.yaml" + vars: + name: hawkular-metrics-certificate + labels: + metrics-infra: hawkular-metrics + data: + hawkular-metrics.certificate: > + "{{ hawkular_metrics_secret.results[5].stdout }}" + hawkular-metrics-ca.certificate: > + "{{ hawkular_metrics_secret.results[6].stdout }}" +- name: generate hawkular-metrics-account secret template + template: + src: secret.j2 + dest: "{{ mktemp.stdout }}/templates/hawkular_metrics_account.yaml" + vars: + name: hawkular-metrics-account + labels: + metrics-infra: hawkular-metrics + data: + hawkular-metrics.username: "{{ 'hawkular'|b64encode }}" + hawkular-metrics.password: > + "{{ hawkular_metrics_password.stdout|b64encode }}" +- name: generate cassandra secret template + template: + src: secret.j2 + dest: "{{ mktemp.stdout }}/templates/cassandra_secrets.yaml" + vars: + name: hawkular-cassandra-secrets + labels: + metrics-infra: hawkular-cassandra + data: + cassandra.keystore: "{{ hawkular_metrics_secret.results[7].stdout }}" + cassandra.keystore.password: > + {{ hawkular_metrics_secret.results[8].stdout }} + cassandra.keystore.alias: "{{ 'hawkular-cassandra'|b64encode }}" + cassandra.truststore: "{{ hawkular_metrics_secret.results[9].stdout }}" + cassandra.truststore.password: > + {{ hawkular_metrics_secret.results[10].stdout }} + cassandra.pem: "{{ hawkular_metrics_secret.results[10].stdout }}" +- name: generate cassandra-certificate secret template + template: + src: secret.j2 + dest: "{{ mktemp.stdout }}/templates/cassandra_certificate.yaml" + vars: + name: hawkular-cassandra-certificate + labels: + metrics-infra: hawkular-cassandra + data: + cassandra.certificate: > + {{ hawkular_metrics_secret.results[11].stdout }} + cassandra-ca.certificate: > + {{ hawkular_metrics_secret.results[7].stdout }} diff --git a/roles/openshift_metrics/tasks/generate_rolebindings.yaml b/roles/openshift_metrics/tasks/generate_rolebindings.yaml new file mode 100644 index 000000000..d1bc7374a --- /dev/null +++ b/roles/openshift_metrics/tasks/generate_rolebindings.yaml @@ -0,0 +1,30 @@ +--- +- name: generate view role binding for the hawkular service account + template: + src: rolebinding.j2 + dest: "{{ mktemp.stdout }}/templates/hawkular-rolebinding.yaml" + vars: + obj_name: hawkular-view + labels: + metrics-infra: hawkular + roleRef: + name: view + subjects: + - kind: ServiceAccount + name: hawkular +- name: generate cluster-reader role binding for the heapster service account + template: + src: rolebinding.j2 + dest: "{{ mktemp.stdout }}/templates/heapster-rolebinding.yaml" + vars: + cluster: True + obj_name: heapster-cluster-reader + labels: + metrics-infra: heapster + roleRef: + kind: ClusterRole + name: cluster-reader + subjects: + - kind: ServiceAccount + name: heapster + namespace: "{{ metrics_project }}" diff --git a/roles/openshift_metrics/tasks/generate_serviceaccounts.yaml b/roles/openshift_metrics/tasks/generate_serviceaccounts.yaml new file mode 100644 index 000000000..9230e0423 --- /dev/null +++ b/roles/openshift_metrics/tasks/generate_serviceaccounts.yaml @@ -0,0 +1,25 @@ +--- +- name: Generating serviceaccounts for hawkular metrics/cassandra + template: src=serviceaccount.j2 dest={{mktemp.stdout}}/templates/metrics-{{obj_name}}-sa.yaml + vars: + obj_name: "{{item.name}}" + labels: + metrics-infra: support + secrets: + - hawkular-{{item.secret}}-secrets + with_items: + - name: hawkular + secret: hawkular-metrics-secrets + - name: cassandra + secret: hawkular-cassandra-secrets + +- name: Generating serviceaccount for heapster + template: src=serviceaccount.j2 dest={{mktemp.stdout}}/templates/metrics-{{obj_name}}-sa.yaml + vars: + obj_name: heapster + labels: + metrics-infra: support + secrets: + - heapster-secrets + - hawkular-metrics-certificate + - hawkular-metrics-account diff --git a/roles/openshift_metrics/tasks/generate_services.yaml b/roles/openshift_metrics/tasks/generate_services.yaml new file mode 100644 index 000000000..4f7616a1c --- /dev/null +++ b/roles/openshift_metrics/tasks/generate_services.yaml @@ -0,0 +1,43 @@ +--- +- name: Generate service for heapster + template: src=service.j2 dest={{mktemp.stdout}}/templates/metrics-{{obj_name}}-svc.yaml + vars: + obj_name: heapster + ports: + - {port: 80, targetPort: http-endpoint} + selector: + name: "{{obj_name}}" + labels: + metrics-infra: "{{obj_name}}" + name: "{{obj_name}}" + +- name: Generate service for hawkular-metrics + template: src=service.j2 dest={{mktemp.stdout}}/templates/metrics-{{obj_name}}-svc.yaml + vars: + obj_name: hawkular-metrics + ports: + - {port: 443, targetPort: https-endpoint} + selector: + name: "{{obj_name}}" + labels: + metrics-infra: "{{obj_name}}" + name: "{{obj_name}}" + +- name: Generate services for cassandra + template: src=service.j2 dest={{mktemp.stdout}}/templates/metrics-{{obj_name}}-svc.yaml + vars: + obj_name: hawkular-{{item}} + ports: + - {name: cql-port, port: 9042, targetPort: cql-port} + - {name: thrift-port, port: 9160, targetPort: thrift-port} + - {name: tcp-port, port: 7000, targetPort: tcp-port} + - {name: ssl-port, port: 7001, targetPort: ssl-port} + selector: + type: hawkular-cassandra + labels: + metrics-infra: hawkular-cassandra + name: hawkular-cassandra + headless: "{{ item == 'cassandra-nodes' }}" + with_items: + - cassandra + - cassandra-nodes diff --git a/roles/openshift_metrics/tasks/install_hawkular.yaml b/roles/openshift_metrics/tasks/install_hawkular.yaml new file mode 100644 index 000000000..670396f6e --- /dev/null +++ b/roles/openshift_metrics/tasks/install_hawkular.yaml @@ -0,0 +1,57 @@ +--- +- name: generate hawkular-metrics replication controller + template: + src: hawkular_metrics_rc.j2 + dest: "{{ mktemp.stdout }}/templates/hawkular_metrics_rc.yaml" +- name: generate hawkular-cassandra replication controllers + template: + src: hawkular_cassandra_rc.j2 + dest: "{{ mktemp.stdout }}/templates/hawkular-cassandra-rc{{ item }}.yaml" + vars: + node: "{{ item }}" + master: "{{ (item == '1')|string|lower }}" + with_sequence: count={{ hawkular_cassandra_nodes }} +- name: generate hawkular-cassandra persistent volume claims + template: + src: pvc.j2 + dest: "{{ mktemp.stdout }}/templates/hawkular-cassandra-pvc{{ item }}.yaml" + vars: + obj_name: "{{ hawkular_cassandra_pv_prefix }}-{{ item }}" + labels: + metrics-infra: hawkular-cassandra + access_modes: + - ReadWriteOnce + size: "{{ hawkular_cassandra_pv_size }}" + with_sequence: count={{ hawkular_cassandra_nodes }} + when: hawkular_cassandra_storage_type == 'pv' +- name: generate hawkular-cassandra persistent volume claims (dynamic) + template: + src: pvc.j2 + dest: "{{ mktemp.stdout }}/templates/hawkular-cassandra-pvc{{ item }}.yaml" + vars: + obj_name: "{{ hawkular_cassandra_pv_prefix }}-{{ item }}" + labels: + metrics-infra: hawkular-cassandra + annotations: + volume.alpha.kubernetes.io/storage-class: dynamic + access_modes: + - ReadWriteOnce + size: "{{ hawkular_cassandra_pv_size }}" + with_sequence: count={{ hawkular_cassandra_nodes }} + when: hawkular_cassandra_storage_type == 'dynamic' +- name: generate the hawkular-metrics route + template: + src: route.j2 + dest: "{{ mktemp.stdout }}/templates/hawkular-metrics-route.yaml" + vars: + name: hawkular-metrics + labels: + metrics-infra: hawkular-metrics + host: hawkular-metrics.example.com + to: + kind: Service + name: hawkular-metrics + tls: + termination: reencrypt + destination_ca_certificate: > + {{ hawkular_metrics_secret.results[6].stdout|b64decode }} diff --git a/roles/openshift_metrics/tasks/install_heapster.yaml b/roles/openshift_metrics/tasks/install_heapster.yaml new file mode 100644 index 000000000..a8f849a88 --- /dev/null +++ b/roles/openshift_metrics/tasks/install_heapster.yaml @@ -0,0 +1,3 @@ +--- +- name: Generate heapster replication controller + template: src=heapster.j2 dest={{mktemp.stdout}}/templates/metrics-heapster-rc.yaml diff --git a/roles/openshift_metrics/tasks/install_metrics.yaml b/roles/openshift_metrics/tasks/install_metrics.yaml new file mode 100644 index 000000000..34b4a47fe --- /dev/null +++ b/roles/openshift_metrics/tasks/install_metrics.yaml @@ -0,0 +1,17 @@ +--- +# This is the base configuration for installing the other components +- name: Create temp directory for doing work in + command: mktemp -td openshift-metrics-ansible-XXXXXX + register: mktemp + changed_when: False + +- debug: msg="Created temp dir {{mktemp.stdout}}" + +- name: Create temp directory for all our templates + file: path={{mktemp.stdout}}/templates state=directory mode=0755 + changed_when: False + +- include: generate_serviceaccounts.yaml +- include: generate_services.yaml +- include: generate_certificates.yaml +- include: generate_rolebindings.yaml diff --git a/roles/openshift_metrics/tasks/main.yaml b/roles/openshift_metrics/tasks/main.yaml new file mode 100644 index 000000000..e9a5fbebd --- /dev/null +++ b/roles/openshift_metrics/tasks/main.yaml @@ -0,0 +1,24 @@ +--- +- name: check that hawkular_metrics_hostname is set + fail: msg='the hawkular_metrics_hostname variable is required' + when: "{{ hawkular_metrics_hostname is not defined }}" +- name: check the value of hawkular_cassandra_storage_type + fail: + msg: > + hawkular_cassandra_storage_type ({{ hawkular_cassandra_storage_type }}) + is invalid, must be one of: emptydir, pv, dynamic + when: hawkular_cassandra_storage_type not in hawkular_cassandra_storage_types +- name: Install Metrics + include: "{{ role_path }}/tasks/install_{{ include_file }}.yaml" + with_items: + - metrics + - heapster + - hawkular + loop_control: + loop_var: include_file +- name: create objects + command: > + {{ openshift.common.client_binary }} -n '{{ metrics_project }}' + apply -f {{ item }} + with_fileglob: + - "{{ mktemp.stdout }}/templates/*.yaml" diff --git a/roles/openshift_metrics/tasks/setup_certificate.yaml b/roles/openshift_metrics/tasks/setup_certificate.yaml new file mode 100644 index 000000000..46ac4ea7f --- /dev/null +++ b/roles/openshift_metrics/tasks/setup_certificate.yaml @@ -0,0 +1,50 @@ +--- +- name: generate {{ component }} keys + command: > + {{ openshift.common.admin_binary }} ca create-server-cert + --key='{{ mktemp.stdout }}/certs/{{ component }}.key' + --cert='{{ mktemp.stdout }}/certs/{{ component }}.crt' + --hostnames='{{ hostnames }}' + --signer-cert='{{ mktemp.stdout }}/certs/ca.crt' + --signer-key='{{ mktemp.stdout }}/certs/ca.key' + --signer-serial='{{ mktemp.stdout }}/certs/ca.serial.txt' +- name: generate {{ component }} certificate + shell: > + cat + '{{ mktemp.stdout|quote }}/certs/{{ component|quote }}.key' + '{{ mktemp.stdout|quote }}/certs/{{ component|quote }}.crt' + > '{{ mktemp.stdout|quote }}/certs/{{ component|quote }}.pem' +- name: generate random password for the {{ component }} keystore + shell: tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c15 + register: keystore_pwd +- name: create the password file for {{ component }} + shell: > + echo '{{ keystore_pwd.stdout|quote }}' + > '{{ mktemp.stdout }}/certs/{{ component|quote }}-keystore.pwd' +- name: create the {{ component }} pkcs12 from the pem file + command: > + openssl pkcs12 -export + -in '{{ mktemp.stdout }}/certs/{{ component }}.pem' + -out '{{ mktemp.stdout }}/certs/{{ component }}.pkcs12' + -name '{{ component }}' -noiter -nomaciter + -password 'pass:{{ keystore_pwd.stdout }}' +- name: create the {{ component }} keystore from the pkcs12 file + command: > + keytool -v -importkeystore + -srckeystore '{{ mktemp.stdout }}/certs/{{ component }}.pkcs12' + -srcstoretype PKCS12 + -destkeystore '{{ mktemp.stdout }}/certs/{{ component }}.keystore' + -deststoretype JKS + -deststorepass '{{ keystore_pwd.stdout }}' + -srcstorepass '{{ keystore_pwd.stdout }}' +- name: create the {{ component }} certificate + command: > + keytool -noprompt -export + -alias '{{ component }}' + -file '{{ mktemp.stdout }}/certs/{{ component }}.cert' + -keystore '{{ mktemp.stdout }}/certs/{{ component }}.keystore' + -storepass '{{ keystore_pwd.stdout }}' +- name: generate random password for the {{ component }} truststore + shell: > + tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c15 + > '{{ mktemp.stdout }}/certs/{{ component|quote }}-truststore.pwd' diff --git a/roles/openshift_metrics/templates/hawkular_cassandra_rc.j2 b/roles/openshift_metrics/templates/hawkular_cassandra_rc.j2 new file mode 100644 index 000000000..bb8866263 --- /dev/null +++ b/roles/openshift_metrics/templates/hawkular_cassandra_rc.j2 @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: hawkular-cassandra-{{ node }} + labels: + metrics-infra: hawkular-cassandra + name: hawkular-cassandra + type: hawkular-cassandra +spec: + selector: + name: hawkular-cassandra-{{ node }} + replicas: 1 + template: + version: v1 + metadata: + labels: + metrics-infra: hawkular-cassandra + name: hawkular-cassandra-{{ node }} + type: hawkular-cassandra + spec: + serviceAccount: cassandra + containers: + - image: "{{ image_prefix }}metrics-cassandra:{{ image_version }}" + name: hawkular-cassandra-{{ node }} + ports: + - name: cql-port + containerPort: 9042 + - name: thift-port + containerPort: 9160 + - name: tcp-port + containerPort: 7000 + - name: ssl-port + containerPort: 7001 + command: + - "/opt/apache-cassandra/bin/cassandra-docker.sh" + - "--cluster_name=hawkular-metrics" + - "--data_volume=/cassandra_data" + - "--internode_encryption=all" + - "--require_node_auth=true" + - "--enable_client_encryption=true" + - "--require_client_auth=true" + - "--keystore_file=/secret/cassandra.keystore" + - "--keystore_password_file=/secret/cassandra.keystore.password" + - "--truststore_file=/secret/cassandra.truststore" + - "--truststore_password_file=/secret/cassandra.truststore.password" + - "--cassandra_pem_file=/secret/cassandra.pem" + env: + - name: CASSANDRA_MASTER + value: "{{ master }}" + - name: CASSANDRA_DATA_VOLUME + value: "/cassandra_data" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MEMORY_LIMIT + valueFrom: + resourceFieldRef: + resource: limits.memory + - name: CPU_LIMIT + valueFrom: + resourceFieldRef: + resource: limits.cpu + divisor: 1m + volumeMounts: + - name: cassandra-data + mountPath: "/cassandra_data" + - name: hawkular-cassandra-secrets + mountPath: "/secret" + readinessProbe: + exec: + command: + - "/opt/apache-cassandra/bin/cassandra-docker-ready.sh" + lifecycle: + preStop: + exec: + command: + - "/opt/apache-cassandra/bin/cassandra-prestop.sh" + postStart: + exec: + command: + - "/opt/apache-cassandra/bin/cassandra-poststart.sh" + terminationGracePeriodSeconds: 1800 + volumes: + - name: cassandra-data +{% if hawkular_cassandra_storage_type == 'emptydir' %} + emptyDir: {} +{% else %} + persistentVolumeClaim: + claimName: "{{ hawkular_cassandra_pv_prefix }}-{{ node }}" +{% endif %} + - name: hawkular-cassandra-secrets + secret: + secretName: hawkular-cassandra-secrets diff --git a/roles/openshift_metrics/templates/hawkular_metrics_rc.j2 b/roles/openshift_metrics/templates/hawkular_metrics_rc.j2 new file mode 100644 index 000000000..bcfe9dc84 --- /dev/null +++ b/roles/openshift_metrics/templates/hawkular_metrics_rc.j2 @@ -0,0 +1,88 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: hawkular-metrics + labels: + metrics-infra: hawkular-metrics + name: hawkular-metrics +spec: + selector: + name: hawkular-metrics + replicas: 1 + template: + version: v1 + metadata: + labels: + metrics-infra: hawkular-metrics + name: hawkular-metrics + spec: + serviceAccount: hawkular + containers: + - image: {{image_prefix}}metrics-hawkular-metrics:{{image_version}} + name: hawkular-metrics + ports: + - name: http-endpoint + containerPort: 8080 + - name: https-endpoint + containerPort: 8443 + - name: ping + containerPort: 8888 + command: + - "/opt/hawkular/scripts/hawkular-metrics-wrapper.sh" + - "-b" + - 0.0.0.0 + - "-Dhawkular.metrics.cassandra.nodes=hawkular-cassandra" + - "-Dhawkular.metrics.cassandra.use-ssl" + - "-Dhawkular.metrics.openshift.auth-methods=openshift-oauth,htpasswd" + - "-Dhawkular.metrics.openshift.htpasswd-file=/secrets/hawkular-metrics.htpasswd.file" + - "-Dhawkular.metrics.allowed-cors-access-control-allow-headers=authorization" + - "-Dhawkular.metrics.default-ttl={{metrics_duration}}" + - "-Dhawkular-alerts.cassandra-nodes=hawkular-cassandra" + - "-Dhawkular-alerts.cassandra-use-ssl" + - "-Dhawkular.alerts.openshift.auth-methods=openshift-oauth,htpasswd" + - "-Dhawkular.alerts.openshift.htpasswd-file=/secrets/hawkular-metrics.htpasswd.file" + - "-Dhawkular.alerts.allowed-cors-access-control-allow-headers=authorization" + - "-Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true" + - "-Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true" + - "-DKUBERNETES_MASTER_URL={{master_url}}" + - "-DUSER_WRITE_ACCESS={{hawkular_user_write_access}}" + - "--hmw.keystore=/secrets/hawkular-metrics.keystore" + - "--hmw.truststore=/secrets/hawkular-metrics.truststore" + - "--hmw.keystore_password_file=/secrets/hawkular-metrics.keystore.password" + - "--hmw.truststore_password_file=/secrets/hawkular-metrics.truststore.password" + - "--hmw.jgroups_keystore=/secrets/hawkular-metrics.jgroups.keystore" + - "--hmw.jgroups_keystore_password_file=/secrets/hawkular-metrics.jgroups.keystore.password" + - "--hmw.jgroups_alias_file=/secrets/hawkular-metrics.jgroups.alias" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MASTER_URL + value: "{{ master_url }}" + - name: OPENSHIFT_KUBE_PING_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPENSHIFT_KUBE_PING_LABELS + value: "metrics-infra=hawkular-metrics,name=hawkular-metrics" + volumeMounts: + - name: hawkular-metrics-secrets + mountPath: "/secrets" + - name: hawkular-metrics-client-secrets + mountPath: "/client-secrets" + readinessProbe: + exec: + command: + - "/opt/hawkular/scripts/hawkular-metrics-readiness.py" + livenessProbe: + exec: + command: + - "/opt/hawkular/scripts/hawkular-metrics-liveness.py" + volumes: + - name: hawkular-metrics-secrets + secret: + secretName: hawkular-metrics-secrets + - name: hawkular-metrics-client-secrets + secret: + secretName: hawkular-metrics-account diff --git a/roles/openshift_metrics/templates/heapster.j2 b/roles/openshift_metrics/templates/heapster.j2 new file mode 100644 index 000000000..779be0145 --- /dev/null +++ b/roles/openshift_metrics/templates/heapster.j2 @@ -0,0 +1,66 @@ +apiVersion: "v1" +kind: "ReplicationController" +metadata: + name: heapster + labels: + metrics-infra: heapster + name: heapster +spec: + selector: + name: heapster + replicas: 1 + template: + version: v1 + metadata: + name: heapster + labels: + metrics-infra: heapster + name: heapster + spec: + serviceAccountName: heapster + containers: + - name: heapster + image: {{image_prefix}}metrics-heapster:{{image_version}} + ports: + - containerPort: 8082 + name: "http-endpoint" + command: + - "heapster-wrapper.sh" + - "--wrapper.allowed_users_file=/secrets/heapster.allowed-users" + - "--source=kubernetes:{{master_url}}?useServiceAccount=true&kubeletHttps=true&kubeletPort=10250" + - "--tls_cert=/secrets/heapster.cert" + - "--tls_key=/secrets/heapster.key" + - "--tls_client_ca=/secrets/heapster.client-ca" + - "--allowed_users=%allowed_users%" + - "--metric_resolution={{metrics_resolution}}" +{% if not heapster_standalone %} + - "--wrapper.username_file=/hawkular-account/hawkular-metrics.username" + - "--wrapper.password_file=/hawkular-account/hawkular-metrics.password" + - "--wrapper.endpoint_check=https://hawkular-metrics:443/hawkular/metrics/status" + - "--sink=hawkular:https://hawkular-metrics:443?tenant=_system&labelToTenant=pod_namespace&labelNodeId={{metrics_node_id}}&caCert=/hawkular-cert/hawkular-metrics-ca.certificate&user=%username%&pass=%password%&filter=label(container_name:^system.slice.*|^user.slice)" +{% endif %} + volumeMounts: + - name: heapster-secrets + mountPath: "/secrets" +{% if not heapster_standalone %} + - name: hawkular-metrics-certificate + mountPath: "/hawkular-cert" + - name: hawkular-metrics-account + mountPath: "/hawkular-account" + readinessProbe: + exec: + command: + - "/opt/heapster-readiness.sh" +{% endif %} + volumes: + - name: heapster-secrets + secret: + secretName: heapster-secrets +{% if not heapster_standalone %} + - name: hawkular-metrics-certificate + secret: + secretName: hawkular-metrics-certificate + - name: hawkular-metrics-account + secret: + secretName: hawkular-metrics-account +{% endif %} diff --git a/roles/openshift_metrics/templates/pvc.j2 b/roles/openshift_metrics/templates/pvc.j2 new file mode 100644 index 000000000..8fbfa8b5d --- /dev/null +++ b/roles/openshift_metrics/templates/pvc.j2 @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{obj_name}} +{% if labels is not defined %} + labels: + logging-infra: support +{% elif labels %} + labels: +{% for key, value in labels.iteritems() %} + {{ key }}: {{ value }} +{% endfor %} +{% endif %} +{% if annotations is defined and annotations %} + annotations: +{% for key,value in annotations.iteritems() %} + {{key}}: {{value}} +{% endfor %} +{% endif %} +spec: + accessModes: +{% for mode in access_modes %} + - {{ mode }} +{% endfor %} + resources: + requests: + storage: {{size}} diff --git a/roles/openshift_metrics/templates/rolebinding.j2 b/roles/openshift_metrics/templates/rolebinding.j2 new file mode 100644 index 000000000..5230f0780 --- /dev/null +++ b/roles/openshift_metrics/templates/rolebinding.j2 @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: {% if cluster is defined and cluster %}Cluster{% endif %}RoleBinding +metadata: + name: {{obj_name}} +{% if labels is defined %} + labels: +{% for k, v in labels.iteritems() %} + {{ k }}: {{ v }} +{% endfor %} +{% endif %} +roleRef: +{% if 'kind' in roleRef %} + kind: {{ roleRef.kind }} +{% endif %} + name: {{ roleRef.name }} +subjects: +{% for sub in subjects %} + - kind: {{ sub.kind }} + name: {{ sub.name }} +{% if 'namespace' in sub %} + namespace: {{ sub.namespace }} +{% endif %} +{% endfor %} diff --git a/roles/openshift_metrics/templates/route.j2 b/roles/openshift_metrics/templates/route.j2 new file mode 100644 index 000000000..a720c4959 --- /dev/null +++ b/roles/openshift_metrics/templates/route.j2 @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Route +metadata: + name: {{ name }} +{% if labels is defined and labels %} + labels: +{% for k, v in labels.iteritems() %} + {{ k }}: {{ v }} +{% endfor %} +{% endif %} +spec: + host: {{ host }} + to: + kind: {{ to.kind }} + name: {{ to.name }} +{% if tls is defined %} + tls: + termination: {{ tls.termination }} +{% if tls.termination == 'reencrypt' %} + destinationCACertificate: | +{{ tls.destination_ca_certificate|indent(6, true) }} +{% endif %} +{% endif %} diff --git a/roles/openshift_metrics/templates/secret.j2 b/roles/openshift_metrics/templates/secret.j2 new file mode 100644 index 000000000..370890c7d --- /dev/null +++ b/roles/openshift_metrics/templates/secret.j2 @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "{{ name }}" + labels: +{% for k, v in labels.iteritems() %} + {{ k }}: {{ v }} +{% endfor %} +data: +{% for k, v in data.iteritems() %} + {{ k }}: {{ v }} +{% endfor %} diff --git a/roles/openshift_metrics/templates/service.j2 b/roles/openshift_metrics/templates/service.j2 new file mode 100644 index 000000000..8df89127b --- /dev/null +++ b/roles/openshift_metrics/templates/service.j2 @@ -0,0 +1,32 @@ +apiVersion: "v1" +kind: "Service" +metadata: + name: "{{obj_name}}" +{% if labels is defined%} + labels: +{% for key, value in labels.iteritems() %} + {{key}}: {{value}} +{% endfor %} +{% endif %} +spec: +{% if headless is defined and headless %} + portalIP: None + clusterIP: None +{% endif %} + ports: +{% for port in ports %} + - +{% for key, value in port.iteritems() %} + {{key}}: {{value}} +{% endfor %} +{% if port.targetPort is undefined %} + clusterIP: "None" +{% endif %} +{% endfor %} +{% if service_targetPort is defined %} + targetPort: {{service_targetPort}} +{% endif %} + selector: + {% for key, value in selector.iteritems() %} + {{key}}: {{value}} + {% endfor %} diff --git a/roles/openshift_metrics/templates/serviceaccount.j2 b/roles/openshift_metrics/templates/serviceaccount.j2 new file mode 100644 index 000000000..b22acc594 --- /dev/null +++ b/roles/openshift_metrics/templates/serviceaccount.j2 @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{obj_name}} +{% if labels is defined%} + labels: +{% for key, value in labels.iteritems() %} + {{key}}: {{value}} +{% endfor %} +{% endif %} +{% if secrets is defined %} +secrets: +{% for name in secrets %} +- name: {{ name }} +{% endfor %} +{% endif %} diff --git a/roles/openshift_metrics/vars/main.yaml b/roles/openshift_metrics/vars/main.yaml new file mode 100644 index 000000000..eb02a87fd --- /dev/null +++ b/roles/openshift_metrics/vars/main.yaml @@ -0,0 +1,4 @@ +hawkular_cassandra_storage_types: +- emptydir +- pv +- dynamic -- cgit v1.2.3