summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--inventory/byo/hosts.example3
-rw-r--r--playbooks/aws/openshift-cluster/tasks/launch_instances.yml13
-rw-r--r--playbooks/common/openshift-master/config.yml2
-rw-r--r--playbooks/libvirt/openshift-cluster/launch.yml8
-rw-r--r--playbooks/libvirt/openshift-cluster/tasks/launch_instances.yml5
-rw-r--r--roles/etcd_ca/tasks/main.yml1
-rw-r--r--roles/openshift_cluster_metrics/README.md36
-rw-r--r--roles/openshift_cluster_metrics/files/cluster-metrics/grafana.yaml53
-rw-r--r--roles/openshift_cluster_metrics/files/cluster-metrics/heapster-serviceaccount.yaml4
-rw-r--r--roles/openshift_cluster_metrics/files/cluster-metrics/heapster.yaml30
-rw-r--r--roles/openshift_cluster_metrics/files/cluster-metrics/influxdb.yaml67
-rw-r--r--roles/openshift_cluster_metrics/tasks/main.yml50
-rwxr-xr-xroles/openshift_facts/library/openshift_facts.py19
-rw-r--r--roles/openshift_master/templates/master.yaml.v1.j210
-rw-r--r--roles/openshift_node/defaults/main.yml4
-rw-r--r--roles/openshift_node/templates/node.yaml.v1.j21
-rw-r--r--roles/openshift_node/templates/partials/kubeletArguments.j25
17 files changed, 301 insertions, 10 deletions
diff --git a/inventory/byo/hosts.example b/inventory/byo/hosts.example
index 0f9f2caf9..16b89d458 100644
--- a/inventory/byo/hosts.example
+++ b/inventory/byo/hosts.example
@@ -20,6 +20,9 @@ ansible_ssh_user=root
# deployment type valid values are origin, online and enterprise
deployment_type=enterprise
+# Enable cluster metrics
+#use_cluster_metrics=true
+
# Pre-release registry URL
#oreg_url=rcm-img-docker01.build.eng.bos.redhat.com:5001/openshift3/ose-${component}:${version}
diff --git a/playbooks/aws/openshift-cluster/tasks/launch_instances.yml b/playbooks/aws/openshift-cluster/tasks/launch_instances.yml
index 236d84e74..e9ebc3e02 100644
--- a/playbooks/aws/openshift-cluster/tasks/launch_instances.yml
+++ b/playbooks/aws/openshift-cluster/tasks/launch_instances.yml
@@ -147,6 +147,18 @@
tag_host-type_{{ host_type }}, tag_env-host-type_{{ env_host_type }},
tag_sub-host-type_{{ sub_host_type }}"
+- set_fact:
+ node_label:
+ region: "{{ec2_region}}"
+ type: "{{sub_host_type}}"
+ when: host_type == "node"
+
+- set_fact:
+ node_label:
+ region: "{{ec2_region}}"
+ type: "{{host_type}}"
+ when: host_type != "node"
+
- name: Add new instances groups and variables
add_host:
hostname: "{{ item.0 }}"
@@ -156,6 +168,7 @@
groups: "{{ instance_groups }}"
ec2_private_ip_address: "{{ item.1.private_ip }}"
ec2_ip_address: "{{ item.1.public_ip }}"
+ openshift_node_labels: "{{ node_label }}"
with_together:
- instances
- ec2.instances
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 904ad2dab..acf85fc04 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -215,6 +215,8 @@
- role: openshift_master_cluster
when: openshift_master_ha | bool
- openshift_examples
+ - role: openshift_cluster_metrics
+ when: openshift.common.use_cluster_metrics | bool
# Additional instance config for online deployments
- name: Additional instance config
diff --git a/playbooks/libvirt/openshift-cluster/launch.yml b/playbooks/libvirt/openshift-cluster/launch.yml
index 830f9d216..d3e768de5 100644
--- a/playbooks/libvirt/openshift-cluster/launch.yml
+++ b/playbooks/libvirt/openshift-cluster/launch.yml
@@ -17,6 +17,14 @@
- include: tasks/configure_libvirt.yml
+ - include: ../../common/openshift-cluster/set_etcd_launch_facts_tasks.yml
+ - include: tasks/launch_instances.yml
+ vars:
+ instances: "{{ etcd_names }}"
+ cluster: "{{ cluster_id }}"
+ type: "{{ k8s_type }}"
+ g_sub_host_type: "default"
+
- include: ../../common/openshift-cluster/set_master_launch_facts_tasks.yml
- include: tasks/launch_instances.yml
vars:
diff --git a/playbooks/libvirt/openshift-cluster/tasks/launch_instances.yml b/playbooks/libvirt/openshift-cluster/tasks/launch_instances.yml
index 4cb494056..2a0c90b46 100644
--- a/playbooks/libvirt/openshift-cluster/tasks/launch_instances.yml
+++ b/playbooks/libvirt/openshift-cluster/tasks/launch_instances.yml
@@ -63,8 +63,9 @@
shell: 'virsh -c {{ libvirt_uri }} net-dhcp-leases openshift-ansible | egrep -c ''{{ instances | join("|") }}'''
register: nb_allocated_ips
until: nb_allocated_ips.stdout == '{{ instances | length }}'
- retries: 30
+ retries: 60
delay: 1
+ when: instances | length != 0
- name: Collect IP addresses of the VMs
shell: 'virsh -c {{ libvirt_uri }} net-dhcp-leases openshift-ansible | awk ''$6 == "{{ item }}" {gsub(/\/.*/, "", $5); print $5}'''
@@ -72,7 +73,7 @@
with_items: instances
- set_fact:
- ips: "{{ scratch_ip.results | oo_collect('stdout') }}"
+ ips: "{{ scratch_ip.results | default([]) | oo_collect('stdout') }}"
- name: Add new instances
add_host:
diff --git a/roles/etcd_ca/tasks/main.yml b/roles/etcd_ca/tasks/main.yml
index ab151fe5b..8a266f732 100644
--- a/roles/etcd_ca/tasks/main.yml
+++ b/roles/etcd_ca/tasks/main.yml
@@ -37,6 +37,7 @@
openssl req -config openssl.cnf -newkey rsa:4096
-keyout ca.key -new -out ca.crt -x509 -extensions etcd_v3_ca_self
-batch -nodes -subj /CN=etcd-signer@{{ ansible_date_time.epoch }}
+ -days 365
args:
chdir: "{{ etcd_ca_dir }}"
creates: "{{ etcd_ca_dir }}/ca.crt"
diff --git a/roles/openshift_cluster_metrics/README.md b/roles/openshift_cluster_metrics/README.md
new file mode 100644
index 000000000..9fdfab8e3
--- /dev/null
+++ b/roles/openshift_cluster_metrics/README.md
@@ -0,0 +1,36 @@
+#openshift_cluster_metrics
+
+This role configures Cluster wide metrics. It does setting up three services:
+* Metrics are stored in InfluxDB for querying.
+* Heapster reads all nodes and pods from the master, then connects to eachs node's kubelet to retrieve pod metrics.
+* Grafan allows users to create dashboards of metrics from InfluxDB
+
+## Requirements
+
+Running OpenShift cluster
+
+## Role Variables
+
+```
+# Enable cluster metrics
+use_cluster_metrics=true
+```
+
+## Dependencies
+
+None
+
+## Example Playbook
+
+TODO
+
+## Security Note
+Opening up the read-only port exposes information about the running pods (such as namespace, pod name, labels, etc.) to unauthenticated clients. The requirement to open up this read-only port will be fixed in future versions.
+
+##License
+
+Apache License, Version 2.0
+
+## Author Information
+
+Diego Castro (diego.castro@getupcloud.com)
diff --git a/roles/openshift_cluster_metrics/files/cluster-metrics/grafana.yaml b/roles/openshift_cluster_metrics/files/cluster-metrics/grafana.yaml
new file mode 100644
index 000000000..bff422efc
--- /dev/null
+++ b/roles/openshift_cluster_metrics/files/cluster-metrics/grafana.yaml
@@ -0,0 +1,53 @@
+apiVersion: "v1"
+kind: "List"
+items:
+ -
+ apiVersion: "v1"
+ kind: "Service"
+ metadata:
+ labels:
+ provider: "fabric8"
+ component: "grafana"
+ name: "grafana"
+ spec:
+ ports:
+ -
+ port: 80
+ targetPort: "http"
+ selector:
+ provider: "fabric8"
+ component: "grafana"
+ -
+ apiVersion: "v1"
+ kind: "ReplicationController"
+ metadata:
+ labels:
+ provider: "fabric8"
+ component: "grafana"
+ name: "grafana"
+ spec:
+ replicas: 1
+ selector:
+ provider: "fabric8"
+ component: "grafana"
+ template:
+ metadata:
+ labels:
+ provider: "fabric8"
+ component: "grafana"
+ spec:
+ containers:
+ -
+ env:
+ -
+ name: "INFLUXDB_SERVICE_NAME"
+ value: "INFLUXDB_MONITORING"
+ -
+ name: "GRAFANA_DEFAULT_DASHBOARD"
+ value: "/dashboard/file/kubernetes.json"
+ image: "fabric8/grafana:1.9.1_2"
+ name: "grafana"
+ ports:
+ -
+ containerPort: 3000
+ name: "http" \ No newline at end of file
diff --git a/roles/openshift_cluster_metrics/files/cluster-metrics/heapster-serviceaccount.yaml b/roles/openshift_cluster_metrics/files/cluster-metrics/heapster-serviceaccount.yaml
new file mode 100644
index 000000000..1de2ad699
--- /dev/null
+++ b/roles/openshift_cluster_metrics/files/cluster-metrics/heapster-serviceaccount.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: heapster \ No newline at end of file
diff --git a/roles/openshift_cluster_metrics/files/cluster-metrics/heapster.yaml b/roles/openshift_cluster_metrics/files/cluster-metrics/heapster.yaml
new file mode 100644
index 000000000..83e314074
--- /dev/null
+++ b/roles/openshift_cluster_metrics/files/cluster-metrics/heapster.yaml
@@ -0,0 +1,30 @@
+apiVersion: "v1"
+kind: "List"
+items:
+ -
+ apiVersion: "v1"
+ kind: "ReplicationController"
+ metadata:
+ labels:
+ provider: "fabric8"
+ component: "heapster"
+ name: "heapster"
+ spec:
+ replicas: 1
+ selector:
+ provider: "fabric8"
+ component: "heapster"
+ template:
+ metadata:
+ labels:
+ provider: "fabric8"
+ component: "heapster"
+ spec:
+ containers:
+ -
+ args:
+ - "-source=kubernetes:https://kubernetes.default.svc.cluster.local?auth=&insecure=true&useServiceAccount=true"
+ - "-sink=influxdb:http://influxdb-monitoring.default.svc.cluster.local:8086"
+ image: "kubernetes/heapster:V0.14.2"
+ name: "heapster"
+ serviceAccount: "heapster" \ No newline at end of file
diff --git a/roles/openshift_cluster_metrics/files/cluster-metrics/influxdb.yaml b/roles/openshift_cluster_metrics/files/cluster-metrics/influxdb.yaml
new file mode 100644
index 000000000..6f67c3d7c
--- /dev/null
+++ b/roles/openshift_cluster_metrics/files/cluster-metrics/influxdb.yaml
@@ -0,0 +1,67 @@
+apiVersion: "v1"
+kind: "List"
+items:
+ -
+ apiVersion: "v1"
+ kind: "Service"
+ metadata:
+ labels:
+ provider: "fabric8"
+ component: "influxdb-monitoring"
+ name: "influxdb-monitoring"
+ spec:
+ ports:
+ -
+ port: 8086
+ targetPort: "http"
+ selector:
+ provider: "fabric8"
+ component: "influxdb-monitoring"
+ -
+ apiVersion: "v1"
+ kind: "ReplicationController"
+ metadata:
+ labels:
+ provider: "fabric8"
+ component: "influxdb-monitoring"
+ name: "influxdb-monitoring"
+ spec:
+ replicas: 1
+ selector:
+ provider: "fabric8"
+ component: "influxdb-monitoring"
+ template:
+ metadata:
+ labels:
+ provider: "fabric8"
+ component: "influxdb-monitoring"
+ spec:
+ containers:
+ -
+ env:
+ -
+ name: "PRE_CREATE_DB"
+ value: "k8s;grafana"
+ image: "fabric8/influxdb:0.8.8"
+ name: "influxdb"
+ ports:
+ -
+ containerPort: 8090
+ name: "raft"
+ -
+ containerPort: 8099
+ name: "protobuf"
+ -
+ containerPort: 8083
+ name: "admin"
+ -
+ containerPort: 8086
+ name: "http"
+ volumeMounts:
+ -
+ mountPath: "/data"
+ name: "influxdb-data"
+ volumes:
+ -
+ emptyDir:
+ name: "influxdb-data" \ No newline at end of file
diff --git a/roles/openshift_cluster_metrics/tasks/main.yml b/roles/openshift_cluster_metrics/tasks/main.yml
new file mode 100644
index 000000000..3938aba4c
--- /dev/null
+++ b/roles/openshift_cluster_metrics/tasks/main.yml
@@ -0,0 +1,50 @@
+---
+
+- name: Install cluster metrics templates
+ copy:
+ src: cluster-metrics
+ dest: /etc/openshift/
+
+- name: Create InfluxDB Services
+ command: >
+ {{ openshift.common.client_binary }} create -f
+ /etc/openshift/cluster-metrics/influxdb.yaml
+ register: oex_influxdb_services
+ failed_when: "'already exists' not in oex_influxdb_services.stderr and oex_influxdb_services.rc != 0"
+ changed_when: false
+
+- name: Create Heapster Service Account
+ command: >
+ {{ openshift.common.client_binary }} create -f
+ /etc/openshift/cluster-metrics/heapster-serviceaccount.yaml
+ register: oex_heapster_serviceaccount
+ failed_when: "'already exists' not in oex_heapster_serviceaccount.stderr and oex_heapster_serviceaccount.rc != 0"
+ changed_when: false
+
+- name: Add cluster-reader role to Heapster
+ command: >
+ {{ openshift.common.admin_binary }} policy
+ add-cluster-role-to-user
+ cluster-reader
+ system:serviceaccount:default:heapster
+ register: oex_cluster_header_role
+ register: oex_cluster_header_role
+ failed_when: "'already exists' not in oex_cluster_header_role.stderr and oex_cluster_header_role.rc != 0"
+ changed_when: false
+
+- name: Create Heapster Services
+ command: >
+ {{ openshift.common.client_binary }} create -f
+ /etc/openshift/cluster-metrics/heapster.yaml
+ register: oex_heapster_services
+ failed_when: "'already exists' not in oex_heapster_services.stderr and oex_heapster_services.rc != 0"
+ changed_when: false
+
+- name: Create Grafana Services
+ command: >
+ {{ openshift.common.client_binary }} create -f
+ /etc/openshift/cluster-metrics/grafana.yaml
+ register: oex_grafana_services
+ failed_when: "'already exists' not in oex_grafana_services.stderr and oex_grafana_services.rc != 0"
+ changed_when: false
+
diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py
index 4e0989c5f..c1c4e1b5c 100755
--- a/roles/openshift_facts/library/openshift_facts.py
+++ b/roles/openshift_facts/library/openshift_facts.py
@@ -323,6 +323,24 @@ def set_fluentd_facts_if_unset(facts):
facts['common']['use_fluentd'] = use_fluentd
return facts
+def set_cluster_metrics_facts_if_unset(facts):
+ """ Set cluster metrics facts if not already present in facts dict
+ dict: the facts dict updated with the generated cluster metrics facts if
+ missing
+ Args:
+ facts (dict): existing facts
+ Returns:
+ dict: the facts dict updated with the generated cluster metrics
+ facts if they were not already present
+
+ """
+ if 'common' in facts:
+ deployment_type = facts['common']['deployment_type']
+ if 'use_cluster_metrics' not in facts['common']:
+ use_cluster_metrics = True if deployment_type == 'origin' else False
+ facts['common']['use_cluster_metrics'] = use_cluster_metrics
+ return facts
+
def set_identity_providers_if_unset(facts):
""" Set identity_providers fact if not already present in facts dict
@@ -700,6 +718,7 @@ class OpenShiftFacts(object):
facts['current_config'] = get_current_config(facts)
facts = set_url_facts_if_unset(facts)
facts = set_fluentd_facts_if_unset(facts)
+ facts = set_cluster_metrics_facts_if_unset(facts)
facts = set_identity_providers_if_unset(facts)
facts = set_registry_url_if_unset(facts)
facts = set_sdn_facts_if_unset(facts)
diff --git a/roles/openshift_master/templates/master.yaml.v1.j2 b/roles/openshift_master/templates/master.yaml.v1.j2
index 44567aa22..fff123d0d 100644
--- a/roles/openshift_master/templates/master.yaml.v1.j2
+++ b/roles/openshift_master/templates/master.yaml.v1.j2
@@ -2,9 +2,6 @@ apiLevels:
- v1beta3
- v1
apiVersion: v1
-{% if api_server_args is defined and api_server_args %}
-apiServerArguments: {{ api_server_args }}
-{% endif %}
assetConfig:
logoutURL: ""
masterPublicURL: {{ openshift.master.public_api_url }}
@@ -16,9 +13,6 @@ assetConfig:
keyFile: master.server.key
maxRequestsInFlight: 0
requestTimeoutSeconds: 0
-{% if controller_args is defined and controller_args %}
-controllerArguments: {{ controller_args }}
-{% endif %}
corsAllowedOrigins:
{% for origin in ['127.0.0.1', 'localhost', openshift.common.hostname, openshift.common.ip, openshift.common.public_hostname, openshift.common.public_ip] %}
- {{ origin }}
@@ -74,8 +68,8 @@ kubernetesMasterConfig:
apiLevels:
- v1beta3
- v1
- apiServerArguments: null
- controllerArguments: null
+ apiServerArguments: {{ api_server_args if api_server_args is defined else 'null' }}
+ controllerArguments: {{ controller_args if controller_args is defined else 'null' }}
{# TODO: support overriding masterCount #}
masterCount: 1
masterIP: ""
diff --git a/roles/openshift_node/defaults/main.yml b/roles/openshift_node/defaults/main.yml
index be51195f2..1dbcc4301 100644
--- a/roles/openshift_node/defaults/main.yml
+++ b/roles/openshift_node/defaults/main.yml
@@ -6,3 +6,7 @@ os_firewall_allow:
port: 80/tcp
- service: https
port: 443/tcp
+- service: Openshift kubelet ReadOnlyPort
+ port: 10255/tcp
+- service: Openshift kubelet ReadOnlyPort udp
+ port: 10255/udp
diff --git a/roles/openshift_node/templates/node.yaml.v1.j2 b/roles/openshift_node/templates/node.yaml.v1.j2
index 7b1c0f808..e176e7511 100644
--- a/roles/openshift_node/templates/node.yaml.v1.j2
+++ b/roles/openshift_node/templates/node.yaml.v1.j2
@@ -21,3 +21,4 @@ servingInfo:
clientCA: ca.crt
keyFile: server.key
volumeDirectory: {{ openshift_data_dir }}/openshift.local.volumes
+{% include 'partials/kubeletArguments.j2' %} \ No newline at end of file
diff --git a/roles/openshift_node/templates/partials/kubeletArguments.j2 b/roles/openshift_node/templates/partials/kubeletArguments.j2
new file mode 100644
index 000000000..6c3bd04c5
--- /dev/null
+++ b/roles/openshift_node/templates/partials/kubeletArguments.j2
@@ -0,0 +1,5 @@
+{% if openshift.common.use_cluster_metrics | bool %}
+kubeletArguments:
+ "read-only-port":
+ - "10255"
+{% endif %} \ No newline at end of file