From e9cdb4d014bc20ce680c6019bdaf011f2190114c Mon Sep 17 00:00:00 2001
From: Vishal Patil <vishal.patil@nuagenetworks.net>
Date: Mon, 26 Sep 2016 18:28:07 -0400
Subject: Changes for Nuage HA
---
roles/nuage_common/defaults/main.yaml | 3 +++
roles/nuage_master/meta/main.yml | 1 +
roles/nuage_master/templates/nuage-openshift-monitor.j2 | 9 ++++++++-
roles/nuage_master/vars/main.yaml | 1 -
roles/openshift_loadbalancer_facts/meta/main.yml | 3 ++-
roles/openshift_loadbalancer_facts/tasks/main.yml | 16 ++++++++++++++++
6 files changed, 30 insertions(+), 3 deletions(-)
diff --git a/roles/nuage_common/defaults/main.yaml b/roles/nuage_common/defaults/main.yaml
index d285bdfa3..9b777213e 100644
--- a/roles/nuage_common/defaults/main.yaml
+++ b/roles/nuage_common/defaults/main.yaml
@@ -8,3 +8,6 @@ nuage_ca_serial: "{{ nuage_ca_dir }}/nuageMonCA.serial.txt"
nuage_master_mon_dir: /usr/share/nuage-openshift-monitor
nuage_node_plugin_dir: /usr/share/vsp-openshift
+
+nuage_mon_rest_server_port: "{{ nuage_openshift_monitor_rest_server_port | default('9443') }}"
+
diff --git a/roles/nuage_master/meta/main.yml b/roles/nuage_master/meta/main.yml
index fdead100c..d56529b4d 100644
--- a/roles/nuage_master/meta/main.yml
+++ b/roles/nuage_master/meta/main.yml
@@ -14,6 +14,7 @@ galaxy_info:
- system
dependencies:
- role: nuage_ca
+- role: nuage_common
- role: os_firewall
os_firewall_allow:
- service: openshift-monitor
diff --git a/roles/nuage_master/templates/nuage-openshift-monitor.j2 b/roles/nuage_master/templates/nuage-openshift-monitor.j2
index 075de9d9e..c663aa0fa 100644
--- a/roles/nuage_master/templates/nuage-openshift-monitor.j2
+++ b/roles/nuage_master/templates/nuage-openshift-monitor.j2
@@ -30,4 +30,11 @@ logLevel: {{ nuage_mon_log_level }}
# Parameters related to the nuage monitor REST server
nuageMonServer:
URL: {{ nuage_mon_rest_server_url }}
- certificateDirectory: {{ cert_output_dir }}
+ certificateDirectory: {{ cert_output_dir }}
+# etcd config required for HA
+etcdClientConfig:
+ ca: {{ etcd_ca_dir }}/{{ "ca.crt" if (openshift.master.embedded_etcd | bool) else "master.etcd-ca.crt" }}
+ certFile: {{ etcd_conf_dir }}/master.etcd-client.crt
+ keyFile: {{ etcd_conf_dir }}/master.etcd-client.key
+ urls:
+ - {{ openshift.common.hostname }}:{{ openshift.master.etcd_port }}
diff --git a/roles/nuage_master/vars/main.yaml b/roles/nuage_master/vars/main.yaml
index 92e716a45..b395eba99 100644
--- a/roles/nuage_master/vars/main.yaml
+++ b/roles/nuage_master/vars/main.yaml
@@ -8,7 +8,6 @@ cert_output_dir: /usr/share/nuage-openshift-monitor
kube_config: /usr/share/nuage-openshift-monitor/nuage.kubeconfig
kubemon_yaml: /usr/share/nuage-openshift-monitor/nuage-openshift-monitor.yaml
master_config_yaml: "{{ openshift_master_config_dir }}/master-config.yaml"
-nuage_mon_rest_server_port: "{{ nuage_openshift_monitor_rest_server_port | default('9443') }}"
nuage_mon_rest_server_url: "0.0.0.0:{{ nuage_mon_rest_server_port }}"
nuage_mon_rest_server_logdir: "{{ nuage_openshift_monitor_log_dir | default('/var/log/nuage-openshift-monitor') }}"
nuage_mon_log_level: "{{ nuage_openshift_monitor_log_level | default('3') }}"
diff --git a/roles/openshift_loadbalancer_facts/meta/main.yml b/roles/openshift_loadbalancer_facts/meta/main.yml
index 4c5b6552b..46959355b 100644
--- a/roles/openshift_loadbalancer_facts/meta/main.yml
+++ b/roles/openshift_loadbalancer_facts/meta/main.yml
@@ -10,4 +10,5 @@ galaxy_info:
versions:
- 7
dependencies:
-- role: openshift_facts
+ - role: openshift_facts
+ - role: nuage_common
diff --git a/roles/openshift_loadbalancer_facts/tasks/main.yml b/roles/openshift_loadbalancer_facts/tasks/main.yml
index dc244c0be..5936ce5ba 100644
--- a/roles/openshift_loadbalancer_facts/tasks/main.yml
+++ b/roles/openshift_loadbalancer_facts/tasks/main.yml
@@ -20,6 +20,14 @@
binds:
- "*:{{ openshift.loadbalancer.frontend_port }}"
default_backend: atomic-openshift-api
+ - name: nuage-monitor
+ mode: tcp
+ options:
+ - tcplog
+ binds:
+ - "*:{{ nuage_mon_rest_server_port }}"
+ default_backend: nuage-monitor
+ when: openshift.common.use_nuage | bool
backends:
- name: atomic-openshift-api
mode: tcp
@@ -28,3 +36,11 @@
servers: "{{ hostvars
| oo_select_keys(groups['oo_masters'])
| oo_haproxy_backend_masters(openshift.loadbalancer.frontend_port) }}"
+ - name: nuage-monitor
+ mode: tcp
+ option: tcplog
+ balance: source
+ servers: "{{ hostvars
+ | oo_select_keys(groups['oo_masters'])
+ | oo_haproxy_backend_masters(nuage_mon_rest_server_port) }}"
+ when: openshift.common.use_nuage | bool
--
cgit v1.2.3
From b0ddb188ef1dc8484f4d9e1f7ae58dcd4ac6a299 Mon Sep 17 00:00:00 2001
From: Vishal Patil <vishal.patil@nuagenetworks.net>
Date: Thu, 29 Sep 2016 14:23:32 -0400
Subject: Addressed review comments
---
roles/nuage_master/meta/main.yml | 13 +++++++------
roles/nuage_master/templates/nuage-openshift-monitor.j2 | 8 ++++----
2 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/roles/nuage_master/meta/main.yml b/roles/nuage_master/meta/main.yml
index d56529b4d..51b89fbf6 100644
--- a/roles/nuage_master/meta/main.yml
+++ b/roles/nuage_master/meta/main.yml
@@ -13,9 +13,10 @@ galaxy_info:
- cloud
- system
dependencies:
-- role: nuage_ca
-- role: nuage_common
-- role: os_firewall
- os_firewall_allow:
- - service: openshift-monitor
- port: "{{ nuage_mon_rest_server_port }}/tcp"
+ - role: nuage_ca
+ - role: nuage_common
+ - role: openshift_etcd_client_certificates
+ - role: os_firewall
+ os_firewall_allow:
+ - service: openshift-monitor
+ port: "{{ nuage_mon_rest_server_port }}/tcp"
diff --git a/roles/nuage_master/templates/nuage-openshift-monitor.j2 b/roles/nuage_master/templates/nuage-openshift-monitor.j2
index c663aa0fa..b2539517b 100644
--- a/roles/nuage_master/templates/nuage-openshift-monitor.j2
+++ b/roles/nuage_master/templates/nuage-openshift-monitor.j2
@@ -33,8 +33,8 @@ nuageMonServer:
certificateDirectory: {{ cert_output_dir }}
# etcd config required for HA
etcdClientConfig:
- ca: {{ etcd_ca_dir }}/{{ "ca.crt" if (openshift.master.embedded_etcd | bool) else "master.etcd-ca.crt" }}
- certFile: {{ etcd_conf_dir }}/master.etcd-client.crt
- keyFile: {{ etcd_conf_dir }}/master.etcd-client.key
+ ca: {{ openshift_master_config_dir }}/{{ "ca.crt" if (openshift.master.embedded_etcd | bool) else "master.etcd-ca.crt" }}
+ certFile: {{ openshift_master_config_dir }}/master.etcd-client.crt
+ keyFile: {{ openshift_master_config_dir }}/master.etcd-client.key
urls:
- - {{ openshift.common.hostname }}:{{ openshift.master.etcd_port }}
+ - {{ openshift.master.etcd_urls }}
--
cgit v1.2.3
From e297a8c887c6fd1ba880a6977fdfe50a3a1ea2ee Mon Sep 17 00:00:00 2001
From: Andrew Butcher <abutcher@redhat.com>
Date: Mon, 3 Oct 2016 11:28:44 -0400
Subject: Filterize haproxy frontends/backends and add method for providing
additional frontends/backends.
---
filter_plugins/oo_filters.py | 32 ++++++++++++++
playbooks/common/openshift-loadbalancer/config.yml | 11 +++++
roles/openshift_facts/library/openshift_facts.py | 8 ----
roles/openshift_loadbalancer/README.md | 51 +++++++++++++++++++---
roles/openshift_loadbalancer/meta/main.yml | 4 +-
roles/openshift_loadbalancer/tasks/main.yml | 4 +-
.../templates/haproxy.cfg.j2 | 8 ++--
roles/openshift_loadbalancer_facts/README.md | 34 ---------------
roles/openshift_loadbalancer_facts/meta/main.yml | 14 ------
roles/openshift_loadbalancer_facts/tasks/main.yml | 46 -------------------
10 files changed, 96 insertions(+), 116 deletions(-)
delete mode 100644 roles/openshift_loadbalancer_facts/README.md
delete mode 100644 roles/openshift_loadbalancer_facts/meta/main.yml
delete mode 100644 roles/openshift_loadbalancer_facts/tasks/main.yml
diff --git a/filter_plugins/oo_filters.py b/filter_plugins/oo_filters.py
index 1c12f2e07..8bb1c8de2 100644
--- a/filter_plugins/oo_filters.py
+++ b/filter_plugins/oo_filters.py
@@ -7,6 +7,7 @@ Custom filters for use in openshift-ansible
from ansible import errors
from collections import Mapping
+from distutils.util import strtobool
from distutils.version import LooseVersion
from operator import itemgetter
import OpenSSL.crypto
@@ -850,6 +851,35 @@ class FilterModule(object):
# netloc wasn't parsed, assume url was missing scheme and path
return parse_result.path
+ @staticmethod
+ def oo_openshift_loadbalancer_frontends(api_port, servers_hostvars, use_nuage=False, nuage_rest_port=None):
+ loadbalancer_frontends = [{'name': 'atomic-openshift-api',
+ 'mode': 'tcp',
+ 'options': ['tcplog'],
+ 'binds': ["*:{0}".format(api_port)],
+ 'default_backend': 'atomic-openshift-api'}]
+ if bool(strtobool(str(use_nuage))) and nuage_rest_port is not None:
+ loadbalancer_frontends.append({'name': 'nuage-monitor',
+ 'mode': 'tcp',
+ 'options': ['tcplog'],
+ 'binds': ["*:{0}".format(nuage_rest_port)],
+ 'default_backend': 'nuage-monitor'})
+ return loadbalancer_frontends
+
+ @staticmethod
+ def oo_openshift_loadbalancer_backends(api_port, servers_hostvars, use_nuage=False, nuage_rest_port=None):
+ loadbalancer_backends = [{'name': 'atomic-openshift-api',
+ 'mode': 'tcp',
+ 'option': 'tcplog',
+ 'balance': 'source',
+ 'servers': FilterModule.oo_haproxy_backend_masters(servers_hostvars, api_port)}]
+ if bool(strtobool(str(use_nuage))) and nuage_rest_port is not None:
+ loadbalancer_backends.append({'name': 'nuage-monitor',
+ 'mode': 'tcp',
+ 'option': 'tcplog',
+ 'balance': 'source',
+ 'servers': FilterModule.oo_haproxy_backend_masters(servers_hostvars, nuage_rest_port)})
+ return loadbalancer_backends
def filters(self):
""" returns a mapping of filters to methods """
@@ -883,4 +913,6 @@ class FilterModule(object):
"oo_merge_dicts": self.oo_merge_dicts,
"oo_hostname_from_url": self.oo_hostname_from_url,
"oo_merge_hostvars": self.oo_merge_hostvars,
+ "oo_openshift_loadbalancer_frontends": self.oo_openshift_loadbalancer_frontends,
+ "oo_openshift_loadbalancer_backends": self.oo_openshift_loadbalancer_backends
}
diff --git a/playbooks/common/openshift-loadbalancer/config.yml b/playbooks/common/openshift-loadbalancer/config.yml
index f4392173a..e3567552e 100644
--- a/playbooks/common/openshift-loadbalancer/config.yml
+++ b/playbooks/common/openshift-loadbalancer/config.yml
@@ -1,5 +1,16 @@
---
- name: Configure load balancers
hosts: oo_lb_to_config
+ vars:
+ openshift_loadbalancer_frontends: "{{ (openshift_master_api_port | default(8843)
+ | oo_openshift_loadbalancer_frontends(hostvars | oo_select_keys(groups['oo_masters']),
+ openshift_use_nuage | default(false),
+ nuage_mon_rest_server_port | default(none)))
+ + openshift_loadbalancer_additional_frontends | default([]) }}"
+ openshift_loadbalancer_backends: "{{ (openshift_master_api_port | default(8843)
+ | oo_openshift_loadbalancer_backends(hostvars | oo_select_keys(groups['oo_masters']),
+ openshift_use_nuage | default(false),
+ nuage_mon_rest_server_port | default(none)))
+ + openshift_loadbalancer_additional_backends | default([]) }}"
roles:
- role: openshift_loadbalancer
diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py
index b2d007ec9..6c10e856a 100755
--- a/roles/openshift_facts/library/openshift_facts.py
+++ b/roles/openshift_facts/library/openshift_facts.py
@@ -1600,7 +1600,6 @@ class OpenShiftFacts(object):
'docker',
'etcd',
'hosted',
- 'loadbalancer',
'master',
'node']
@@ -1840,13 +1839,6 @@ class OpenShiftFacts(object):
router=dict()
)
- if 'loadbalancer' in roles:
- loadbalancer = dict(frontend_port='8443',
- default_maxconn='20000',
- global_maxconn='20000',
- limit_nofile='100000')
- defaults['loadbalancer'] = loadbalancer
-
return defaults
def guess_host_provider(self):
diff --git a/roles/openshift_loadbalancer/README.md b/roles/openshift_loadbalancer/README.md
index 81fc282be..03e837e46 100644
--- a/roles/openshift_loadbalancer/README.md
+++ b/roles/openshift_loadbalancer/README.md
@@ -1,27 +1,68 @@
OpenShift HAProxy Loadbalancer
==============================
-TODO
+OpenShift HaProxy Loadbalancer Configuration
Requirements
------------
-TODO
+This role is intended to be applied to the [lb] host group which is
+separate from OpenShift infrastructure components.
+
+This role is not re-entrant. All haproxy configuration lives in a single file.
Role Variables
--------------
-TODO
+From this role:
+
+| Name | Default value | |
+|----------------------------------------|---------------|-------------------------------------------------------|
+| openshift_loadbalancer_limit_nofile | 100000 | Limit number of open files. |
+| openshift_loadbalancer_global_maxconn | 20000 | Maximum per-process number of concurrent connections. |
+| openshift_loadbalancer_default_maxconn | 20000 | Maximum per-process number of concurrent connections. |
+| openshift_loadbalancer_frontends | none | List of frontends. See example below. |
+| openshift_loadbalancer_backends | none | List of backends. See example below. |
Dependencies
------------
-TODO
+* openshift_facts
+* os_firewall
+* openshift_repos
Example Playbook
----------------
-TODO
+```
+- name: Configure loadbalancer hosts
+ hosts: lb
+ roles:
+ - role: openshift_loadbalancer
+ openshift_loadbalancer_frontends:
+ - name: atomic-openshift-api
+ mode: tcp
+ options:
+ - tcplog
+ binds:
+ - "*:8443"
+ default_backend: atomic-openshift-api
+ openshift_loadbalancer_backends:
+ - name: atomic-openshift-api
+ mode: tcp
+ option: tcplog
+ balance: source
+ servers:
+ - name: master1
+ address: "192.168.122.221:8443"
+ opts: check
+ - name: master2
+ address: "192.168.122.222:8443"
+ opts: check
+ - name: master3
+ address: "192.168.122.223:8443"
+ opts: check
+```
License
-------
diff --git a/roles/openshift_loadbalancer/meta/main.yml b/roles/openshift_loadbalancer/meta/main.yml
index ed846a1ba..e1d78cfd0 100644
--- a/roles/openshift_loadbalancer/meta/main.yml
+++ b/roles/openshift_loadbalancer/meta/main.yml
@@ -10,11 +10,11 @@ galaxy_info:
versions:
- 7
dependencies:
-- role: openshift_loadbalancer_facts
+- role: openshift_facts
- role: os_firewall
os_firewall_allow:
- service: haproxy stats
port: "9000/tcp"
- service: haproxy balance
- port: "{{ openshift.loadbalancer.frontend_port }}/tcp"
+ port: "{{ openshift_master_api_port | default(8443) }}/tcp"
- role: openshift_repos
diff --git a/roles/openshift_loadbalancer/tasks/main.yml b/roles/openshift_loadbalancer/tasks/main.yml
index 03a7c0e4a..bb4982e2d 100644
--- a/roles/openshift_loadbalancer/tasks/main.yml
+++ b/roles/openshift_loadbalancer/tasks/main.yml
@@ -7,15 +7,13 @@
file:
path: /etc/systemd/system/haproxy.service.d
state: directory
- when: "'limit_nofile' in openshift.loadbalancer"
- name: Configure the nofile limits for haproxy
ini_file:
dest: /etc/systemd/system/haproxy.service.d/limits.conf
section: Service
option: LimitNOFILE
- value: "{{ openshift.loadbalancer.limit_nofile }}"
- when: "'limit_nofile' in openshift.loadbalancer"
+ value: "{{ openshift_loadbalancer_limit_nofile | default(100000) }}"
notify: restart haproxy
register: nofile_limit_result
diff --git a/roles/openshift_loadbalancer/templates/haproxy.cfg.j2 b/roles/openshift_loadbalancer/templates/haproxy.cfg.j2
index b9a279f5f..79e695001 100644
--- a/roles/openshift_loadbalancer/templates/haproxy.cfg.j2
+++ b/roles/openshift_loadbalancer/templates/haproxy.cfg.j2
@@ -3,7 +3,7 @@
global
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
- maxconn {{ openshift.loadbalancer.global_maxconn }}
+ maxconn {{ openshift_loadbalancer_global_maxconn | default(20000) }}
user haproxy
group haproxy
daemon
@@ -32,14 +32,14 @@ defaults
timeout server 300s
timeout http-keep-alive 10s
timeout check 10s
- maxconn {{ openshift.loadbalancer.default_maxconn }}
+ maxconn {{ openshift_loadbalancer_default_maxconn | default(20000) }}
listen stats :9000
mode http
stats enable
stats uri /
-{% for frontend in openshift.loadbalancer.frontends %}
+{% for frontend in openshift_loadbalancer_frontends %}
frontend {{ frontend.name }}
{% for bind in frontend.binds %}
bind {{ bind }}
@@ -60,7 +60,7 @@ frontend {{ frontend.name }}
{% endif %}
{% endfor %}
-{% for backend in openshift.loadbalancer.backends %}
+{% for backend in openshift_loadbalancer_backends %}
backend {{ backend.name }}
balance {{ backend.balance }}
{% if 'mode' in backend %}
diff --git a/roles/openshift_loadbalancer_facts/README.md b/roles/openshift_loadbalancer_facts/README.md
deleted file mode 100644
index 57537cc03..000000000
--- a/roles/openshift_loadbalancer_facts/README.md
+++ /dev/null
@@ -1,34 +0,0 @@
-OpenShift HAProxy Loadbalancer Facts
-====================================
-
-TODO
-
-Requirements
-------------
-
-TODO
-
-Role Variables
---------------
-
-TODO
-
-Dependencies
-------------
-
-TODO
-
-Example Playbook
-----------------
-
-TODO
-
-License
--------
-
-Apache License, Version 2.0
-
-Author Information
-------------------
-
-Andrew Butcher (abutcher@redhat.com)
diff --git a/roles/openshift_loadbalancer_facts/meta/main.yml b/roles/openshift_loadbalancer_facts/meta/main.yml
deleted file mode 100644
index 46959355b..000000000
--- a/roles/openshift_loadbalancer_facts/meta/main.yml
+++ /dev/null
@@ -1,14 +0,0 @@
----
-galaxy_info:
- author: Andrew Butcher
- description: OpenShift loadbalancer facts
- company: Red Hat, Inc.
- license: Apache License, Version 2.0
- min_ansible_version: 1.9
- platforms:
- - name: EL
- versions:
- - 7
-dependencies:
- - role: openshift_facts
- - role: nuage_common
diff --git a/roles/openshift_loadbalancer_facts/tasks/main.yml b/roles/openshift_loadbalancer_facts/tasks/main.yml
deleted file mode 100644
index 5936ce5ba..000000000
--- a/roles/openshift_loadbalancer_facts/tasks/main.yml
+++ /dev/null
@@ -1,46 +0,0 @@
----
-- name: Set haproxy frontend port
- openshift_facts:
- role: loadbalancer
- local_facts:
- frontend_port: "{{ openshift_master_api_port | default(None) }}"
-
-- name: Set loadbalancer facts
- openshift_facts:
- role: loadbalancer
- local_facts:
- limit_nofile: "{{ openshift_loadbalancer_limit_nofile | default(None) }}"
- default_maxconn: "{{ openshift_loadbalancer_default_maxconn | default(None) }}"
- global_maxconn: "{{ openshift_loadbalancer_global_maxconn | default(None) }}"
- frontends:
- - name: atomic-openshift-api
- mode: tcp
- options:
- - tcplog
- binds:
- - "*:{{ openshift.loadbalancer.frontend_port }}"
- default_backend: atomic-openshift-api
- - name: nuage-monitor
- mode: tcp
- options:
- - tcplog
- binds:
- - "*:{{ nuage_mon_rest_server_port }}"
- default_backend: nuage-monitor
- when: openshift.common.use_nuage | bool
- backends:
- - name: atomic-openshift-api
- mode: tcp
- option: tcplog
- balance: source
- servers: "{{ hostvars
- | oo_select_keys(groups['oo_masters'])
- | oo_haproxy_backend_masters(openshift.loadbalancer.frontend_port) }}"
- - name: nuage-monitor
- mode: tcp
- option: tcplog
- balance: source
- servers: "{{ hostvars
- | oo_select_keys(groups['oo_masters'])
- | oo_haproxy_backend_masters(nuage_mon_rest_server_port) }}"
- when: openshift.common.use_nuage | bool
--
cgit v1.2.3