summaryrefslogtreecommitdiffstats
path: root/roles
diff options
context:
space:
mode:
Diffstat (limited to 'roles')
-rw-r--r--roles/docker/tasks/main.yml4
-rw-r--r--roles/etcd_certificates/tasks/client.yml8
-rw-r--r--roles/etcd_certificates/tasks/server.yml12
-rw-r--r--roles/nuage_master/defaults/main.yaml4
-rw-r--r--roles/nuage_master/vars/main.yaml4
-rw-r--r--roles/openshift_common/meta/main.yml1
-rw-r--r--roles/openshift_docker/tasks/main.yml2
-rwxr-xr-xroles/openshift_facts/library/openshift_facts.py32
-rw-r--r--roles/openshift_manageiq/tasks/main.yaml10
-rw-r--r--roles/openshift_manageiq/vars/main.yml3
-rw-r--r--roles/openshift_master/defaults/main.yml28
-rw-r--r--roles/openshift_master/meta/main.yml22
-rw-r--r--roles/openshift_master/templates/atomic-openshift-master.j26
-rw-r--r--roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j26
-rw-r--r--roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j26
-rw-r--r--roles/openshift_node/meta/main.yml1
-rw-r--r--roles/openshift_node/tasks/main.yml11
-rwxr-xr-xroles/openshift_node_dnsmasq/files/networkmanager/99-origin-dns.sh17
18 files changed, 101 insertions, 76 deletions
diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml
index e4a31c692..b9b2666fb 100644
--- a/roles/docker/tasks/main.yml
+++ b/roles/docker/tasks/main.yml
@@ -16,7 +16,7 @@
when: docker_storage_check.stat.exists | bool and not docker_version_result | skipped and docker_version_result.stdout | default('0.0', True) | version_compare('1.9', '>=') and docker_version | version_compare('1.9', '<')
- name: Downgrade docker if necessary
- command: "{{ ansible_pkg_mgr }} downgrade -y docker-{{ docker_version }}"
+ command: "{{ ansible_pkg_mgr }} swap -y docker-* docker-*{{ docker_version }}"
register: docker_downgrade_result
when: not docker_version_result | skipped and docker_version_result.stdout | default('0.0', True) | version_compare(docker_version, 'gt')
@@ -69,7 +69,7 @@
reg_flag: --insecure-registry
notify:
- restart docker
-
+
- name: Set Proxy Settings
lineinfile:
dest: /etc/sysconfig/docker
diff --git a/roles/etcd_certificates/tasks/client.yml b/roles/etcd_certificates/tasks/client.yml
index b497a46c0..a9f130bb9 100644
--- a/roles/etcd_certificates/tasks/client.yml
+++ b/roles/etcd_certificates/tasks/client.yml
@@ -4,7 +4,7 @@
path: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}"
state: directory
mode: 0700
- with_items: "{{ etcd_needing_client_certs | default([]) }}"
+ with_items: etcd_needing_client_certs | default([])
- name: Create the client csr
command: >
@@ -19,7 +19,7 @@
~ item.etcd_cert_prefix ~ 'client.csr' }}"
environment:
SAN: "IP:{{ item.etcd_ip }}"
- with_items: "{{ etcd_needing_client_certs | default([]) }}"
+ with_items: etcd_needing_client_certs | default([])
- name: Sign and create the client crt
command: >
@@ -33,10 +33,10 @@
~ item.etcd_cert_prefix ~ 'client.crt' }}"
environment:
SAN: "IP:{{ item.etcd_ip }}"
- with_items: "{{ etcd_needing_client_certs | default([]) }}"
+ with_items: etcd_needing_client_certs | default([])
- file:
src: "{{ etcd_ca_cert }}"
dest: "{{ etcd_generated_certs_dir}}/{{ item.etcd_cert_subdir }}/{{ item.etcd_cert_prefix }}ca.crt"
state: hard
- with_items: "{{ etcd_needing_client_certs | default([]) }}"
+ with_items: etcd_needing_client_certs | default([])
diff --git a/roles/etcd_certificates/tasks/server.yml b/roles/etcd_certificates/tasks/server.yml
index 934b8b805..223917ccd 100644
--- a/roles/etcd_certificates/tasks/server.yml
+++ b/roles/etcd_certificates/tasks/server.yml
@@ -4,7 +4,7 @@
path: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}"
state: directory
mode: 0700
- with_items: "{{ etcd_needing_server_certs | default([]) }}"
+ with_items: etcd_needing_server_certs | default([])
- name: Create the server csr
command: >
@@ -19,7 +19,7 @@
~ item.etcd_cert_prefix ~ 'server.csr' }}"
environment:
SAN: "IP:{{ item.etcd_ip }}"
- with_items: "{{ etcd_needing_server_certs | default([]) }}"
+ with_items: etcd_needing_server_certs | default([])
- name: Sign and create the server crt
command: >
@@ -33,7 +33,7 @@
~ item.etcd_cert_prefix ~ 'server.crt' }}"
environment:
SAN: "IP:{{ item.etcd_ip }}"
- with_items: "{{ etcd_needing_server_certs | default([]) }}"
+ with_items: etcd_needing_server_certs | default([])
- name: Create the peer csr
command: >
@@ -48,7 +48,7 @@
~ item.etcd_cert_prefix ~ 'peer.csr' }}"
environment:
SAN: "IP:{{ item.etcd_ip }}"
- with_items: "{{ etcd_needing_server_certs | default([]) }}"
+ with_items: etcd_needing_server_certs | default([])
- name: Sign and create the peer crt
command: >
@@ -62,10 +62,10 @@
~ item.etcd_cert_prefix ~ 'peer.crt' }}"
environment:
SAN: "IP:{{ item.etcd_ip }}"
- with_items: "{{ etcd_needing_server_certs | default([]) }}"
+ with_items: etcd_needing_server_certs | default([])
- file:
src: "{{ etcd_ca_cert }}"
dest: "{{ etcd_generated_certs_dir}}/{{ item.etcd_cert_subdir }}/{{ item.etcd_cert_prefix }}ca.crt"
state: hard
- with_items: "{{ etcd_needing_server_certs | default([]) }}"
+ with_items: etcd_needing_server_certs | default([])
diff --git a/roles/nuage_master/defaults/main.yaml b/roles/nuage_master/defaults/main.yaml
new file mode 100644
index 000000000..cf670a9e1
--- /dev/null
+++ b/roles/nuage_master/defaults/main.yaml
@@ -0,0 +1,4 @@
+---
+nuage_master_cspadminpasswd: ""
+nuage_master_adminusername: admin
+nuage_master_adminuserpasswd: admin
diff --git a/roles/nuage_master/vars/main.yaml b/roles/nuage_master/vars/main.yaml
index d3536eb33..5bc735bb6 100644
--- a/roles/nuage_master/vars/main.yaml
+++ b/roles/nuage_master/vars/main.yaml
@@ -28,7 +28,3 @@ nuage_service_account_config:
nuage_tasks:
- policy add-cluster-role-to-user cluster-reader {{ nuage_service_account }}
-
-nuage_master_cspadminpasswd: ''
-nuage_master_adminusername: 'admin'
-nuage_master_adminuserpasswd: 'admin'
diff --git a/roles/openshift_common/meta/main.yml b/roles/openshift_common/meta/main.yml
index 02150406d..f1cf3e161 100644
--- a/roles/openshift_common/meta/main.yml
+++ b/roles/openshift_common/meta/main.yml
@@ -12,6 +12,5 @@ galaxy_info:
categories:
- cloud
dependencies:
-- role: os_firewall
- role: openshift_facts
- role: openshift_repos
diff --git a/roles/openshift_docker/tasks/main.yml b/roles/openshift_docker/tasks/main.yml
index 10f47f9b2..4e5cec82f 100644
--- a/roles/openshift_docker/tasks/main.yml
+++ b/roles/openshift_docker/tasks/main.yml
@@ -25,5 +25,5 @@
- role: docker
local_facts:
openshift_image_tag: "{{ l_image_tag | default(None) }}"
- openshift_version: "{{ l_image_tag.split('-')[0] if l_image_tag is defined else '' | oo_image_tag_to_rpm_version }}"
+ openshift_version: "{{ l_image_tag.split('-')[0] | oo_image_tag_to_rpm_version if l_image_tag is defined else '' }}"
when: openshift.common.is_containerized is defined and openshift.common.is_containerized | bool
diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py
index 643984982..b1ad537bc 100755
--- a/roles/openshift_facts/library/openshift_facts.py
+++ b/roles/openshift_facts/library/openshift_facts.py
@@ -19,8 +19,8 @@ EXAMPLES = '''
import ConfigParser
import copy
+import io
import os
-import StringIO
import yaml
from distutils.util import strtobool
from distutils.version import LooseVersion
@@ -499,12 +499,12 @@ def set_dnsmasq_facts_if_unset(facts):
"""
if 'common' in facts:
- if 'use_dnsmasq' not in facts['common'] and facts['common']['version_gte_3_2_or_1_2']:
+ if 'use_dnsmasq' not in facts['common'] and safe_get_bool(facts['common']['version_gte_3_2_or_1_2']):
facts['common']['use_dnsmasq'] = True
else:
facts['common']['use_dnsmasq'] = False
if 'master' in facts and 'dns_port' not in facts['master']:
- if facts['common']['use_dnsmasq']:
+ if safe_get_bool(facts['common']['use_dnsmasq']):
facts['master']['dns_port'] = 8053
else:
facts['master']['dns_port'] = 53
@@ -689,7 +689,7 @@ def set_etcd_facts_if_unset(facts):
If anything goes wrong parsing these, the fact will not be set.
"""
- if 'master' in facts and facts['master']['embedded_etcd']:
+ if 'master' in facts and safe_get_bool(facts['master']['embedded_etcd']):
etcd_facts = facts['etcd'] if 'etcd' in facts else dict()
if 'etcd_data_dir' not in etcd_facts:
@@ -716,8 +716,8 @@ def set_etcd_facts_if_unset(facts):
# Read ETCD_DATA_DIR from /etc/etcd/etcd.conf:
try:
# Add a fake section for parsing:
- ini_str = '[root]\n' + open('/etc/etcd/etcd.conf', 'r').read()
- ini_fp = StringIO.StringIO(ini_str)
+ ini_str = unicode('[root]\n' + open('/etc/etcd/etcd.conf', 'r').read(), 'utf-8')
+ ini_fp = io.StringIO(ini_str)
config = ConfigParser.RawConfigParser()
config.readfp(ini_fp)
etcd_data_dir = config.get('root', 'ETCD_DATA_DIR')
@@ -1374,18 +1374,19 @@ def set_proxy_facts(facts):
if 'common' in facts:
common = facts['common']
if 'http_proxy' in common or 'https_proxy' in common:
+ if 'no_proxy' in common and \
+ isinstance(common['no_proxy'], basestring):
+ common['no_proxy'] = common['no_proxy'].split(",")
+ elif 'no_proxy' not in common:
+ common['no_proxy'] = []
if 'generate_no_proxy_hosts' in common and \
- common['generate_no_proxy_hosts']:
- if 'no_proxy' in common and \
- isinstance(common['no_proxy'], basestring):
- common['no_proxy'] = common['no_proxy'].split(",")
- else:
- common['no_proxy'] = []
+ safe_get_bool(common['generate_no_proxy_hosts']):
if 'no_proxy_internal_hostnames' in common:
common['no_proxy'].extend(common['no_proxy_internal_hostnames'].split(','))
common['no_proxy'].append('.' + common['dns_domain'])
- common['no_proxy'].append(common['hostname'])
- common['no_proxy'] = sort_unique(common['no_proxy'])
+ # We always add ourselves no matter what
+ common['no_proxy'].append(common['hostname'])
+ common['no_proxy'] = sort_unique(common['no_proxy'])
facts['common'] = common
if 'builddefaults' in facts:
@@ -1735,6 +1736,9 @@ class OpenShiftFacts(object):
nfs=dict(
directory='/exports',
options='*(rw,root_squash)'),
+ openstack=dict(
+ filesystem='ext4',
+ volumeID='123'),
host=None,
access_modes=['ReadWriteMany'],
create_pv=True
diff --git a/roles/openshift_manageiq/tasks/main.yaml b/roles/openshift_manageiq/tasks/main.yaml
index 2a651df65..de0a7000e 100644
--- a/roles/openshift_manageiq/tasks/main.yaml
+++ b/roles/openshift_manageiq/tasks/main.yaml
@@ -59,6 +59,16 @@
failed_when: "'already exists' not in osmiq_perm_task.stderr and osmiq_perm_task.rc != 0"
changed_when: osmiq_perm_task.rc == 0
+- name: Configure 3_2 role/user permissions
+ command: >
+ {{ openshift.common.admin_binary }} {{item}}
+ --config={{manage_iq_tmp_conf}}
+ with_items: "{{manage_iq_openshift_3_2_tasks}}"
+ register: osmiq_perm_3_2_task
+ failed_when: osmiq_perm_3_2_task.rc != 0
+ changed_when: osmiq_perm_3_2_task.rc == 0
+ when: openshift.common.version_gte_3_2_or_1_2 | bool
+
- name: Clean temporary configuration file
command: >
rm -f {{manage_iq_tmp_conf}}
diff --git a/roles/openshift_manageiq/vars/main.yml b/roles/openshift_manageiq/vars/main.yml
index 69ee2cb4c..b2aed79c7 100644
--- a/roles/openshift_manageiq/vars/main.yml
+++ b/roles/openshift_manageiq/vars/main.yml
@@ -30,3 +30,6 @@ manage_iq_tasks:
- policy add-scc-to-user privileged system:serviceaccount:management-infra:management-admin
- policy add-cluster-role-to-user system:image-puller system:serviceaccount:management-infra:inspector-admin
- policy add-scc-to-user privileged system:serviceaccount:management-infra:inspector-admin
+
+manage_iq_openshift_3_2_tasks:
+ - policy add-cluster-role-to-user system:image-auditor system:serviceaccount:management-infra:management-admin
diff --git a/roles/openshift_master/defaults/main.yml b/roles/openshift_master/defaults/main.yml
index 09bde6002..dbd62c80f 100644
--- a/roles/openshift_master/defaults/main.yml
+++ b/roles/openshift_master/defaults/main.yml
@@ -1,32 +1,4 @@
---
openshift_node_ips: []
-
# TODO: update setting these values based on the facts
-os_firewall_allow:
-- service: etcd embedded
- port: 4001/tcp
-- service: api server https
- port: "{{ openshift.master.api_port }}/tcp"
-- service: api controllers https
- port: "{{ openshift.master.controllers_port }}/tcp"
-- service: skydns tcp
- port: "{{ openshift.master.dns_port }}/tcp"
-- service: skydns udp
- port: "{{ openshift.master.dns_port }}/udp"
-- service: Fluentd td-agent tcp
- port: 24224/tcp
-- service: Fluentd td-agent udp
- port: 24224/udp
-- service: pcsd
- port: 2224/tcp
-- service: Corosync UDP
- port: 5404/udp
-- service: Corosync UDP
- port: 5405/udp
-os_firewall_deny:
-- service: api server http
- port: 8080/tcp
-- service: former etcd peer port
- port: 7001/tcp
-
openshift_version: "{{ openshift_pkg_version | default(openshift_image_tag | default(openshift.docker.openshift_image_tag | default(''))) }}"
diff --git a/roles/openshift_master/meta/main.yml b/roles/openshift_master/meta/main.yml
index e882e0b8b..d8834d27f 100644
--- a/roles/openshift_master/meta/main.yml
+++ b/roles/openshift_master/meta/main.yml
@@ -18,3 +18,25 @@ dependencies:
- role: openshift_builddefaults
- role: openshift_master_facts
- role: openshift_hosted_facts
+- role: os_firewall
+ os_firewall_allow:
+ - service: etcd embedded
+ port: 4001/tcp
+ - service: api server https
+ port: "{{ openshift.master.api_port }}/tcp"
+ - service: api controllers https
+ port: "{{ openshift.master.controllers_port }}/tcp"
+ - service: skydns tcp
+ port: "{{ openshift.master.dns_port }}/tcp"
+ - service: skydns udp
+ port: "{{ openshift.master.dns_port }}/udp"
+ - service: Fluentd td-agent tcp
+ port: 24224/tcp
+ - service: Fluentd td-agent udp
+ port: 24224/udp
+ - service: pcsd
+ port: 2224/tcp
+ - service: Corosync UDP
+ port: 5404/udp
+ - service: Corosync UDP
+ port: 5405/udp
diff --git a/roles/openshift_master/templates/atomic-openshift-master.j2 b/roles/openshift_master/templates/atomic-openshift-master.j2
index 4cf632841..026787421 100644
--- a/roles/openshift_master/templates/atomic-openshift-master.j2
+++ b/roles/openshift_master/templates/atomic-openshift-master.j2
@@ -12,11 +12,11 @@ AWS_SECRET_ACCESS_KEY={{ openshift.cloudprovider.aws.secret_key }}
# Proxy configuration
# See https://docs.openshift.com/enterprise/latest/install_config/install/advanced_install.html#configuring-global-proxy
{% if 'http_proxy' in openshift.common %}
-HTTP_PROXY='{{ openshift.common.http_proxy | default('') }}'
+HTTP_PROXY={{ openshift.common.http_proxy | default('') }}
{% endif %}
{% if 'https_proxy' in openshift.common %}
-HTTPS_PROXY='{{ openshift.common.https_proxy | default('')}}'
+HTTPS_PROXY={{ openshift.common.https_proxy | default('')}}
{% endif %}
{% if 'no_proxy' in openshift.common %}
-NO_PROXY='{{ openshift.common.no_proxy | default('') | join(',') }},{{ openshift.common.portal_net }},{{ openshift.master.sdn_cluster_network_cidr }}'
+NO_PROXY={{ openshift.common.no_proxy | default('') | join(',') }},{{ openshift.common.portal_net }},{{ openshift.master.sdn_cluster_network_cidr }}
{% endif %}
diff --git a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2 b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2
index 01a8428a0..02c22e374 100644
--- a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2
+++ b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2
@@ -12,11 +12,11 @@ AWS_SECRET_ACCESS_KEY={{ openshift.cloudprovider.aws.secret_key }}
# Proxy configuration
# See https://docs.openshift.com/enterprise/latest/install_config/install/advanced_install.html#configuring-global-proxy
{% if 'http_proxy' in openshift.common %}
-HTTP_PROXY='{{ openshift.common.http_proxy | default('') }}'
+HTTP_PROXY={{ openshift.common.http_proxy | default('') }}
{% endif %}
{% if 'https_proxy' in openshift.common %}
-HTTPS_PROXY='{{ openshift.common.https_proxy | default('')}}'
+HTTPS_PROXY={{ openshift.common.https_proxy | default('')}}
{% endif %}
{% if 'no_proxy' in openshift.common %}
-NO_PROXY='{{ openshift.common.no_proxy | default('') | join(',') }},{{ openshift.common.portal_net }},{{ openshift.master.sdn_cluster_network_cidr }}'
+NO_PROXY={{ openshift.common.no_proxy | default('') | join(',') }},{{ openshift.common.portal_net }},{{ openshift.master.sdn_cluster_network_cidr }}
{% endif %}
diff --git a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2 b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2
index 89ccb1eed..644640577 100644
--- a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2
+++ b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2
@@ -12,11 +12,11 @@ AWS_SECRET_ACCESS_KEY={{ openshift.cloudprovider.aws.secret_key }}
# Proxy configuration
# See https://docs.openshift.com/enterprise/latest/install_config/install/advanced_install.html#configuring-global-proxy
{% if 'http_proxy' in openshift.common %}
-HTTP_PROXY='{{ openshift.common.http_proxy | default('') }}'
+HTTP_PROXY={{ openshift.common.http_proxy | default('') }}
{% endif %}
{% if 'https_proxy' in openshift.common %}
-HTTPS_PROXY='{{ openshift.common.https_proxy | default('')}}'
+HTTPS_PROXY={{ openshift.common.https_proxy | default('')}}
{% endif %}
{% if 'no_proxy' in openshift.common %}
-NO_PROXY='{{ openshift.common.no_proxy | default('') | join(',') }},{{ openshift.common.portal_net }},{{ openshift.master.sdn_cluster_network_cidr }}'
+NO_PROXY={{ openshift.common.no_proxy | default('') | join(',') }},{{ openshift.common.portal_net }},{{ openshift.master.sdn_cluster_network_cidr }}
{% endif %}
diff --git a/roles/openshift_node/meta/main.yml b/roles/openshift_node/meta/main.yml
index ca0c332ea..db1776632 100644
--- a/roles/openshift_node/meta/main.yml
+++ b/roles/openshift_node/meta/main.yml
@@ -17,4 +17,5 @@ dependencies:
- role: openshift_common
- role: openshift_node_dnsmasq
when: openshift.common.use_dnsmasq
+- role: os_firewall
diff --git a/roles/openshift_node/tasks/main.yml b/roles/openshift_node/tasks/main.yml
index 06fde88af..be70a170d 100644
--- a/roles/openshift_node/tasks/main.yml
+++ b/roles/openshift_node/tasks/main.yml
@@ -112,6 +112,17 @@
- name: Start and enable node
service: name={{ openshift.common.service_type }}-node enabled=yes state=started
register: node_start_result
+ ignore_errors: yes
+
+- name: Check logs on failure
+ command: journalctl -xe
+ register: node_failure
+ when: node_start_result | failed
+
+- name: Dump failure information
+ debug: var=node_failure
+ when: node_start_result | failed
+
- set_fact:
node_service_status_changed: "{{ node_start_result | changed }}"
diff --git a/roles/openshift_node_dnsmasq/files/networkmanager/99-origin-dns.sh b/roles/openshift_node_dnsmasq/files/networkmanager/99-origin-dns.sh
index 691fa32f3..09bae1777 100755
--- a/roles/openshift_node_dnsmasq/files/networkmanager/99-origin-dns.sh
+++ b/roles/openshift_node_dnsmasq/files/networkmanager/99-origin-dns.sh
@@ -8,10 +8,12 @@
# a pod would fail.
#
# To use this,
-# Drop this script in /etc/NetworkManager/dispatcher.d/
-# systemctl restart NetworkManager
-# Configure node-config.yaml to set dnsIP: to the ip address of this
-# node
+# - If this host is also a master, reconfigure master dnsConfig to listen on
+# 8053 to avoid conflicts on port 53 and open port 8053 in the firewall
+# - Drop this script in /etc/NetworkManager/dispatcher.d/
+# - systemctl restart NetworkManager
+# - Configure node-config.yaml to set dnsIP: to the ip address of this
+# node
#
# Test it:
# host kubernetes.default.svc.cluster.local
@@ -31,7 +33,8 @@ if [[ $2 =~ ^(up|dhcp4-change)$ ]]; then
def_route=$(/sbin/ip route list match 0.0.0.0/0 | awk '{print $3 }')
def_route_int=$(/sbin/ip route get to ${def_route} | awk '{print $3}')
def_route_ip=$(/sbin/ip route get to ${def_route} | awk '{print $5}')
- if [[ ${DEVICE_IFACE} == ${def_route_int} ]]; then
+ if [[ ${DEVICE_IFACE} == ${def_route_int} && \
+ -n "${IP4_NAMESERVERS}" ]]; then
if [ ! -f /etc/dnsmasq.d/origin-dns.conf ]; then
cat << EOF > /etc/dnsmasq.d/origin-dns.conf
strict-order
@@ -42,8 +45,8 @@ server=/30.172.in-addr.arpa/172.30.0.1
EOF
fi
# zero out our upstream servers list and feed it into dnsmasq
- echo '' > /etc/dnsmasq.d/origin-upstream-dns.conf
- for ns in ${DHCP4_DOMAIN_NAME_SERVERS}; do
+ echo -n > /etc/dnsmasq.d/origin-upstream-dns.conf
+ for ns in ${IP4_NAMESERVERS}; do
echo "server=${ns}" >> /etc/dnsmasq.d/origin-upstream-dns.conf
done
systemctl restart dnsmasq