14 files changed, 260 insertions, 52 deletions
diff --git a/roles/openshift_node/defaults/main.yml b/roles/openshift_node/defaults/main.yml
index 47073ee0f..cc000496a 100644
--- a/roles/openshift_node/defaults/main.yml
+++ b/roles/openshift_node/defaults/main.yml
@@ -1,5 +1,8 @@
 ---
-os_firewall_allow:
+r_openshift_node_firewall_enabled: "{{ os_firewall_enabled | default(True) }}"
+r_openshift_node_use_firewalld: "{{ os_firewall_use_firewalld | default(False) }}"
+r_openshift_node_os_firewall_deny: []
+r_openshift_node_os_firewall_allow:
 - service: Kubernetes kubelet
   port: 10250/tcp
 - service: http
@@ -8,7 +11,18 @@ os_firewall_allow:
   port: 443/tcp
 - service: OpenShift OVS sdn
   port: 4789/udp
-  when: openshift.common.use_openshift_sdn | default(true) | bool
+  cond: openshift.common.use_openshift_sdn | default(true) | bool
 - service: Calico BGP Port
   port: 179/tcp
-  when: openshift.common.use_calico | bool
+  cond: "{{ openshift.common.use_calico | bool }}"
+- service: Kubernetes service NodePort TCP
+  port: "{{ openshift_node_port_range | default('') }}/tcp"
+  cond: "{{ openshift_node_port_range is defined }}"
+- service: Kubernetes service NodePort UDP
+  port: "{{ openshift_node_port_range | default('') }}/udp"
+  cond: "{{ openshift_node_port_range is defined }}"
+
+oreg_url: ''
+oreg_host: "{{ oreg_url.split('/')[0] if '.' in oreg_url.split('/')[0] else '' }}"
+oreg_auth_credentials_path: "{{ openshift.common.data_dir }}/.docker"
+oreg_auth_credentials_replace: False
diff --git a/roles/openshift_node/meta/main.yml b/roles/openshift_node/meta/main.yml
index 4fb841add..06373de04 100644
--- a/roles/openshift_node/meta/main.yml
+++ b/roles/openshift_node/meta/main.yml
@@ -14,36 +14,11 @@ galaxy_info:
 dependencies:
 - role: openshift_node_facts
 - role: lib_openshift
+- role: lib_os_firewall
 - role: openshift_common
 - role: openshift_clock
 - role: openshift_docker
 - role: openshift_node_certificates
 - role: openshift_cloud_provider
-- role: os_firewall
-  os_firewall_allow:
-  - service: Kubernetes kubelet
-    port: 10250/tcp
-  - service: http
-    port: 80/tcp
-  - service: https
-    port: 443/tcp
-- role: os_firewall
-  os_firewall_allow:
-  - service: OpenShift OVS sdn
-    port: 4789/udp
-  when: openshift.common.use_openshift_sdn | default(true) | bool
-- role: os_firewall
-  os_firewall_allow:
-  - service: Calico BGP Port
-    port: 179/tcp
-  when: openshift.common.use_calico | bool
-
-- role: os_firewall
-  os_firewall_allow:
-  - service: Kubernetes service NodePort TCP
-    port: "{{ openshift_node_port_range | default('') }}/tcp"
-  - service: Kubernetes service NodePort UDP
-    port: "{{ openshift_node_port_range | default('') }}/udp"
-  when: openshift_node_port_range is defined
 - role: openshift_node_dnsmasq
   when: openshift.common.use_dnsmasq | bool
diff --git a/roles/openshift_node/tasks/firewall.yml b/roles/openshift_node/tasks/firewall.yml
new file mode 100644
index 000000000..255aa886a
--- /dev/null
+++ b/roles/openshift_node/tasks/firewall.yml
@@ -0,0 +1,40 @@
+---
+- when: r_openshift_node_firewall_enabled | bool and not r_openshift_node_use_firewalld | bool
+  block:
+  - name: Add iptables allow rules
+    os_firewall_manage_iptables:
+      name: "{{ item.service }}"
+      action: add
+      protocol: "{{ item.port.split('/')[1] }}"
+      port: "{{ item.port.split('/')[0] }}"
+    when: item.cond | default(True)
+    with_items: "{{ r_openshift_node_os_firewall_allow }}"
+
+  - name: Remove iptables rules
+    os_firewall_manage_iptables:
+      name: "{{ item.service }}"
+      action: remove
+      protocol: "{{ item.port.split('/')[1] }}"
+      port: "{{ item.port.split('/')[0] }}"
+    when: item.cond | default(True)
+    with_items: "{{ r_openshift_node_os_firewall_deny }}"
+
+- when: r_openshift_node_firewall_enabled | bool and r_openshift_node_use_firewalld | bool
+  block:
+  - name: Add firewalld allow rules
+    firewalld:
+      port: "{{ item.port }}"
+      permanent: true
+      immediate: true
+      state: enabled
+    when: item.cond | default(True)
+    with_items: "{{ r_openshift_node_os_firewall_allow }}"
+
+  - name: Remove firewalld allow rules
+    firewalld:
+      port: "{{ item.port }}"
+      permanent: true
+      immediate: true
+      state: disabled
+    when: item.cond | default(True)
+    with_items: "{{ r_openshift_node_os_firewall_deny }}"
diff --git a/roles/openshift_node/tasks/main.yml b/roles/openshift_node/tasks/main.yml
index 87b1f6537..525dd1d1a 100644
--- a/roles/openshift_node/tasks/main.yml
+++ b/roles/openshift_node/tasks/main.yml
@@ -2,9 +2,41 @@
 # TODO: allow for overriding default ports where possible
 - fail:
     msg: "SELinux is disabled, This deployment type requires that SELinux is enabled."
-  when: >
-    (not ansible_selinux or ansible_selinux.status != 'enabled') and
-    deployment_type in ['enterprise', 'online', 'atomic-enterprise', 'openshift-enterprise']
+  when:
+    - (not ansible_selinux or ansible_selinux.status != 'enabled') and deployment_type in ['enterprise', 'online', 'atomic-enterprise', 'openshift-enterprise']
+    - not openshift_use_crio | default(false)
+
+- name: setup firewall
+  include: firewall.yml
+  static: yes
+
+- name: Set node facts
+  openshift_facts:
+    role: "{{ item.role }}"
+    local_facts: "{{ item.local_facts }}"
+  with_items:
+    # Reset node labels to an empty dictionary.
+    - role: node
+      local_facts:
+        labels: {}
+    - role: node
+      local_facts:
+        annotations: "{{ openshift_node_annotations | default(none) }}"
+        debug_level: "{{ openshift_node_debug_level | default(openshift.common.debug_level) }}"
+        iptables_sync_period: "{{ openshift_node_iptables_sync_period | default(None) }}"
+        kubelet_args: "{{ openshift_node_kubelet_args | default(None) }}"
+        labels: "{{ lookup('oo_option', 'openshift_node_labels') | default( openshift_node_labels | default(none), true) }}"
+        registry_url: "{{ oreg_url_node | default(oreg_url) | default(None) }}"
+        schedulable: "{{ openshift_schedulable | default(openshift_scheduleable) | default(None) }}"
+        sdn_mtu: "{{ openshift_node_sdn_mtu | default(None) }}"
+        storage_plugin_deps: "{{ osn_storage_plugin_deps | default(None) }}"
+        set_node_ip: "{{ openshift_set_node_ip | default(None) }}"
+        node_image: "{{ osn_image | default(None) }}"
+        ovs_image: "{{ osn_ovs_image | default(None) }}"
+        proxy_mode: "{{ openshift_node_proxy_mode | default('iptables') }}"
+        local_quota_per_fsgroup: "{{ openshift_node_local_quota_per_fsgroup | default(None) }}"
+        dns_ip: "{{ openshift_dns_ip | default(none) | get_dns_ip(hostvars[inventory_hostname])}}"
+        env_vars: "{{ openshift_node_env_vars | default(None) }}"
 
 # https://docs.openshift.com/container-platform/3.4/admin_guide/overcommit.html#disabling-swap-memory
 - name: Check for swap usage
@@ -38,25 +70,15 @@
     - openshift_disable_swap | default(true) | bool
 # End Disable Swap Block
 
-# We have to add tuned-profiles in the same transaction otherwise we run into depsolving
-# problems because the rpms don't pin the version properly. This was fixed in 3.1 packaging.
 - name: Install Node package
   package:
-    name: "{{ openshift.common.service_type }}-node{{ openshift_pkg_version | default('') | oo_image_tag_to_rpm_version(include_dash=True) }},tuned-profiles-{{ openshift.common.service_type }}-node{{ openshift_pkg_version | default('') | oo_image_tag_to_rpm_version(include_dash=True) }}"
+    name: "{{ openshift.common.service_type }}-node{{ openshift_pkg_version | default('') | oo_image_tag_to_rpm_version(include_dash=True) }}"
     state: present
   when: not openshift.common.is_containerized | bool
 
-- name: Check for tuned package
-  command: rpm -q tuned
-  args:
-    warn: no
-  register: tuned_installed
-  changed_when: false
-  failed_when: false
-
-- name: Set atomic-guest tuned profile
-  command: "tuned-adm profile atomic-guest"
-  when: tuned_installed.rc == 0 and openshift.common.is_atomic | bool
+- name: setup tuned
+  include: tuned.yml
+  static: yes
 
 - name: Install sdn-ovs package
   package:
@@ -66,6 +88,13 @@
     - openshift.common.use_openshift_sdn | default(true) | bool
     - not openshift.common.is_containerized | bool
 
+- name: Restart cri-o
+  systemd:
+    name: cri-o
+    enabled: yes
+    state: restarted
+  when: openshift_use_crio | default(false)
+
 - name: Install conntrack-tools package
   package:
     name: "conntrack-tools"
@@ -121,6 +150,21 @@
   notify:
     - restart node
 
+- name: Check for credentials file for registry auth
+  stat:
+    path: "{{oreg_auth_credentials_path }}"
+  when:
+    - oreg_auth_user is defined
+  register: node_oreg_auth_credentials_stat
+
+- name: Create credentials for registry auth
+  command: "docker --config={{ oreg_auth_credentials_path }} login -u {{ oreg_auth_user }} -p {{ oreg_auth_password }} {{ oreg_host }}"
+  when:
+    - oreg_auth_user is defined
+    - (not node_oreg_auth_credentials_stat.stat.exists or oreg_auth_credentials_replace) | bool
+  notify:
+    - restart node
+
 - name: Configure AWS Cloud Provider Settings
   lineinfile:
     dest: /etc/sysconfig/{{ openshift.common.service_type }}-node
diff --git a/roles/openshift_node/tasks/node_system_container.yml b/roles/openshift_node/tasks/node_system_container.yml
index d99f657bc..b2dceedbe 100644
--- a/roles/openshift_node/tasks/node_system_container.yml
+++ b/roles/openshift_node/tasks/node_system_container.yml
@@ -1,12 +1,12 @@
 ---
 - name: Pre-pull node system container image
   command: >
-    atomic pull --storage=ostree {{ openshift.common.system_images_registry }}/{{ openshift.node.node_system_image }}:{{ openshift_image_tag }}
+    atomic pull --storage=ostree {{ 'docker:' if openshift.common.system_images_registry == 'docker' else openshift.common.system_images_registry + '/' }}{{ openshift.node.node_system_image }}:{{ openshift_image_tag }}
   register: pull_result
   changed_when: "'Pulling layer' in pull_result.stdout"
 
 - name: Install or Update node system container
   oc_atomic_container:
     name: "{{ openshift.common.service_type }}-node"
-    image: "{{ openshift.common.system_images_registry }}/{{ openshift.node.node_system_image }}:{{ openshift_image_tag }}"
+    image: "{{ 'docker:' if openshift.common.system_images_registry == 'docker' else openshift.common.system_images_registry + '/' }}{{ openshift.node.node_system_image }}:{{ openshift_image_tag }}"
     state: latest
diff --git a/roles/openshift_node/tasks/openvswitch_system_container.yml b/roles/openshift_node/tasks/openvswitch_system_container.yml
index c8d653880..e09063aa5 100644
--- a/roles/openshift_node/tasks/openvswitch_system_container.yml
+++ b/roles/openshift_node/tasks/openvswitch_system_container.yml
@@ -1,14 +1,25 @@
 ---
+- set_fact:
+    l_use_crio: "{{ openshift_use_crio | default(false) }}"
+
+- set_fact:
+    l_service_name: "cri-o"
+  when: l_use_crio
+
+- set_fact:
+    l_service_name: "{{ openshift.docker.service_name }}"
+  when: not l_use_crio
+
 - name: Pre-pull OpenVSwitch system container image
   command: >
-    atomic pull --storage=ostree {{ openshift.common.system_images_registry }}/{{ openshift.node.ovs_system_image }}:{{ openshift_image_tag }}
+    atomic pull --storage=ostree {{ 'docker:' if openshift.common.system_images_registry == 'docker' else openshift.common.system_images_registry + '/' }}{{ openshift.node.ovs_system_image }}:{{ openshift_image_tag }}
   register: pull_result
   changed_when: "'Pulling layer' in pull_result.stdout"
 
 - name: Install or Update OpenVSwitch system container
   oc_atomic_container:
     name: openvswitch
-    image: "{{ openshift.common.system_images_registry }}/{{ openshift.node.ovs_system_image }}:{{ openshift_image_tag }}"
+    image: "{{ 'docker:' if openshift.common.system_images_registry == 'docker' else openshift.common.system_images_registry + '/' }}{{ openshift.node.ovs_system_image }}:{{ openshift_image_tag }}"
     state: latest
     values:
-      - "DOCKER_SERVICE={{ openshift.docker.service_name }}.service"
+      - "DOCKER_SERVICE={{ l_service_name }}"
diff --git a/roles/openshift_node/tasks/tuned.yml b/roles/openshift_node/tasks/tuned.yml
new file mode 100644
index 000000000..425bf6a26
--- /dev/null
+++ b/roles/openshift_node/tasks/tuned.yml
@@ -0,0 +1,41 @@
+---
+- name: Check for tuned package
+  command: rpm -q tuned
+  args:
+    warn: no
+  register: tuned_installed
+  changed_when: false
+  failed_when: false
+
+- name: Tuned service setup
+  block:
+  - name: Set tuned OpenShift variables
+    set_fact:
+      openshift_tuned_guest_profile: "{{ 'atomic-guest' if openshift.common.is_atomic else 'virtual-guest' }}"
+      tuned_etc_directory: '/etc/tuned'
+      tuned_templates_source: '../templates/tuned'
+
+  - name: Ensure directory structure exists
+    file:
+      state: directory
+      dest: '{{ tuned_etc_directory }}/{{ item.path }}'
+    with_filetree: '{{ tuned_templates_source }}'
+    when: item.state == 'directory'
+
+  - name: Ensure files are populated from templates
+    template:
+      src: '{{ item.src }}'
+      dest: '{{ tuned_etc_directory }}/{{ item.path }}'
+    with_filetree: '{{ tuned_templates_source }}'
+    when: item.state == 'file'
+
+  - name: Make tuned use the recommended tuned profile on restart
+    file: path=/etc/tuned/active_profile state=absent
+
+  - name: Restart tuned service
+    systemd:
+      state: restarted
+      daemon_reload: yes
+      name: tuned
+
+  when: tuned_installed.rc == 0 | bool
diff --git a/roles/openshift_node/templates/node.service.j2 b/roles/openshift_node/templates/node.service.j2
index e12a52c15..0856737f6 100644
--- a/roles/openshift_node/templates/node.service.j2
+++ b/roles/openshift_node/templates/node.service.j2
@@ -8,6 +8,7 @@ Wants={{ openshift.docker.service_name }}.service
 Documentation=https://github.com/openshift/origin
 Requires=dnsmasq.service
 After=dnsmasq.service
+{% if openshift_use_crio|default(false) %}Wants=cri-o.service{% endif %}
 
 [Service]
 Type=notify
diff --git a/roles/openshift_node/templates/node.yaml.v1.j2 b/roles/openshift_node/templates/node.yaml.v1.j2
index 351c8c9f6..711afcadb 100644
--- a/roles/openshift_node/templates/node.yaml.v1.j2
+++ b/roles/openshift_node/templates/node.yaml.v1.j2
@@ -16,6 +16,21 @@ imageConfig:
   latest: false
 kind: NodeConfig
 kubeletArguments: {{ openshift.node.kubelet_args | default(None) | to_padded_yaml(level=1) }}
+{% if openshift_use_crio | default(False) %}
+  container-runtime:
+  - remote
+  container-runtime-endpoint:
+  - /var/run/crio.sock
+  experimental-cri:
+  - 'true'
+  image-service-endpoint:
+  - /var/run/crio.sock
+  node-labels:
+  - router=true
+  - registry=true
+  runtime-request-timeout:
+  - 10m
+{% endif %}
 {% if openshift.common.version_gte_3_3_or_1_3 | bool %}
 masterClientConnectionOverrides:
   acceptContentTypes: application/vnd.kubernetes.protobuf,application/json
diff --git a/roles/openshift_node/templates/openshift.docker.node.dep.service b/roles/openshift_node/templates/openshift.docker.node.dep.service
index 4c47f8c0d..8734e7443 100644
--- a/roles/openshift_node/templates/openshift.docker.node.dep.service
+++ b/roles/openshift_node/templates/openshift.docker.node.dep.service
@@ -3,7 +3,7 @@ Requires={{ openshift.docker.service_name }}.service
 After={{ openshift.docker.service_name }}.service
 PartOf={{ openshift.common.service_type }}-node.service
 Before={{ openshift.common.service_type }}-node.service
-
+{% if openshift_use_crio|default(false) %}Wants=cri-o.service{% endif %}
 
 [Service]
 ExecStart=/bin/bash -c "if [[ -f /usr/bin/docker-current ]]; then echo \"DOCKER_ADDTL_BIND_MOUNTS=--volume=/usr/bin/docker-current:/usr/bin/docker-current:ro --volume=/etc/sysconfig/docker:/etc/sysconfig/docker:ro\" > /etc/sysconfig/{{ openshift.common.service_type }}-node-dep; else echo \"#DOCKER_ADDTL_BIND_MOUNTS=\" > /etc/sysconfig/{{ openshift.common.service_type }}-node-dep; fi"
diff --git a/roles/openshift_node/templates/tuned/openshift-control-plane/tuned.conf b/roles/openshift_node/templates/tuned/openshift-control-plane/tuned.conf
new file mode 100644
index 000000000..f22f21065
--- /dev/null
+++ b/roles/openshift_node/templates/tuned/openshift-control-plane/tuned.conf
@@ -0,0 +1,25 @@
+#
+# tuned configuration
+#
+
+[main]
+summary=Optimize systems running OpenShift control plane
+include=openshift
+
+[sysctl]
+# ktune sysctl settings, maximizing i/o throughput
+#
+# Minimal preemption granularity for CPU-bound tasks:
+# (default: 1 msec#  (1 + ilog(ncpus)), units: nanoseconds)
+kernel.sched_min_granularity_ns=10000000
+
+# The total time the scheduler will consider a migrated process
+# "cache hot" and thus less likely to be re-migrated
+# (system default is 500000, i.e. 0.5 ms)
+kernel.sched_migration_cost_ns=5000000
+
+# SCHED_OTHER wake-up granularity.
+#
+# Preemption granularity when tasks wake up.  Lower the value to improve 
+# wake-up latency and throughput for latency critical tasks.
+kernel.sched_wakeup_granularity_ns = 4000000
diff --git a/roles/openshift_node/templates/tuned/openshift-node/tuned.conf b/roles/openshift_node/templates/tuned/openshift-node/tuned.conf
new file mode 100644
index 000000000..78c7d19c9
--- /dev/null
+++ b/roles/openshift_node/templates/tuned/openshift-node/tuned.conf
@@ -0,0 +1,10 @@
+#
+# tuned configuration
+#
+
+[main]
+summary=Optimize systems running OpenShift nodes
+include=openshift
+
+[sysctl]
+net.ipv4.tcp_fastopen=3
diff --git a/roles/openshift_node/templates/tuned/openshift/tuned.conf b/roles/openshift_node/templates/tuned/openshift/tuned.conf
new file mode 100644
index 000000000..68ac5dadb
--- /dev/null
+++ b/roles/openshift_node/templates/tuned/openshift/tuned.conf
@@ -0,0 +1,24 @@
+#
+# tuned configuration
+#
+
+[main]
+summary=Optimize systems running OpenShift (parent profile)
+include=${f:virt_check:{{ openshift_tuned_guest_profile }}:throughput-performance}
+
+[selinux]
+avc_cache_threshold=65536
+
+[net]
+nf_conntrack_hashsize=131072
+
+[sysctl]
+kernel.pid_max=131072
+net.netfilter.nf_conntrack_max=1048576
+fs.inotify.max_user_watches=65536
+net.ipv4.neigh.default.gc_thresh1=8192
+net.ipv4.neigh.default.gc_thresh2=32768
+net.ipv4.neigh.default.gc_thresh3=65536
+net.ipv6.neigh.default.gc_thresh1=8192
+net.ipv6.neigh.default.gc_thresh2=32768
+net.ipv6.neigh.default.gc_thresh3=65536
diff --git a/roles/openshift_node/templates/tuned/recommend.conf b/roles/openshift_node/templates/tuned/recommend.conf
new file mode 100644
index 000000000..5fa765798
--- /dev/null
+++ b/roles/openshift_node/templates/tuned/recommend.conf
@@ -0,0 +1,8 @@
+[openshift-node]
+/etc/origin/node/node-config.yaml=.*region=primary
+
+[openshift-control-plane,master]
+/etc/origin/master/master-config.yaml=.*
+
+[openshift-control-plane,node]
+/etc/origin/node/node-config.yaml=.*region=infra