From 801779eeb6f6308f81ae7c48409de7686c04a0aa Mon Sep 17 00:00:00 2001 From: Michael Gugino Date: Wed, 13 Dec 2017 12:42:32 -0500 Subject: Relocate filter plugins to lib_utils This commit relocates filter_plugings to lib_utils, changes the namespacing to prevent unintended use of older versions that may be present in filter_plugins/ directory on existing installs. Add lib_utils to meta depends for roles Also consolidate some plugins into lib_utils from various other areas. Update rpm spec, obsolete plugin rpms. --- roles/etcd/tasks/certificates/fetch_client_certificates_from_ca.yml | 2 +- roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'roles/etcd/tasks/certificates') diff --git a/roles/etcd/tasks/certificates/fetch_client_certificates_from_ca.yml b/roles/etcd/tasks/certificates/fetch_client_certificates_from_ca.yml index 119071a72..d4518554c 100644 --- a/roles/etcd/tasks/certificates/fetch_client_certificates_from_ca.yml +++ b/roles/etcd/tasks/certificates/fetch_client_certificates_from_ca.yml @@ -28,7 +28,7 @@ etcd_client_certs_missing: "{{ true if etcd_certificates_redeploy | default(false) | bool else (False in (g_external_etcd_cert_stat_result.results | default({}) - | oo_collect(attribute='stat.exists') + | lib_utils_oo_collect(attribute='stat.exists') | list)) }}" - name: Ensure generated_certs directory present diff --git a/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml b/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml index deb2301d7..59a6b6590 100644 --- a/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml +++ b/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml @@ -21,7 +21,7 @@ etcd_server_certs_missing: "{{ true if etcd_certificates_redeploy | default(false) | bool else (False in (g_etcd_server_cert_stat_result.results | default({}) - | oo_collect(attribute='stat.exists') + | lib_utils_oo_collect(attribute='stat.exists') | list)) }}" - name: Ensure generated_certs directory present -- cgit v1.2.3 From 7923eb92c86a128504436ba0708c96b655de5269 Mon Sep 17 00:00:00 2001 From: Michael Gugino Date: Wed, 3 Jan 2018 15:10:26 -0500 Subject: Remove become=no from various roles and tasks etcd runs some actions locally to copy certs from the CA cert host. This commit ensures that we respect the end user's intended behavior with become when using 'anisble_become' in the inventory. Other roles with similar tasks have been modified in the same manner. We shouldn't hard-code become behavior as it can be unexpected for the end user. This only currently works in the CI because the CI passes the '-b' argument on the command line, which will override the task behavior. --- .../fetch_client_certificates_from_ca.yml | 23 +++++++--------------- .../fetch_server_certificates_from_ca.yml | 18 ++++------------- roles/openshift_examples/tasks/main.yml | 12 +++++++---- roles/openshift_hosted_templates/tasks/main.yml | 12 +++++++---- roles/openshift_node_certificates/tasks/main.yml | 15 +++----------- 5 files changed, 30 insertions(+), 50 deletions(-) (limited to 'roles/etcd/tasks/certificates') diff --git a/roles/etcd/tasks/certificates/fetch_client_certificates_from_ca.yml b/roles/etcd/tasks/certificates/fetch_client_certificates_from_ca.yml index d4518554c..78578a055 100644 --- a/roles/etcd/tasks/certificates/fetch_client_certificates_from_ca.yml +++ b/roles/etcd/tasks/certificates/fetch_client_certificates_from_ca.yml @@ -79,13 +79,6 @@ when: etcd_client_certs_missing | bool delegate_to: "{{ etcd_ca_host }}" -- name: Create local temp directory for syncing certs - local_action: command mktemp -d /tmp/etcd_certificates-XXXXXXX - register: g_etcd_client_mktemp - changed_when: False - when: etcd_client_certs_missing | bool - become: no - - name: Create a tarball of the etcd certs command: > tar -czvf {{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz @@ -101,8 +94,7 @@ - name: Retrieve the etcd cert tarballs fetch: src: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz" - dest: "{{ g_etcd_client_mktemp.stdout }}/" - flat: yes + dest: "/tmp" fail_on_missing: yes validate_checksum: yes when: etcd_client_certs_missing | bool @@ -116,10 +108,15 @@ - name: Unarchive etcd cert tarballs unarchive: - src: "{{ g_etcd_client_mktemp.stdout }}/{{ etcd_cert_subdir }}.tgz" + src: "/tmp/{{ inventory_hostname }}/{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz" dest: "{{ etcd_cert_config_dir }}" when: etcd_client_certs_missing | bool +- name: Delete temporary directory + local_action: file path="/tmp/{{ inventory_hostname }}" state=absent + changed_when: False + when: etcd_client_certs_missing | bool + - file: path: "{{ etcd_cert_config_dir }}/{{ item }}" owner: root @@ -130,9 +127,3 @@ - "{{ etcd_cert_prefix }}client.key" - "{{ etcd_cert_prefix }}ca.crt" when: etcd_client_certs_missing | bool - -- name: Delete temporary directory - local_action: file path="{{ g_etcd_client_mktemp.stdout }}" state=absent - changed_when: False - when: etcd_client_certs_missing | bool - become: no diff --git a/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml b/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml index 59a6b6590..987380d0c 100644 --- a/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml +++ b/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml @@ -105,13 +105,6 @@ when: etcd_server_certs_missing | bool delegate_to: "{{ etcd_ca_host }}" -- name: Create local temp directory for syncing certs - local_action: command mktemp -d /tmp/etcd_certificates-XXXXXXX - become: no - register: g_etcd_server_mktemp - changed_when: False - when: etcd_server_certs_missing | bool - - name: Create a tarball of the etcd certs command: > tar -czvf {{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz @@ -127,8 +120,7 @@ - name: Retrieve etcd cert tarball fetch: src: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz" - dest: "{{ g_etcd_server_mktemp.stdout }}/" - flat: yes + dest: "/tmp" fail_on_missing: yes validate_checksum: yes when: etcd_server_certs_missing | bool @@ -144,7 +136,7 @@ - name: Unarchive cert tarball unarchive: - src: "{{ g_etcd_server_mktemp.stdout }}/{{ etcd_cert_subdir }}.tgz" + src: "/tmp/{{ inventory_hostname }}/{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz" dest: "{{ etcd_cert_config_dir }}" when: etcd_server_certs_missing | bool @@ -161,8 +153,7 @@ - name: Retrieve etcd ca cert tarball fetch: src: "{{ etcd_generated_certs_dir }}/{{ etcd_ca_name }}.tgz" - dest: "{{ g_etcd_server_mktemp.stdout }}/" - flat: yes + dest: "/tmp" fail_on_missing: yes validate_checksum: yes when: etcd_server_certs_missing | bool @@ -177,8 +168,7 @@ when: etcd_server_certs_missing | bool - name: Delete temporary directory - local_action: file path="{{ g_etcd_server_mktemp.stdout }}" state=absent - become: no + local_action: file path="/tmp/{{ inventory_hostname }}" state=absent changed_when: False when: etcd_server_certs_missing | bool diff --git a/roles/openshift_examples/tasks/main.yml b/roles/openshift_examples/tasks/main.yml index a09a598bd..ff04cdf9c 100644 --- a/roles/openshift_examples/tasks/main.yml +++ b/roles/openshift_examples/tasks/main.yml @@ -13,18 +13,23 @@ # use it either due to changes introduced in Ansible 2.x. - name: Create local temp dir for OpenShift examples copy local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX - become: False register: copy_examples_mktemp run_once: True +- name: Create local temp dir for OpenShift examples copy + local_action: command chmod 755 "{{ copy_examples_mktemp.stdout }}" + run_once: True + - name: Create tar of OpenShift examples local_action: command tar -C "{{ role_path }}/files/examples/{{ content_version }}/" -cvf "{{ copy_examples_mktemp.stdout }}/openshift-examples.tar" . args: # Disables the following warning: # Consider using unarchive module rather than running tar warn: no - become: False - register: copy_examples_tar + +- name: Create local temp dir for OpenShift examples copy + local_action: command chmod 744 "{{ copy_examples_mktemp.stdout }}/openshift-examples.tar" + run_once: True - name: Create the remote OpenShift examples directory file: @@ -38,7 +43,6 @@ dest: "{{ examples_base }}/" - name: Cleanup the OpenShift Examples temp dir - become: False local_action: file dest="{{ copy_examples_mktemp.stdout }}" state=absent # Done copying examples diff --git a/roles/openshift_hosted_templates/tasks/main.yml b/roles/openshift_hosted_templates/tasks/main.yml index b2313c297..672d25b4d 100644 --- a/roles/openshift_hosted_templates/tasks/main.yml +++ b/roles/openshift_hosted_templates/tasks/main.yml @@ -1,20 +1,25 @@ --- - name: Create local temp dir for OpenShift hosted templates copy local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX - become: False register: copy_hosted_templates_mktemp run_once: True # AUDIT:changed_when: not set here because this task actually # creates something +- name: Create local temp dir for OpenShift examples copy + local_action: command chmod 755 "{{ copy_hosted_templates_mktemp.stdout }}" + run_once: True + - name: Create tar of OpenShift examples local_action: command tar -C "{{ role_path }}/files/{{ content_version }}/{{ hosted_deployment_type }}" -cvf "{{ copy_hosted_templates_mktemp.stdout }}/openshift-hosted-templates.tar" . args: # Disables the following warning: # Consider using unarchive module rather than running tar warn: no - become: False - register: copy_hosted_templates_tar + +- name: Create local temp dir for OpenShift examples copy + local_action: command chmod 744 "{{ copy_hosted_templates_mktemp.stdout }}/openshift-hosted-templates.tar" + run_once: True - name: Create remote OpenShift hosted templates directory file: @@ -28,7 +33,6 @@ dest: "{{ hosted_base }}/" - name: Cleanup the OpenShift hosted templates temp dir - become: False local_action: file dest="{{ copy_hosted_templates_mktemp.stdout }}" state=absent - name: Modify registry paths if registry_url is not registry.access.redhat.com diff --git a/roles/openshift_node_certificates/tasks/main.yml b/roles/openshift_node_certificates/tasks/main.yml index e95e38fdf..5f73f3bdc 100644 --- a/roles/openshift_node_certificates/tasks/main.yml +++ b/roles/openshift_node_certificates/tasks/main.yml @@ -94,13 +94,6 @@ delegate_to: "{{ openshift_ca_host }}" run_once: true -- name: Create local temp directory for syncing certs - local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX - register: node_cert_mktemp - changed_when: False - when: node_certs_missing | bool - become: no - - name: Create a tarball of the node config directories command: > tar -czvf {{ openshift_node_generated_config_dir }}.tgz @@ -117,8 +110,7 @@ - name: Retrieve the node config tarballs from the master fetch: src: "{{ openshift_node_generated_config_dir }}.tgz" - dest: "{{ node_cert_mktemp.stdout }}/" - flat: yes + dest: "/tmp" fail_on_missing: yes validate_checksum: yes when: node_certs_missing | bool @@ -132,15 +124,14 @@ - name: Unarchive the tarball on the node unarchive: - src: "{{ node_cert_mktemp.stdout }}/{{ openshift_node_cert_subdir }}.tgz" + src: "/tmp/{{ inventory_hostname }}/{{ openshift_node_generated_config_dir }}.tgz" dest: "{{ openshift_node_cert_dir }}" when: node_certs_missing | bool - name: Delete local temp directory - local_action: file path="{{ node_cert_mktemp.stdout }}" state=absent + local_action: file path="/tmp/{{ inventory_hostname }}" state=absent changed_when: False when: node_certs_missing | bool - become: no - name: Copy OpenShift CA to system CA trust copy: -- cgit v1.2.3 From d3fefc32a727fe3c13159c4e9fe4399f35b487a8 Mon Sep 17 00:00:00 2001 From: Michael Gugino Date: Thu, 4 Jan 2018 23:55:34 -0500 Subject: Move more plugins to lib_utils This commit continues moving plugins into lib_utils. This commit does not move any plugins for add-on roles such as logging and metrics. --- .../private/certificates-backup.yml | 1 + roles/etcd/library/delegated_serial_command.py | 274 ------- .../fetch_client_certificates_from_ca.yml | 1 + .../fetch_server_certificates_from_ca.yml | 2 + .../action_plugins/generate_pv_pvcs_list.py | 157 ++++ roles/lib_utils/filter_plugins/oo_cert_expiry.py | 66 ++ roles/lib_utils/filter_plugins/oo_filters.py | 9 + .../filter_plugins/openshift_aws_filters.py | 74 ++ .../filter_plugins/openshift_hosted_filters.py | 42 ++ roles/lib_utils/filter_plugins/openshift_master.py | 532 +++++++++++++ .../lib_utils/library/delegated_serial_command.py | 274 +++++++ roles/lib_utils/library/openshift_cert_expiry.py | 839 +++++++++++++++++++++ .../library/openshift_container_binary_sync.py | 205 +++++ .../openshift_master_facts_default_predicates.py | 143 ++++ .../openshift_master_facts_default_priorities.py | 117 +++ roles/lib_utils/test/conftest.py | 172 +++++ .../test/openshift_master_facts_bad_input_tests.py | 57 ++ .../test/openshift_master_facts_conftest.py | 54 ++ ...nshift_master_facts_default_predicates_tests.py | 193 +++++ ...nshift_master_facts_default_priorities_tests.py | 167 ++++ roles/lib_utils/test/test_fakeopensslclasses.py | 90 +++ roles/lib_utils/test/test_load_and_handle_cert.py | 67 ++ roles/openshift_aws/defaults/main.yml | 2 + .../filter_plugins/openshift_aws_filters.py | 74 -- roles/openshift_aws/tasks/build_node_group.yml | 1 + roles/openshift_aws/tasks/wait_for_groups.yml | 1 + .../filter_plugins/oo_cert_expiry.py | 66 -- .../library/openshift_cert_expiry.py | 839 --------------------- roles/openshift_certificate_expiry/tasks/main.yml | 4 +- .../openshift_certificate_expiry/test/conftest.py | 119 --- .../test/test_fakeopensslclasses.py | 90 --- .../test/test_load_and_handle_cert.py | 67 -- .../library/openshift_container_binary_sync.py | 205 ----- roles/openshift_cli/tasks/main.yml | 2 + .../openshift_checks/disk_availability.py | 2 +- .../filter_plugins/openshift_hosted_filters.py | 42 -- roles/openshift_hosted/tasks/router.yml | 1 + .../filter_plugins/openshift_logging.py | 9 - roles/openshift_logging_fluentd/defaults/main.yml | 1 + roles/openshift_logging_mux/defaults/main.yml | 1 + roles/openshift_master/tasks/main.yml | 1 + .../tasks/upgrade/upgrade_scheduler.yml | 2 + roles/openshift_master_certificates/tasks/main.yml | 1 + .../filter_plugins/openshift_master.py | 532 ------------- roles/openshift_master_facts/tasks/main.yml | 3 + roles/openshift_master_facts/test/conftest.py | 54 -- .../test/openshift_master_facts_bad_input_tests.py | 57 -- ...nshift_master_facts_default_predicates_tests.py | 193 ----- ...nshift_master_facts_default_priorities_tests.py | 167 ---- .../filter_plugins/openshift_named_certificates.py | 21 - .../action_plugins/generate_pv_pvcs_list.py | 157 ---- roles/openshift_persistent_volumes/tasks/main.yml | 3 +- .../filter_plugins/openshift_sanitize_inventory.py | 10 - .../filter_plugins/openshift_storage_glusterfs.py | 23 - .../tasks/glusterfs_config.yml | 1 + .../tasks/glusterfs_registry.yml | 1 + 56 files changed, 3286 insertions(+), 3002 deletions(-) delete mode 100755 roles/etcd/library/delegated_serial_command.py create mode 100644 roles/lib_utils/action_plugins/generate_pv_pvcs_list.py create mode 100644 roles/lib_utils/filter_plugins/oo_cert_expiry.py create mode 100644 roles/lib_utils/filter_plugins/openshift_aws_filters.py create mode 100644 roles/lib_utils/filter_plugins/openshift_hosted_filters.py create mode 100644 roles/lib_utils/filter_plugins/openshift_master.py create mode 100755 roles/lib_utils/library/delegated_serial_command.py create mode 100644 roles/lib_utils/library/openshift_cert_expiry.py create mode 100644 roles/lib_utils/library/openshift_container_binary_sync.py create mode 100644 roles/lib_utils/lookup_plugins/openshift_master_facts_default_predicates.py create mode 100644 roles/lib_utils/lookup_plugins/openshift_master_facts_default_priorities.py create mode 100644 roles/lib_utils/test/conftest.py create mode 100644 roles/lib_utils/test/openshift_master_facts_bad_input_tests.py create mode 100644 roles/lib_utils/test/openshift_master_facts_conftest.py create mode 100644 roles/lib_utils/test/openshift_master_facts_default_predicates_tests.py create mode 100644 roles/lib_utils/test/openshift_master_facts_default_priorities_tests.py create mode 100644 roles/lib_utils/test/test_fakeopensslclasses.py create mode 100644 roles/lib_utils/test/test_load_and_handle_cert.py delete mode 100644 roles/openshift_aws/filter_plugins/openshift_aws_filters.py delete mode 100644 roles/openshift_certificate_expiry/filter_plugins/oo_cert_expiry.py delete mode 100644 roles/openshift_certificate_expiry/library/openshift_cert_expiry.py delete mode 100644 roles/openshift_certificate_expiry/test/conftest.py delete mode 100644 roles/openshift_certificate_expiry/test/test_fakeopensslclasses.py delete mode 100644 roles/openshift_certificate_expiry/test/test_load_and_handle_cert.py delete mode 100644 roles/openshift_cli/library/openshift_container_binary_sync.py delete mode 100644 roles/openshift_hosted/filter_plugins/openshift_hosted_filters.py delete mode 100644 roles/openshift_master_facts/filter_plugins/openshift_master.py delete mode 100644 roles/openshift_master_facts/test/conftest.py delete mode 100644 roles/openshift_master_facts/test/openshift_master_facts_bad_input_tests.py delete mode 100644 roles/openshift_master_facts/test/openshift_master_facts_default_predicates_tests.py delete mode 100644 roles/openshift_master_facts/test/openshift_master_facts_default_priorities_tests.py delete mode 100644 roles/openshift_named_certificates/filter_plugins/openshift_named_certificates.py delete mode 100644 roles/openshift_persistent_volumes/action_plugins/generate_pv_pvcs_list.py delete mode 100644 roles/openshift_storage_glusterfs/filter_plugins/openshift_storage_glusterfs.py (limited to 'roles/etcd/tasks/certificates') diff --git a/playbooks/openshift-master/private/certificates-backup.yml b/playbooks/openshift-master/private/certificates-backup.yml index 4dbc041b0..56af18ca7 100644 --- a/playbooks/openshift-master/private/certificates-backup.yml +++ b/playbooks/openshift-master/private/certificates-backup.yml @@ -28,6 +28,7 @@ path: "{{ openshift.common.config_base }}/master/{{ item }}" state: absent with_items: + # certificates_to_synchronize is a custom filter in lib_utils - "{{ hostvars[inventory_hostname] | certificates_to_synchronize(include_keys=false, include_ca=false) }}" - "etcd.server.crt" - "etcd.server.key" diff --git a/roles/etcd/library/delegated_serial_command.py b/roles/etcd/library/delegated_serial_command.py deleted file mode 100755 index 0cab1ca88..000000000 --- a/roles/etcd/library/delegated_serial_command.py +++ /dev/null @@ -1,274 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -# (c) 2012, Michael DeHaan , and others -# (c) 2016, Andrew Butcher -# -# This module is derrived from the Ansible command module. -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - - -# pylint: disable=unused-wildcard-import,wildcard-import,unused-import,redefined-builtin - -''' delegated_serial_command ''' - -import datetime -import errno -import glob -import shlex -import os -import fcntl -import time - -DOCUMENTATION = ''' ---- -module: delegated_serial_command -short_description: Executes a command on a remote node -version_added: historical -description: - - The M(command) module takes the command name followed by a list - of space-delimited arguments. - - The given command will be executed on all selected nodes. It - will not be processed through the shell, so variables like - C($HOME) and operations like C("<"), C(">"), C("|"), and C("&") - will not work (use the M(shell) module if you need these - features). - - Creates and maintains a lockfile such that this module will - wait for other invocations to proceed. -options: - command: - description: - - the command to run - required: true - default: null - creates: - description: - - a filename or (since 2.0) glob pattern, when it already - exists, this step will B(not) be run. - required: no - default: null - removes: - description: - - a filename or (since 2.0) glob pattern, when it does not - exist, this step will B(not) be run. - version_added: "0.8" - required: no - default: null - chdir: - description: - - cd into this directory before running the command - version_added: "0.6" - required: false - default: null - executable: - description: - - change the shell used to execute the command. Should be an - absolute path to the executable. - required: false - default: null - version_added: "0.9" - warn: - version_added: "1.8" - default: yes - description: - - if command warnings are on in ansible.cfg, do not warn about - this particular line if set to no/false. - required: false - lockfile: - default: yes - description: - - the lockfile that will be created - timeout: - default: yes - description: - - time in milliseconds to wait to obtain the lock -notes: - - If you want to run a command through the shell (say you are using C(<), - C(>), C(|), etc), you actually want the M(shell) module instead. The - M(command) module is much more secure as it's not affected by the user's - environment. - - " C(creates), C(removes), and C(chdir) can be specified after - the command. For instance, if you only want to run a command if - a certain file does not exist, use this." -author: - - Ansible Core Team - - Michael DeHaan - - Andrew Butcher -''' - -EXAMPLES = ''' -# Example from Ansible Playbooks. -- delegated_serial_command: - command: /sbin/shutdown -t now - -# Run the command if the specified file does not exist. -- delegated_serial_command: - command: /usr/bin/make_database.sh arg1 arg2 - creates: /path/to/database -''' - -# Dict of options and their defaults -OPTIONS = {'chdir': None, - 'creates': None, - 'command': None, - 'executable': None, - 'NO_LOG': None, - 'removes': None, - 'warn': True, - 'lockfile': None, - 'timeout': None} - - -def check_command(commandline): - ''' Check provided command ''' - arguments = {'chown': 'owner', 'chmod': 'mode', 'chgrp': 'group', - 'ln': 'state=link', 'mkdir': 'state=directory', - 'rmdir': 'state=absent', 'rm': 'state=absent', 'touch': 'state=touch'} - commands = {'git': 'git', 'hg': 'hg', 'curl': 'get_url or uri', 'wget': 'get_url or uri', - 'svn': 'subversion', 'service': 'service', - 'mount': 'mount', 'rpm': 'yum, dnf or zypper', 'yum': 'yum', 'apt-get': 'apt', - 'tar': 'unarchive', 'unzip': 'unarchive', 'sed': 'template or lineinfile', - 'rsync': 'synchronize', 'dnf': 'dnf', 'zypper': 'zypper'} - become = ['sudo', 'su', 'pbrun', 'pfexec', 'runas'] - warnings = list() - command = os.path.basename(commandline.split()[0]) - # pylint: disable=line-too-long - if command in arguments: - warnings.append("Consider using file module with {0} rather than running {1}".format(arguments[command], command)) - if command in commands: - warnings.append("Consider using {0} module rather than running {1}".format(commands[command], command)) - if command in become: - warnings.append( - "Consider using 'become', 'become_method', and 'become_user' rather than running {0}".format(command,)) - return warnings - - -# pylint: disable=too-many-statements,too-many-branches,too-many-locals -def main(): - ''' Main module function ''' - module = AnsibleModule( # noqa: F405 - argument_spec=dict( - _uses_shell=dict(type='bool', default=False), - command=dict(required=True), - chdir=dict(), - executable=dict(), - creates=dict(), - removes=dict(), - warn=dict(type='bool', default=True), - lockfile=dict(default='/tmp/delegated_serial_command.lock'), - timeout=dict(type='int', default=30) - ) - ) - - shell = module.params['_uses_shell'] - chdir = module.params['chdir'] - executable = module.params['executable'] - command = module.params['command'] - creates = module.params['creates'] - removes = module.params['removes'] - warn = module.params['warn'] - lockfile = module.params['lockfile'] - timeout = module.params['timeout'] - - if command.strip() == '': - module.fail_json(rc=256, msg="no command given") - - iterated = 0 - lockfd = open(lockfile, 'w+') - while iterated < timeout: - try: - fcntl.flock(lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB) - break - # pylint: disable=invalid-name - except IOError as e: - if e.errno != errno.EAGAIN: - module.fail_json(msg="I/O Error {0}: {1}".format(e.errno, e.strerror)) - else: - iterated += 1 - time.sleep(0.1) - - if chdir: - chdir = os.path.abspath(os.path.expanduser(chdir)) - os.chdir(chdir) - - if creates: - # do not run the command if the line contains creates=filename - # and the filename already exists. This allows idempotence - # of command executions. - path = os.path.expanduser(creates) - if glob.glob(path): - module.exit_json( - cmd=command, - stdout="skipped, since %s exists" % path, - changed=False, - stderr=False, - rc=0 - ) - - if removes: - # do not run the command if the line contains removes=filename - # and the filename does not exist. This allows idempotence - # of command executions. - path = os.path.expanduser(removes) - if not glob.glob(path): - module.exit_json( - cmd=command, - stdout="skipped, since %s does not exist" % path, - changed=False, - stderr=False, - rc=0 - ) - - warnings = list() - if warn: - warnings = check_command(command) - - if not shell: - command = shlex.split(command) - startd = datetime.datetime.now() - - # pylint: disable=invalid-name - rc, out, err = module.run_command(command, executable=executable, use_unsafe_shell=shell) - - fcntl.flock(lockfd, fcntl.LOCK_UN) - lockfd.close() - - endd = datetime.datetime.now() - delta = endd - startd - - if out is None: - out = '' - if err is None: - err = '' - - module.exit_json( - cmd=command, - stdout=out.rstrip("\r\n"), - stderr=err.rstrip("\r\n"), - rc=rc, - start=str(startd), - end=str(endd), - delta=str(delta), - changed=True, - warnings=warnings, - iterated=iterated - ) - - -# import module snippets -# pylint: disable=wrong-import-position -from ansible.module_utils.basic import * # noqa: F402,F403 - -main() diff --git a/roles/etcd/tasks/certificates/fetch_client_certificates_from_ca.yml b/roles/etcd/tasks/certificates/fetch_client_certificates_from_ca.yml index 78578a055..ce295d2f5 100644 --- a/roles/etcd/tasks/certificates/fetch_client_certificates_from_ca.yml +++ b/roles/etcd/tasks/certificates/fetch_client_certificates_from_ca.yml @@ -57,6 +57,7 @@ # Certificates must be signed serially in order to avoid competing # for the serial file. +# delegated_serial_command is a custom module in lib_utils - name: Sign and create the client crt delegated_serial_command: command: > diff --git a/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml b/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml index 987380d0c..7c8b87d99 100644 --- a/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml +++ b/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml @@ -50,6 +50,7 @@ # Certificates must be signed serially in order to avoid competing # for the serial file. +# delegated_serial_command is a custom module in lib_utils - name: Sign and create the server crt delegated_serial_command: command: > @@ -83,6 +84,7 @@ # Certificates must be signed serially in order to avoid competing # for the serial file. +# delegated_serial_command is a custom module in lib_utils - name: Sign and create the peer crt delegated_serial_command: command: > diff --git a/roles/lib_utils/action_plugins/generate_pv_pvcs_list.py b/roles/lib_utils/action_plugins/generate_pv_pvcs_list.py new file mode 100644 index 000000000..eb13a58ba --- /dev/null +++ b/roles/lib_utils/action_plugins/generate_pv_pvcs_list.py @@ -0,0 +1,157 @@ +""" +Ansible action plugin to generate pv and pvc dictionaries lists +""" + +from ansible.plugins.action import ActionBase +from ansible import errors + + +class ActionModule(ActionBase): + """Action plugin to execute health checks.""" + + def get_templated(self, var_to_template): + """Return a properly templated ansible variable""" + return self._templar.template(self.task_vars.get(var_to_template)) + + def build_common(self, varname=None): + """Retrieve common variables for each pv and pvc type""" + volume = self.get_templated(str(varname) + '_volume_name') + size = self.get_templated(str(varname) + '_volume_size') + labels = self.task_vars.get(str(varname) + '_labels') + if labels: + labels = self._templar.template(labels) + else: + labels = dict() + access_modes = self.get_templated(str(varname) + '_access_modes') + return (volume, size, labels, access_modes) + + def build_pv_nfs(self, varname=None): + """Build pv dictionary for nfs storage type""" + host = self.task_vars.get(str(varname) + '_host') + if host: + self._templar.template(host) + elif host is None: + groups = self.task_vars.get('groups') + default_group_name = self.get_templated('openshift_persistent_volumes_default_nfs_group') + if groups and default_group_name and default_group_name in groups and len(groups[default_group_name]) > 0: + host = groups['oo_nfs_to_config'][0] + else: + raise errors.AnsibleModuleError("|failed no storage host detected") + volume, size, labels, access_modes = self.build_common(varname=varname) + directory = self.get_templated(str(varname) + '_nfs_directory') + path = directory + '/' + volume + return dict( + name="{0}-volume".format(volume), + capacity=size, + labels=labels, + access_modes=access_modes, + storage=dict( + nfs=dict( + server=host, + path=path))) + + def build_pv_openstack(self, varname=None): + """Build pv dictionary for openstack storage type""" + volume, size, labels, access_modes = self.build_common(varname=varname) + filesystem = self.get_templated(str(varname) + '_openstack_filesystem') + volume_id = self.get_templated(str(varname) + '_openstack_volumeID') + return dict( + name="{0}-volume".format(volume), + capacity=size, + labels=labels, + access_modes=access_modes, + storage=dict( + cinder=dict( + fsType=filesystem, + volumeID=volume_id))) + + def build_pv_glusterfs(self, varname=None): + """Build pv dictionary for glusterfs storage type""" + volume, size, labels, access_modes = self.build_common(varname=varname) + endpoints = self.get_templated(str(varname) + '_glusterfs_endpoints') + path = self.get_templated(str(varname) + '_glusterfs_path') + read_only = self.get_templated(str(varname) + '_glusterfs_readOnly') + return dict( + name="{0}-volume".format(volume), + capacity=size, + labels=labels, + access_modes=access_modes, + storage=dict( + glusterfs=dict( + endpoints=endpoints, + path=path, + readOnly=read_only))) + + def build_pv_dict(self, varname=None): + """Check for the existence of PV variables""" + kind = self.task_vars.get(str(varname) + '_kind') + if kind: + kind = self._templar.template(kind) + create_pv = self.task_vars.get(str(varname) + '_create_pv') + if create_pv and self._templar.template(create_pv): + if kind == 'nfs': + return self.build_pv_nfs(varname=varname) + + elif kind == 'openstack': + return self.build_pv_openstack(varname=varname) + + elif kind == 'glusterfs': + return self.build_pv_glusterfs(varname=varname) + + elif not (kind == 'object' or kind == 'dynamic'): + msg = "|failed invalid storage kind '{0}' for component '{1}'".format( + kind, + varname) + raise errors.AnsibleModuleError(msg) + return None + + def build_pvc_dict(self, varname=None): + """Check for the existence of PVC variables""" + kind = self.task_vars.get(str(varname) + '_kind') + if kind: + kind = self._templar.template(kind) + create_pv = self.task_vars.get(str(varname) + '_create_pv') + if create_pv: + create_pv = self._templar.template(create_pv) + create_pvc = self.task_vars.get(str(varname) + '_create_pvc') + if create_pvc: + create_pvc = self._templar.template(create_pvc) + if kind != 'object' and create_pv and create_pvc: + volume, size, _, access_modes = self.build_common(varname=varname) + return dict( + name="{0}-claim".format(volume), + capacity=size, + access_modes=access_modes) + return None + + def run(self, tmp=None, task_vars=None): + """Run generate_pv_pvcs_list action plugin""" + result = super(ActionModule, self).run(tmp, task_vars) + # Ignore settting self.task_vars outside of init. + # pylint: disable=W0201 + self.task_vars = task_vars or {} + + result["changed"] = False + result["failed"] = False + result["msg"] = "persistent_volumes list and persistent_volume_claims list created" + vars_to_check = ['openshift_hosted_registry_storage', + 'openshift_hosted_router_storage', + 'openshift_hosted_etcd_storage', + 'openshift_logging_storage', + 'openshift_loggingops_storage', + 'openshift_metrics_storage', + 'openshift_prometheus_storage', + 'openshift_prometheus_alertmanager_storage', + 'openshift_prometheus_alertbuffer_storage'] + persistent_volumes = [] + persistent_volume_claims = [] + for varname in vars_to_check: + pv_dict = self.build_pv_dict(varname) + if pv_dict: + persistent_volumes.append(pv_dict) + pvc_dict = self.build_pvc_dict(varname) + if pvc_dict: + persistent_volume_claims.append(pvc_dict) + result["persistent_volumes"] = persistent_volumes + result["persistent_volume_claims"] = persistent_volume_claims + return result diff --git a/roles/lib_utils/filter_plugins/oo_cert_expiry.py b/roles/lib_utils/filter_plugins/oo_cert_expiry.py new file mode 100644 index 000000000..58b228fee --- /dev/null +++ b/roles/lib_utils/filter_plugins/oo_cert_expiry.py @@ -0,0 +1,66 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" +Custom filters for use in openshift-ansible +""" + + +# Disabling too-many-public-methods, since filter methods are necessarily +# public +# pylint: disable=too-many-public-methods +class FilterModule(object): + """ Custom ansible filters """ + + @staticmethod + def oo_cert_expiry_results_to_json(hostvars, play_hosts): + """Takes results (`hostvars`) from the openshift_cert_expiry role +check and serializes them into proper machine-readable JSON +output. This filter parameter **MUST** be the playbook `hostvars` +variable. The `play_hosts` parameter is so we know what to loop over +when we're extrating the values. + +Returns: + +Results are collected into two top-level keys under the `json_results` +dict: + +* `json_results.data` [dict] - Each individual host check result, keys are hostnames +* `json_results.summary` [dict] - Summary of number of `warning` and `expired` +certificates + +Example playbook usage: + + - name: Generate expiration results JSON + run_once: yes + delegate_to: localhost + when: openshift_certificate_expiry_save_json_results|bool + copy: + content: "{{ hostvars|oo_cert_expiry_results_to_json() }}" + dest: "{{ openshift_certificate_expiry_json_results_path }}" + + """ + json_result = { + 'data': {}, + 'summary': {}, + } + + for host in play_hosts: + json_result['data'][host] = hostvars[host]['check_results']['check_results'] + + total_warnings = sum([hostvars[h]['check_results']['summary']['warning'] for h in play_hosts]) + total_expired = sum([hostvars[h]['check_results']['summary']['expired'] for h in play_hosts]) + total_ok = sum([hostvars[h]['check_results']['summary']['ok'] for h in play_hosts]) + total_total = sum([hostvars[h]['check_results']['summary']['total'] for h in play_hosts]) + + json_result['summary']['warning'] = total_warnings + json_result['summary']['expired'] = total_expired + json_result['summary']['ok'] = total_ok + json_result['summary']['total'] = total_total + + return json_result + + def filters(self): + """ returns a mapping of filters to methods """ + return { + "oo_cert_expiry_results_to_json": self.oo_cert_expiry_results_to_json, + } diff --git a/roles/lib_utils/filter_plugins/oo_filters.py b/roles/lib_utils/filter_plugins/oo_filters.py index a2ea287cf..fc14b5633 100644 --- a/roles/lib_utils/filter_plugins/oo_filters.py +++ b/roles/lib_utils/filter_plugins/oo_filters.py @@ -589,6 +589,14 @@ that result to this filter plugin. return secret_name +def map_from_pairs(source, delim="="): + ''' Returns a dict given the source and delim delimited ''' + if source == '': + return dict() + + return dict(item.split(delim) for item in source.split(",")) + + class FilterModule(object): """ Custom ansible filter mapping """ @@ -618,4 +626,5 @@ class FilterModule(object): "lib_utils_oo_contains_rule": lib_utils_oo_contains_rule, "lib_utils_oo_selector_to_string_list": lib_utils_oo_selector_to_string_list, "lib_utils_oo_filter_sa_secrets": lib_utils_oo_filter_sa_secrets, + "map_from_pairs": map_from_pairs } diff --git a/roles/lib_utils/filter_plugins/openshift_aws_filters.py b/roles/lib_utils/filter_plugins/openshift_aws_filters.py new file mode 100644 index 000000000..dfcb11da3 --- /dev/null +++ b/roles/lib_utils/filter_plugins/openshift_aws_filters.py @@ -0,0 +1,74 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +''' +Custom filters for use in openshift_aws +''' + +from ansible import errors + + +class FilterModule(object): + ''' Custom ansible filters for use by openshift_aws role''' + + @staticmethod + def scale_groups_serial(scale_group_info, upgrade=False): + ''' This function will determine what the deployment serial should be and return it + + Search through the tags and find the deployment_serial tag. Once found, + determine if an increment is needed during an upgrade. + if upgrade is true then increment the serial and return it + else return the serial + ''' + if scale_group_info == []: + return 1 + + scale_group_info = scale_group_info[0] + + if not isinstance(scale_group_info, dict): + raise errors.AnsibleFilterError("|filter plugin failed: Expected scale_group_info to be a dict") + + serial = None + + for tag in scale_group_info['tags']: + if tag['key'] == 'deployment_serial': + serial = int(tag['value']) + if upgrade: + serial += 1 + break + else: + raise errors.AnsibleFilterError("|filter plugin failed: deployment_serial tag was not found") + + return serial + + @staticmethod + def scale_groups_match_capacity(scale_group_info): + ''' This function will verify that the scale group instance count matches + the scale group desired capacity + + ''' + for scale_group in scale_group_info: + if scale_group['desired_capacity'] != len(scale_group['instances']): + return False + + return True + + @staticmethod + def build_instance_tags(clusterid): + ''' This function will return a dictionary of the instance tags. + + The main desire to have this inside of a filter_plugin is that we + need to build the following key. + + {"kubernetes.io/cluster/{{ openshift_aws_clusterid }}": "{{ openshift_aws_clusterid}}"} + + ''' + tags = {'clusterid': clusterid, + 'kubernetes.io/cluster/{}'.format(clusterid): clusterid} + + return tags + + def filters(self): + ''' returns a mapping of filters to methods ''' + return {'build_instance_tags': self.build_instance_tags, + 'scale_groups_match_capacity': self.scale_groups_match_capacity, + 'scale_groups_serial': self.scale_groups_serial} diff --git a/roles/lib_utils/filter_plugins/openshift_hosted_filters.py b/roles/lib_utils/filter_plugins/openshift_hosted_filters.py new file mode 100644 index 000000000..003ce5f9e --- /dev/null +++ b/roles/lib_utils/filter_plugins/openshift_hosted_filters.py @@ -0,0 +1,42 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +''' +Custom filters for use in openshift_hosted +''' + + +class FilterModule(object): + ''' Custom ansible filters for use by openshift_hosted role''' + + @staticmethod + def get_router_replicas(replicas=None, router_nodes=None): + ''' This function will return the number of replicas + based on the results from the defined + openshift_hosted_router_replicas OR + the query from oc_obj on openshift nodes with a selector OR + default to 1 + + ''' + # We always use what they've specified if they've specified a value + if replicas is not None: + return replicas + + replicas = 1 + + # Ignore boolean expression limit of 5. + # pylint: disable=too-many-boolean-expressions + if (isinstance(router_nodes, dict) and + 'results' in router_nodes and + 'results' in router_nodes['results'] and + isinstance(router_nodes['results']['results'], list) and + len(router_nodes['results']['results']) > 0 and + 'items' in router_nodes['results']['results'][0]): + + if len(router_nodes['results']['results'][0]['items']) > 0: + replicas = len(router_nodes['results']['results'][0]['items']) + + return replicas + + def filters(self): + ''' returns a mapping of filters to methods ''' + return {'get_router_replicas': self.get_router_replicas} diff --git a/roles/lib_utils/filter_plugins/openshift_master.py b/roles/lib_utils/filter_plugins/openshift_master.py new file mode 100644 index 000000000..ff15f693b --- /dev/null +++ b/roles/lib_utils/filter_plugins/openshift_master.py @@ -0,0 +1,532 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +''' +Custom filters for use in openshift-master +''' +import copy +import sys + +from ansible import errors +from ansible.parsing.yaml.dumper import AnsibleDumper +from ansible.plugins.filter.core import to_bool as ansible_bool + +# ansible.compat.six goes away with Ansible 2.4 +try: + from ansible.compat.six import string_types, u +except ImportError: + from ansible.module_utils.six import string_types, u + +import yaml + + +class IdentityProviderBase(object): + """ IdentityProviderBase + + Attributes: + name (str): Identity provider Name + login (bool): Is this identity provider a login provider? + challenge (bool): Is this identity provider a challenge provider? + provider (dict): Provider specific config + _idp (dict): internal copy of the IDP dict passed in + _required (list): List of lists of strings for required attributes + _optional (list): List of lists of strings for optional attributes + _allow_additional (bool): Does this provider support attributes + not in _required and _optional + + Args: + api_version(str): OpenShift config version + idp (dict): idp config dict + + Raises: + AnsibleFilterError: + """ + # disabling this check since the number of instance attributes are + # necessary for this class + # pylint: disable=too-many-instance-attributes + def __init__(self, api_version, idp): + if api_version not in ['v1']: + raise errors.AnsibleFilterError("|failed api version {0} unknown".format(api_version)) + + self._idp = copy.deepcopy(idp) + + if 'name' not in self._idp: + raise errors.AnsibleFilterError("|failed identity provider missing a name") + + if 'kind' not in self._idp: + raise errors.AnsibleFilterError("|failed identity provider missing a kind") + + self.name = self._idp.pop('name') + self.login = ansible_bool(self._idp.pop('login', False)) + self.challenge = ansible_bool(self._idp.pop('challenge', False)) + self.provider = dict(apiVersion=api_version, kind=self._idp.pop('kind')) + + mm_keys = ('mappingMethod', 'mapping_method') + mapping_method = None + for key in mm_keys: + if key in self._idp: + mapping_method = self._idp.pop(key) + if mapping_method is None: + mapping_method = self.get_default('mappingMethod') + self.mapping_method = mapping_method + + valid_mapping_methods = ['add', 'claim', 'generate', 'lookup'] + if self.mapping_method not in valid_mapping_methods: + raise errors.AnsibleFilterError("|failed unknown mapping method " + "for provider {0}".format(self.__class__.__name__)) + self._required = [] + self._optional = [] + self._allow_additional = True + + @staticmethod + def validate_idp_list(idp_list): + ''' validates a list of idps ''' + names = [x.name for x in idp_list] + if len(set(names)) != len(names): + raise errors.AnsibleFilterError("|failed more than one provider configured with the same name") + + for idp in idp_list: + idp.validate() + + def validate(self): + ''' validate an instance of this idp class ''' + pass + + @staticmethod + def get_default(key): + ''' get a default value for a given key ''' + if key == 'mappingMethod': + return 'claim' + else: + return None + + def set_provider_item(self, items, required=False): + ''' set a provider item based on the list of item names provided. ''' + for item in items: + provider_key = items[0] + if item in self._idp: + self.provider[provider_key] = self._idp.pop(item) + break + else: + default = self.get_default(provider_key) + if default is not None: + self.provider[provider_key] = default + elif required: + raise errors.AnsibleFilterError("|failed provider {0} missing " + "required key {1}".format(self.__class__.__name__, provider_key)) + + def set_provider_items(self): + ''' set the provider items for this idp ''' + for items in self._required: + self.set_provider_item(items, True) + for items in self._optional: + self.set_provider_item(items) + if self._allow_additional: + for key in self._idp.keys(): + self.set_provider_item([key]) + else: + if len(self._idp) > 0: + raise errors.AnsibleFilterError("|failed provider {0} " + "contains unknown keys " + "{1}".format(self.__class__.__name__, ', '.join(self._idp.keys()))) + + def to_dict(self): + ''' translate this idp to a dictionary ''' + return dict(name=self.name, challenge=self.challenge, + login=self.login, mappingMethod=self.mapping_method, + provider=self.provider) + + +class LDAPPasswordIdentityProvider(IdentityProviderBase): + """ LDAPPasswordIdentityProvider + + Attributes: + + Args: + api_version(str): OpenShift config version + idp (dict): idp config dict + + Raises: + AnsibleFilterError: + """ + def __init__(self, api_version, idp): + super(LDAPPasswordIdentityProvider, self).__init__(api_version, idp) + self._allow_additional = False + self._required += [['attributes'], ['url'], ['insecure']] + self._optional += [['ca'], + ['bindDN', 'bind_dn'], + ['bindPassword', 'bind_password']] + + self._idp['insecure'] = ansible_bool(self._idp.pop('insecure', False)) + + if 'attributes' in self._idp and 'preferred_username' in self._idp['attributes']: + pref_user = self._idp['attributes'].pop('preferred_username') + self._idp['attributes']['preferredUsername'] = pref_user + + def validate(self): + ''' validate this idp instance ''' + if not isinstance(self.provider['attributes'], dict): + raise errors.AnsibleFilterError("|failed attributes for provider " + "{0} must be a dictionary".format(self.__class__.__name__)) + + attrs = ['id', 'email', 'name', 'preferredUsername'] + for attr in attrs: + if attr in self.provider['attributes'] and not isinstance(self.provider['attributes'][attr], list): + raise errors.AnsibleFilterError("|failed {0} attribute for " + "provider {1} must be a list".format(attr, self.__class__.__name__)) + + unknown_attrs = set(self.provider['attributes'].keys()) - set(attrs) + if len(unknown_attrs) > 0: + raise errors.AnsibleFilterError("|failed provider {0} has unknown " + "attributes: {1}".format(self.__class__.__name__, ', '.join(unknown_attrs))) + + +class KeystonePasswordIdentityProvider(IdentityProviderBase): + """ KeystoneIdentityProvider + + Attributes: + + Args: + api_version(str): OpenShift config version + idp (dict): idp config dict + + Raises: + AnsibleFilterError: + """ + def __init__(self, api_version, idp): + super(KeystonePasswordIdentityProvider, self).__init__(api_version, idp) + self._allow_additional = False + self._required += [['url'], ['domainName', 'domain_name']] + self._optional += [['ca'], ['certFile', 'cert_file'], ['keyFile', 'key_file']] + + +class RequestHeaderIdentityProvider(IdentityProviderBase): + """ RequestHeaderIdentityProvider + + Attributes: + + Args: + api_version(str): OpenShift config version + idp (dict): idp config dict + + Raises: + AnsibleFilterError: + """ + def __init__(self, api_version, idp): + super(RequestHeaderIdentityProvider, self).__init__(api_version, idp) + self._allow_additional = False + self._required += [['headers']] + self._optional += [['challengeURL', 'challenge_url'], + ['loginURL', 'login_url'], + ['clientCA', 'client_ca'], + ['clientCommonNames', 'client_common_names'], + ['emailHeaders', 'email_headers'], + ['nameHeaders', 'name_headers'], + ['preferredUsernameHeaders', 'preferred_username_headers']] + + def validate(self): + ''' validate this idp instance ''' + if not isinstance(self.provider['headers'], list): + raise errors.AnsibleFilterError("|failed headers for provider {0} " + "must be a list".format(self.__class__.__name__)) + + +class AllowAllPasswordIdentityProvider(IdentityProviderBase): + """ AllowAllPasswordIdentityProvider + + Attributes: + + Args: + api_version(str): OpenShift config version + idp (dict): idp config dict + + Raises: + AnsibleFilterError: + """ + def __init__(self, api_version, idp): + super(AllowAllPasswordIdentityProvider, self).__init__(api_version, idp) + self._allow_additional = False + + +class DenyAllPasswordIdentityProvider(IdentityProviderBase): + """ DenyAllPasswordIdentityProvider + + Attributes: + + Args: + api_version(str): OpenShift config version + idp (dict): idp config dict + + Raises: + AnsibleFilterError: + """ + def __init__(self, api_version, idp): + super(DenyAllPasswordIdentityProvider, self).__init__(api_version, idp) + self._allow_additional = False + + +class HTPasswdPasswordIdentityProvider(IdentityProviderBase): + """ HTPasswdPasswordIdentity + + Attributes: + + Args: + api_version(str): OpenShift config version + idp (dict): idp config dict + + Raises: + AnsibleFilterError: + """ + def __init__(self, api_version, idp): + super(HTPasswdPasswordIdentityProvider, self).__init__(api_version, idp) + self._allow_additional = False + self._required += [['file', 'filename', 'fileName', 'file_name']] + + @staticmethod + def get_default(key): + if key == 'file': + return '/etc/origin/htpasswd' + else: + return IdentityProviderBase.get_default(key) + + +class BasicAuthPasswordIdentityProvider(IdentityProviderBase): + """ BasicAuthPasswordIdentityProvider + + Attributes: + + Args: + api_version(str): OpenShift config version + idp (dict): idp config dict + + Raises: + AnsibleFilterError: + """ + def __init__(self, api_version, idp): + super(BasicAuthPasswordIdentityProvider, self).__init__(api_version, idp) + self._allow_additional = False + self._required += [['url']] + self._optional += [['ca'], ['certFile', 'cert_file'], ['keyFile', 'key_file']] + + +class IdentityProviderOauthBase(IdentityProviderBase): + """ IdentityProviderOauthBase + + Attributes: + + Args: + api_version(str): OpenShift config version + idp (dict): idp config dict + + Raises: + AnsibleFilterError: + """ + def __init__(self, api_version, idp): + super(IdentityProviderOauthBase, self).__init__(api_version, idp) + self._allow_additional = False + self._required += [['clientID', 'client_id'], ['clientSecret', 'client_secret']] + + def validate(self): + ''' validate an instance of this idp class ''' + pass + + +class OpenIDIdentityProvider(IdentityProviderOauthBase): + """ OpenIDIdentityProvider + + Attributes: + + Args: + api_version(str): OpenShift config version + idp (dict): idp config dict + + Raises: + AnsibleFilterError: + """ + def __init__(self, api_version, idp): + IdentityProviderOauthBase.__init__(self, api_version, idp) + self._required += [['claims'], ['urls']] + self._optional += [['ca'], + ['extraScopes'], + ['extraAuthorizeParameters']] + if 'claims' in self._idp and 'preferred_username' in self._idp['claims']: + pref_user = self._idp['claims'].pop('preferred_username') + self._idp['claims']['preferredUsername'] = pref_user + if 'urls' in self._idp and 'user_info' in self._idp['urls']: + user_info = self._idp['urls'].pop('user_info') + self._idp['urls']['userInfo'] = user_info + if 'extra_scopes' in self._idp: + self._idp['extraScopes'] = self._idp.pop('extra_scopes') + if 'extra_authorize_parameters' in self._idp: + self._idp['extraAuthorizeParameters'] = self._idp.pop('extra_authorize_parameters') + + def validate(self): + ''' validate this idp instance ''' + if not isinstance(self.provider['claims'], dict): + raise errors.AnsibleFilterError("|failed claims for provider {0} " + "must be a dictionary".format(self.__class__.__name__)) + + for var, var_type in (('extraScopes', list), ('extraAuthorizeParameters', dict)): + if var in self.provider and not isinstance(self.provider[var], var_type): + raise errors.AnsibleFilterError("|failed {1} for provider " + "{0} must be a {2}".format(self.__class__.__name__, + var, + var_type.__class__.__name__)) + + required_claims = ['id'] + optional_claims = ['email', 'name', 'preferredUsername'] + all_claims = required_claims + optional_claims + + for claim in required_claims: + if claim in required_claims and claim not in self.provider['claims']: + raise errors.AnsibleFilterError("|failed {0} claim missing " + "for provider {1}".format(claim, self.__class__.__name__)) + + for claim in all_claims: + if claim in self.provider['claims'] and not isinstance(self.provider['claims'][claim], list): + raise errors.AnsibleFilterError("|failed {0} claims for " + "provider {1} must be a list".format(claim, self.__class__.__name__)) + + unknown_claims = set(self.provider['claims'].keys()) - set(all_claims) + if len(unknown_claims) > 0: + raise errors.AnsibleFilterError("|failed provider {0} has unknown " + "claims: {1}".format(self.__class__.__name__, ', '.join(unknown_claims))) + + if not isinstance(self.provider['urls'], dict): + raise errors.AnsibleFilterError("|failed urls for provider {0} " + "must be a dictionary".format(self.__class__.__name__)) + + required_urls = ['authorize', 'token'] + optional_urls = ['userInfo'] + all_urls = required_urls + optional_urls + + for url in required_urls: + if url not in self.provider['urls']: + raise errors.AnsibleFilterError("|failed {0} url missing for " + "provider {1}".format(url, self.__class__.__name__)) + + unknown_urls = set(self.provider['urls'].keys()) - set(all_urls) + if len(unknown_urls) > 0: + raise errors.AnsibleFilterError("|failed provider {0} has unknown " + "urls: {1}".format(self.__class__.__name__, ', '.join(unknown_urls))) + + +class GoogleIdentityProvider(IdentityProviderOauthBase): + """ GoogleIdentityProvider + + Attributes: + + Args: + api_version(str): OpenShift config version + idp (dict): idp config dict + + Raises: + AnsibleFilterError: + """ + def __init__(self, api_version, idp): + IdentityProviderOauthBase.__init__(self, api_version, idp) + self._optional += [['hostedDomain', 'hosted_domain']] + + def validate(self): + ''' validate this idp instance ''' + if self.challenge: + raise errors.AnsibleFilterError("|failed provider {0} does not " + "allow challenge authentication".format(self.__class__.__name__)) + + +class GitHubIdentityProvider(IdentityProviderOauthBase): + """ GitHubIdentityProvider + + Attributes: + + Args: + api_version(str): OpenShift config version + idp (dict): idp config dict + + Raises: + AnsibleFilterError: + """ + def __init__(self, api_version, idp): + IdentityProviderOauthBase.__init__(self, api_version, idp) + self._optional += [['organizations'], + ['teams']] + + def validate(self): + ''' validate this idp instance ''' + if self.challenge: + raise errors.AnsibleFilterError("|failed provider {0} does not " + "allow challenge authentication".format(self.__class__.__name__)) + + +class FilterModule(object): + ''' Custom ansible filters for use by the openshift_master role''' + + @staticmethod + def translate_idps(idps, api_version): + ''' Translates a list of dictionaries into a valid identityProviders config ''' + idp_list = [] + + if not isinstance(idps, list): + raise errors.AnsibleFilterError("|failed expects to filter on a list of identity providers") + for idp in idps: + if not isinstance(idp, dict): + raise errors.AnsibleFilterError("|failed identity providers must be a list of dictionaries") + + cur_module = sys.modules[__name__] + idp_class = getattr(cur_module, idp['kind'], None) + idp_inst = idp_class(api_version, idp) if idp_class is not None else IdentityProviderBase(api_version, idp) + idp_inst.set_provider_items() + idp_list.append(idp_inst) + + IdentityProviderBase.validate_idp_list(idp_list) + return u(yaml.dump([idp.to_dict() for idp in idp_list], + allow_unicode=True, + default_flow_style=False, + width=float("inf"), + Dumper=AnsibleDumper)) + + @staticmethod + def certificates_to_synchronize(hostvars, include_keys=True, include_ca=True): + ''' Return certificates to synchronize based on facts. ''' + if not issubclass(type(hostvars), dict): + raise errors.AnsibleFilterError("|failed expects hostvars is a dict") + certs = ['admin.crt', + 'admin.key', + 'admin.kubeconfig', + 'master.kubelet-client.crt', + 'master.kubelet-client.key', + 'master.proxy-client.crt', + 'master.proxy-client.key', + 'service-signer.crt', + 'service-signer.key'] + if bool(include_ca): + certs += ['ca.crt', 'ca.key', 'ca-bundle.crt', 'client-ca-bundle.crt'] + if bool(include_keys): + certs += ['serviceaccounts.private.key', + 'serviceaccounts.public.key'] + return certs + + @staticmethod + def oo_htpasswd_users_from_file(file_contents): + ''' return a dictionary of htpasswd users from htpasswd file contents ''' + htpasswd_entries = {} + if not isinstance(file_contents, string_types): + raise errors.AnsibleFilterError("failed, expects to filter on a string") + for line in file_contents.splitlines(): + user = None + passwd = None + if len(line) == 0: + continue + if ':' in line: + user, passwd = line.split(':', 1) + + if user is None or len(user) == 0 or passwd is None or len(passwd) == 0: + error_msg = "failed, expects each line to be a colon separated string representing the user and passwd" + raise errors.AnsibleFilterError(error_msg) + htpasswd_entries[user] = passwd + return htpasswd_entries + + def filters(self): + ''' returns a mapping of filters to methods ''' + return {"translate_idps": self.translate_idps, + "certificates_to_synchronize": self.certificates_to_synchronize, + "oo_htpasswd_users_from_file": self.oo_htpasswd_users_from_file} diff --git a/roles/lib_utils/library/delegated_serial_command.py b/roles/lib_utils/library/delegated_serial_command.py new file mode 100755 index 000000000..0cab1ca88 --- /dev/null +++ b/roles/lib_utils/library/delegated_serial_command.py @@ -0,0 +1,274 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# (c) 2012, Michael DeHaan , and others +# (c) 2016, Andrew Butcher +# +# This module is derrived from the Ansible command module. +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + + +# pylint: disable=unused-wildcard-import,wildcard-import,unused-import,redefined-builtin + +''' delegated_serial_command ''' + +import datetime +import errno +import glob +import shlex +import os +import fcntl +import time + +DOCUMENTATION = ''' +--- +module: delegated_serial_command +short_description: Executes a command on a remote node +version_added: historical +description: + - The M(command) module takes the command name followed by a list + of space-delimited arguments. + - The given command will be executed on all selected nodes. It + will not be processed through the shell, so variables like + C($HOME) and operations like C("<"), C(">"), C("|"), and C("&") + will not work (use the M(shell) module if you need these + features). + - Creates and maintains a lockfile such that this module will + wait for other invocations to proceed. +options: + command: + description: + - the command to run + required: true + default: null + creates: + description: + - a filename or (since 2.0) glob pattern, when it already + exists, this step will B(not) be run. + required: no + default: null + removes: + description: + - a filename or (since 2.0) glob pattern, when it does not + exist, this step will B(not) be run. + version_added: "0.8" + required: no + default: null + chdir: + description: + - cd into this directory before running the command + version_added: "0.6" + required: false + default: null + executable: + description: + - change the shell used to execute the command. Should be an + absolute path to the executable. + required: false + default: null + version_added: "0.9" + warn: + version_added: "1.8" + default: yes + description: + - if command warnings are on in ansible.cfg, do not warn about + this particular line if set to no/false. + required: false + lockfile: + default: yes + description: + - the lockfile that will be created + timeout: + default: yes + description: + - time in milliseconds to wait to obtain the lock +notes: + - If you want to run a command through the shell (say you are using C(<), + C(>), C(|), etc), you actually want the M(shell) module instead. The + M(command) module is much more secure as it's not affected by the user's + environment. + - " C(creates), C(removes), and C(chdir) can be specified after + the command. For instance, if you only want to run a command if + a certain file does not exist, use this." +author: + - Ansible Core Team + - Michael DeHaan + - Andrew Butcher +''' + +EXAMPLES = ''' +# Example from Ansible Playbooks. +- delegated_serial_command: + command: /sbin/shutdown -t now + +# Run the command if the specified file does not exist. +- delegated_serial_command: + command: /usr/bin/make_database.sh arg1 arg2 + creates: /path/to/database +''' + +# Dict of options and their defaults +OPTIONS = {'chdir': None, + 'creates': None, + 'command': None, + 'executable': None, + 'NO_LOG': None, + 'removes': None, + 'warn': True, + 'lockfile': None, + 'timeout': None} + + +def check_command(commandline): + ''' Check provided command ''' + arguments = {'chown': 'owner', 'chmod': 'mode', 'chgrp': 'group', + 'ln': 'state=link', 'mkdir': 'state=directory', + 'rmdir': 'state=absent', 'rm': 'state=absent', 'touch': 'state=touch'} + commands = {'git': 'git', 'hg': 'hg', 'curl': 'get_url or uri', 'wget': 'get_url or uri', + 'svn': 'subversion', 'service': 'service', + 'mount': 'mount', 'rpm': 'yum, dnf or zypper', 'yum': 'yum', 'apt-get': 'apt', + 'tar': 'unarchive', 'unzip': 'unarchive', 'sed': 'template or lineinfile', + 'rsync': 'synchronize', 'dnf': 'dnf', 'zypper': 'zypper'} + become = ['sudo', 'su', 'pbrun', 'pfexec', 'runas'] + warnings = list() + command = os.path.basename(commandline.split()[0]) + # pylint: disable=line-too-long + if command in arguments: + warnings.append("Consider using file module with {0} rather than running {1}".format(arguments[command], command)) + if command in commands: + warnings.append("Consider using {0} module rather than running {1}".format(commands[command], command)) + if command in become: + warnings.append( + "Consider using 'become', 'become_method', and 'become_user' rather than running {0}".format(command,)) + return warnings + + +# pylint: disable=too-many-statements,too-many-branches,too-many-locals +def main(): + ''' Main module function ''' + module = AnsibleModule( # noqa: F405 + argument_spec=dict( + _uses_shell=dict(type='bool', default=False), + command=dict(required=True), + chdir=dict(), + executable=dict(), + creates=dict(), + removes=dict(), + warn=dict(type='bool', default=True), + lockfile=dict(default='/tmp/delegated_serial_command.lock'), + timeout=dict(type='int', default=30) + ) + ) + + shell = module.params['_uses_shell'] + chdir = module.params['chdir'] + executable = module.params['executable'] + command = module.params['command'] + creates = module.params['creates'] + removes = module.params['removes'] + warn = module.params['warn'] + lockfile = module.params['lockfile'] + timeout = module.params['timeout'] + + if command.strip() == '': + module.fail_json(rc=256, msg="no command given") + + iterated = 0 + lockfd = open(lockfile, 'w+') + while iterated < timeout: + try: + fcntl.flock(lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + # pylint: disable=invalid-name + except IOError as e: + if e.errno != errno.EAGAIN: + module.fail_json(msg="I/O Error {0}: {1}".format(e.errno, e.strerror)) + else: + iterated += 1 + time.sleep(0.1) + + if chdir: + chdir = os.path.abspath(os.path.expanduser(chdir)) + os.chdir(chdir) + + if creates: + # do not run the command if the line contains creates=filename + # and the filename already exists. This allows idempotence + # of command executions. + path = os.path.expanduser(creates) + if glob.glob(path): + module.exit_json( + cmd=command, + stdout="skipped, since %s exists" % path, + changed=False, + stderr=False, + rc=0 + ) + + if removes: + # do not run the command if the line contains removes=filename + # and the filename does not exist. This allows idempotence + # of command executions. + path = os.path.expanduser(removes) + if not glob.glob(path): + module.exit_json( + cmd=command, + stdout="skipped, since %s does not exist" % path, + changed=False, + stderr=False, + rc=0 + ) + + warnings = list() + if warn: + warnings = check_command(command) + + if not shell: + command = shlex.split(command) + startd = datetime.datetime.now() + + # pylint: disable=invalid-name + rc, out, err = module.run_command(command, executable=executable, use_unsafe_shell=shell) + + fcntl.flock(lockfd, fcntl.LOCK_UN) + lockfd.close() + + endd = datetime.datetime.now() + delta = endd - startd + + if out is None: + out = '' + if err is None: + err = '' + + module.exit_json( + cmd=command, + stdout=out.rstrip("\r\n"), + stderr=err.rstrip("\r\n"), + rc=rc, + start=str(startd), + end=str(endd), + delta=str(delta), + changed=True, + warnings=warnings, + iterated=iterated + ) + + +# import module snippets +# pylint: disable=wrong-import-position +from ansible.module_utils.basic import * # noqa: F402,F403 + +main() diff --git a/roles/lib_utils/library/openshift_cert_expiry.py b/roles/lib_utils/library/openshift_cert_expiry.py new file mode 100644 index 000000000..e355266b0 --- /dev/null +++ b/roles/lib_utils/library/openshift_cert_expiry.py @@ -0,0 +1,839 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# pylint: disable=line-too-long,invalid-name + +"""For details on this module see DOCUMENTATION (below)""" + +import base64 +import datetime +import io +import os +import subprocess +import yaml + +# pylint import-error disabled because pylint cannot find the package +# when installed in a virtualenv +from ansible.module_utils.six.moves import configparser # pylint: disable=import-error +from ansible.module_utils.basic import AnsibleModule + +try: + # You can comment this import out and include a 'pass' in this + # block if you're manually testing this module on a NON-ATOMIC + # HOST (or any host that just doesn't have PyOpenSSL + # available). That will force the `load_and_handle_cert` function + # to use the Fake OpenSSL classes. + import OpenSSL.crypto + HAS_OPENSSL = True +except ImportError: + # Some platforms (such as RHEL Atomic) may not have the Python + # OpenSSL library installed. In this case we will use a manual + # work-around to parse each certificate. + # + # Check for 'OpenSSL.crypto' in `sys.modules` later. + HAS_OPENSSL = False + +DOCUMENTATION = ''' +--- +module: openshift_cert_expiry +short_description: Check OpenShift Container Platform (OCP) and Kube certificate expirations on a cluster +description: + - The M(openshift_cert_expiry) module has two basic functions: to flag certificates which will expire in a set window of time from now, and to notify you about certificates which have already expired. + - When the module finishes, a summary of the examination is returned. Each certificate in the summary has a C(health) key with a value of one of the following: + - C(ok) - not expired, and outside of the expiration C(warning_days) window. + - C(warning) - not expired, but will expire between now and the C(warning_days) window. + - C(expired) - an expired certificate. + - Certificate flagging follow this logic: + - If the expiration date is before now then the certificate is classified as C(expired). + - The certificates time to live (expiration date - now) is calculated, if that time window is less than C(warning_days) the certificate is classified as C(warning). + - All other conditions are classified as C(ok). + - The following keys are ALSO present in the certificate summary: + - C(cert_cn) - The common name of the certificate (additional CNs present in SAN extensions are omitted) + - C(days_remaining) - The number of days until the certificate expires. + - C(expiry) - The date the certificate expires on. + - C(path) - The full path to the certificate on the examined host. +version_added: "1.0" +options: + config_base: + description: + - Base path to OCP system settings. + required: false + default: /etc/origin + warning_days: + description: + - Flag certificates which will expire in C(warning_days) days from now. + required: false + default: 30 + show_all: + description: + - Enable this option to show analysis of ALL certificates examined by this module. + - By default only certificates which have expired, or will expire within the C(warning_days) window will be reported. + required: false + default: false + +author: "Tim Bielawa (@tbielawa) " +''' + +EXAMPLES = ''' +# Default invocation, only notify about expired certificates or certificates which will expire within 30 days from now +- openshift_cert_expiry: + +# Expand the warning window to show certificates expiring within a year from now +- openshift_cert_expiry: warning_days=365 + +# Show expired, soon to expire (now + 30 days), and all other certificates examined +- openshift_cert_expiry: show_all=true +''' + + +class FakeOpenSSLCertificate(object): + """This provides a rough mock of what you get from +`OpenSSL.crypto.load_certificate()`. This is a work-around for +platforms missing the Python OpenSSL library. + """ + def __init__(self, cert_string): + """`cert_string` is a certificate in the form you get from running a +.crt through 'openssl x509 -in CERT.cert -text'""" + self.cert_string = cert_string + self.serial = None + self.subject = None + self.extensions = [] + self.not_after = None + self._parse_cert() + + def _parse_cert(self): + """Manually parse the certificate line by line""" + self.extensions = [] + + PARSING_ALT_NAMES = False + PARSING_HEX_SERIAL = False + for line in self.cert_string.split('\n'): + l = line.strip() + if PARSING_ALT_NAMES: + # We're parsing a 'Subject Alternative Name' line + self.extensions.append( + FakeOpenSSLCertificateSANExtension(l)) + + PARSING_ALT_NAMES = False + continue + + if PARSING_HEX_SERIAL: + # Hex serials arrive colon-delimited + serial_raw = l.replace(':', '') + # Convert to decimal + self.serial = int('0x' + serial_raw, base=16) + PARSING_HEX_SERIAL = False + continue + + # parse out the bits that we can + if l.startswith('Serial Number:'): + # Decimal format: + # Serial Number: 11 (0xb) + # => 11 + # Hex Format (large serials): + # Serial Number: + # 0a:de:eb:24:04:75:ab:56:39:14:e9:5a:22:e2:85:bf + # => 14449739080294792594019643629255165375 + if l.endswith(':'): + PARSING_HEX_SERIAL = True + continue + self.serial = int(l.split()[-2]) + + elif l.startswith('Not After :'): + # Not After : Feb 7 18:19:35 2019 GMT + # => strptime(str, '%b %d %H:%M:%S %Y %Z') + # => strftime('%Y%m%d%H%M%SZ') + # => 20190207181935Z + not_after_raw = l.partition(' : ')[-1] + # Last item: ('Not After', ' : ', 'Feb 7 18:19:35 2019 GMT') + not_after_parsed = datetime.datetime.strptime(not_after_raw, '%b %d %H:%M:%S %Y %Z') + self.not_after = not_after_parsed.strftime('%Y%m%d%H%M%SZ') + + elif l.startswith('X509v3 Subject Alternative Name:'): + PARSING_ALT_NAMES = True + continue + + elif l.startswith('Subject:'): + # O = system:nodes, CN = system:node:m01.example.com + self.subject = FakeOpenSSLCertificateSubjects(l.partition(': ')[-1]) + + def get_serial_number(self): + """Return the serial number of the cert""" + return self.serial + + def get_subject(self): + """Subjects must implement get_components() and return dicts or +tuples. An 'openssl x509 -in CERT.cert -text' with 'Subject': + + Subject: Subject: O=system:nodes, CN=system:node:m01.example.com + +might return: [('O=system', 'nodes'), ('CN=system', 'node:m01.example.com')] + """ + return self.subject + + def get_extension(self, i): + """Extensions must implement get_short_name() and return the string +'subjectAltName'""" + return self.extensions[i] + + def get_extension_count(self): + """ get_extension_count """ + return len(self.extensions) + + def get_notAfter(self): + """Returns a date stamp as a string in the form +'20180922170439Z'. strptime the result with format param: +'%Y%m%d%H%M%SZ'.""" + return self.not_after + + +class FakeOpenSSLCertificateSANExtension(object): # pylint: disable=too-few-public-methods + """Mocks what happens when `get_extension` is called on a certificate +object""" + + def __init__(self, san_string): + """With `san_string` as you get from: + + $ openssl x509 -in certificate.crt -text + """ + self.san_string = san_string + self.short_name = 'subjectAltName' + + def get_short_name(self): + """Return the 'type' of this extension. It's always the same though +because we only care about subjectAltName's""" + return self.short_name + + def __str__(self): + """Return this extension and the value as a simple string""" + return self.san_string + + +# pylint: disable=too-few-public-methods +class FakeOpenSSLCertificateSubjects(object): + """Mocks what happens when `get_subject` is called on a certificate +object""" + + def __init__(self, subject_string): + """With `subject_string` as you get from: + + $ openssl x509 -in certificate.crt -text + """ + self.subjects = [] + for s in subject_string.split(', '): + name, _, value = s.partition(' = ') + self.subjects.append((name, value)) + + def get_components(self): + """Returns a list of tuples""" + return self.subjects + + +###################################################################### +def filter_paths(path_list): + """`path_list` - A list of file paths to check. Only files which exist +will be returned + """ + return [p for p in path_list if os.path.exists(os.path.realpath(p))] + + +# pylint: disable=too-many-locals,too-many-branches +# +# TODO: Break this function down into smaller chunks +def load_and_handle_cert(cert_string, now, base64decode=False, ans_module=None): + """Load a certificate, split off the good parts, and return some +useful data + +Params: + +- `cert_string` (string) - a certificate loaded into a string object +- `now` (datetime) - a datetime object of the time to calculate the certificate 'time_remaining' against +- `base64decode` (bool) - run base64.b64decode() on the input +- `ans_module` (AnsibleModule) - The AnsibleModule object for this module (so we can raise errors) + +Returns: +A tuple of the form: + (cert_subject, cert_expiry_date, time_remaining, cert_serial_number) + """ + if base64decode: + _cert_string = base64.b64decode(cert_string).decode('utf-8') + else: + _cert_string = cert_string + + # Disable this. We 'redefine' the type because we are working + # around a missing library on the target host. + # + # pylint: disable=redefined-variable-type + if HAS_OPENSSL: + # No work-around required + cert_loaded = OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, _cert_string) + else: + # Missing library, work-around required. Run the 'openssl' + # command on it to decode it + cmd = 'openssl x509 -text' + try: + openssl_proc = subprocess.Popen(cmd.split(), + stdout=subprocess.PIPE, + stdin=subprocess.PIPE) + except OSError: + ans_module.fail_json(msg="Error: The 'OpenSSL' python library and CLI command were not found on the target host. Unable to parse any certificates. This host will not be included in generated reports.") + else: + openssl_decoded = openssl_proc.communicate(_cert_string.encode('utf-8'))[0].decode('utf-8') + cert_loaded = FakeOpenSSLCertificate(openssl_decoded) + + ###################################################################### + # Read all possible names from the cert + cert_subjects = [] + for name, value in cert_loaded.get_subject().get_components(): + if isinstance(name, bytes) or isinstance(value, bytes): + name = name.decode('utf-8') + value = value.decode('utf-8') + cert_subjects.append('{}:{}'.format(name, value)) + + # To read SANs from a cert we must read the subjectAltName + # extension from the X509 Object. What makes this more difficult + # is that pyOpenSSL does not give extensions as an iterable + san = None + for i in range(cert_loaded.get_extension_count()): + ext = cert_loaded.get_extension(i) + if ext.get_short_name() == 'subjectAltName': + san = ext + + if san is not None: + # The X509Extension object for subjectAltName prints as a + # string with the alt names separated by a comma and a + # space. Split the string by ', ' and then add our new names + # to the list of existing names + cert_subjects.extend(str(san).split(', ')) + + cert_subject = ', '.join(cert_subjects) + ###################################################################### + + # Grab the expiration date + not_after = cert_loaded.get_notAfter() + # example get_notAfter() => 20180922170439Z + if isinstance(not_after, bytes): + not_after = not_after.decode('utf-8') + + cert_expiry_date = datetime.datetime.strptime( + not_after, + '%Y%m%d%H%M%SZ') + + time_remaining = cert_expiry_date - now + + return (cert_subject, cert_expiry_date, time_remaining, cert_loaded.get_serial_number()) + + +def classify_cert(cert_meta, now, time_remaining, expire_window, cert_list): + """Given metadata about a certificate under examination, classify it + into one of three categories, 'ok', 'warning', and 'expired'. + +Params: + +- `cert_meta` dict - A dict with certificate metadata. Required fields + include: 'cert_cn', 'path', 'expiry', 'days_remaining', 'health'. +- `now` (datetime) - a datetime object of the time to calculate the certificate 'time_remaining' against +- `time_remaining` (datetime.timedelta) - a timedelta for how long until the cert expires +- `expire_window` (datetime.timedelta) - a timedelta for how long the warning window is +- `cert_list` list - A list to shove the classified cert into + +Return: +- `cert_list` - The updated list of classified certificates + """ + expiry_str = str(cert_meta['expiry']) + # Categorization + if cert_meta['expiry'] < now: + # This already expired, must NOTIFY + cert_meta['health'] = 'expired' + elif time_remaining < expire_window: + # WARN about this upcoming expirations + cert_meta['health'] = 'warning' + else: + # Not expired or about to expire + cert_meta['health'] = 'ok' + + cert_meta['expiry'] = expiry_str + cert_meta['serial_hex'] = hex(int(cert_meta['serial'])) + cert_list.append(cert_meta) + return cert_list + + +def tabulate_summary(certificates, kubeconfigs, etcd_certs, router_certs, registry_certs): + """Calculate the summary text for when the module finishes +running. This includes counts of each classification and what have +you. + +Params: + +- `certificates` (list of dicts) - Processed `expire_check_result` + dicts with filled in `health` keys for system certificates. +- `kubeconfigs` - as above for kubeconfigs +- `etcd_certs` - as above for etcd certs + +Return: + +- `summary_results` (dict) - Counts of each cert type classification + and total items examined. + """ + items = certificates + kubeconfigs + etcd_certs + router_certs + registry_certs + + summary_results = { + 'system_certificates': len(certificates), + 'kubeconfig_certificates': len(kubeconfigs), + 'etcd_certificates': len(etcd_certs), + 'router_certs': len(router_certs), + 'registry_certs': len(registry_certs), + 'total': len(items), + 'ok': 0, + 'warning': 0, + 'expired': 0 + } + + summary_results['expired'] = len([c for c in items if c['health'] == 'expired']) + summary_results['warning'] = len([c for c in items if c['health'] == 'warning']) + summary_results['ok'] = len([c for c in items if c['health'] == 'ok']) + + return summary_results + + +###################################################################### +# This is our module MAIN function after all, so there's bound to be a +# lot of code bundled up into one block +# +# Reason: These checks are disabled because the issue was introduced +# during a period where the pylint checks weren't enabled for this file +# Status: temporarily disabled pending future refactoring +# pylint: disable=too-many-locals,too-many-statements,too-many-branches +def main(): + """This module examines certificates (in various forms) which compose +an OpenShift Container Platform cluster + """ + + module = AnsibleModule( + argument_spec=dict( + config_base=dict( + required=False, + default="/etc/origin", + type='str'), + warning_days=dict( + required=False, + default=30, + type='int'), + show_all=dict( + required=False, + default=False, + type='bool') + ), + supports_check_mode=True, + ) + + # Basic scaffolding for OpenShift specific certs + openshift_base_config_path = os.path.realpath(module.params['config_base']) + openshift_master_config_path = os.path.join(openshift_base_config_path, + "master", "master-config.yaml") + openshift_node_config_path = os.path.join(openshift_base_config_path, + "node", "node-config.yaml") + openshift_cert_check_paths = [ + openshift_master_config_path, + openshift_node_config_path, + ] + + # Paths for Kubeconfigs. Additional kubeconfigs are conditionally + # checked later in the code + master_kube_configs = ['admin', 'openshift-master', + 'openshift-node', 'openshift-router', + 'openshift-registry'] + + kubeconfig_paths = [] + for m_kube_config in master_kube_configs: + kubeconfig_paths.append( + os.path.join(openshift_base_config_path, "master", m_kube_config + ".kubeconfig") + ) + + # Validate some paths we have the ability to do ahead of time + openshift_cert_check_paths = filter_paths(openshift_cert_check_paths) + kubeconfig_paths = filter_paths(kubeconfig_paths) + + # etcd, where do you hide your certs? Used when parsing etcd.conf + etcd_cert_params = [ + "ETCD_CA_FILE", + "ETCD_CERT_FILE", + "ETCD_PEER_CA_FILE", + "ETCD_PEER_CERT_FILE", + ] + + # Expiry checking stuff + now = datetime.datetime.now() + # todo, catch exception for invalid input and return a fail_json + warning_days = int(module.params['warning_days']) + expire_window = datetime.timedelta(days=warning_days) + + # Module stuff + # + # The results of our cert checking to return from the task call + check_results = {} + check_results['meta'] = {} + check_results['meta']['warning_days'] = warning_days + check_results['meta']['checked_at_time'] = str(now) + check_results['meta']['warn_before_date'] = str(now + expire_window) + check_results['meta']['show_all'] = str(module.params['show_all']) + # All the analyzed certs accumulate here + ocp_certs = [] + + ###################################################################### + # Sure, why not? Let's enable check mode. + if module.check_mode: + check_results['ocp_certs'] = [] + module.exit_json( + check_results=check_results, + msg="Checked 0 total certificates. Expired/Warning/OK: 0/0/0. Warning window: %s days" % module.params['warning_days'], + rc=0, + changed=False + ) + + ###################################################################### + # Check for OpenShift Container Platform specific certs + ###################################################################### + for os_cert in filter_paths(openshift_cert_check_paths): + # Open up that config file and locate the cert and CA + with io.open(os_cert, 'r', encoding='utf-8') as fp: + cert_meta = {} + cfg = yaml.load(fp) + # cert files are specified in parsed `fp` as relative to the path + # of the original config file. 'master-config.yaml' with certFile + # = 'foo.crt' implies that 'foo.crt' is in the same + # directory. certFile = '../foo.crt' is in the parent directory. + cfg_path = os.path.dirname(fp.name) + cert_meta['certFile'] = os.path.join(cfg_path, cfg['servingInfo']['certFile']) + cert_meta['clientCA'] = os.path.join(cfg_path, cfg['servingInfo']['clientCA']) + + ###################################################################### + # Load the certificate and the CA, parse their expiration dates into + # datetime objects so we can manipulate them later + for v in cert_meta.values(): + with io.open(v, 'r', encoding='utf-8') as fp: + cert = fp.read() + (cert_subject, + cert_expiry_date, + time_remaining, + cert_serial) = load_and_handle_cert(cert, now, ans_module=module) + + expire_check_result = { + 'cert_cn': cert_subject, + 'path': fp.name, + 'expiry': cert_expiry_date, + 'days_remaining': time_remaining.days, + 'health': None, + 'serial': cert_serial + } + + classify_cert(expire_check_result, now, time_remaining, expire_window, ocp_certs) + + ###################################################################### + # /Check for OpenShift Container Platform specific certs + ###################################################################### + + ###################################################################### + # Check service Kubeconfigs + ###################################################################### + kubeconfigs = [] + + # There may be additional kubeconfigs to check, but their naming + # is less predictable than the ones we've already assembled. + + try: + # Try to read the standard 'node-config.yaml' file to check if + # this host is a node. + with io.open(openshift_node_config_path, 'r', encoding='utf-8') as fp: + cfg = yaml.load(fp) + + # OK, the config file exists, therefore this is a + # node. Nodes have their own kubeconfig files to + # communicate with the master API. Let's read the relative + # path to that file from the node config. + node_masterKubeConfig = cfg['masterKubeConfig'] + # As before, the path to the 'masterKubeConfig' file is + # relative to `fp` + cfg_path = os.path.dirname(fp.name) + node_kubeconfig = os.path.join(cfg_path, node_masterKubeConfig) + + with io.open(node_kubeconfig, 'r', encoding='utf8') as fp: + # Read in the nodes kubeconfig file and grab the good stuff + cfg = yaml.load(fp) + + c = cfg['users'][0]['user']['client-certificate-data'] + (cert_subject, + cert_expiry_date, + time_remaining, + cert_serial) = load_and_handle_cert(c, now, base64decode=True, ans_module=module) + + expire_check_result = { + 'cert_cn': cert_subject, + 'path': fp.name, + 'expiry': cert_expiry_date, + 'days_remaining': time_remaining.days, + 'health': None, + 'serial': cert_serial + } + + classify_cert(expire_check_result, now, time_remaining, expire_window, kubeconfigs) + except IOError: + # This is not a node + pass + + for kube in filter_paths(kubeconfig_paths): + with io.open(kube, 'r', encoding='utf-8') as fp: + # TODO: Maybe consider catching exceptions here? + cfg = yaml.load(fp) + + # Per conversation, "the kubeconfigs you care about: + # admin, router, registry should all be single + # value". Following that advice we only grab the data for + # the user at index 0 in the 'users' list. There should + # not be more than one user. + c = cfg['users'][0]['user']['client-certificate-data'] + (cert_subject, + cert_expiry_date, + time_remaining, + cert_serial) = load_and_handle_cert(c, now, base64decode=True, ans_module=module) + + expire_check_result = { + 'cert_cn': cert_subject, + 'path': fp.name, + 'expiry': cert_expiry_date, + 'days_remaining': time_remaining.days, + 'health': None, + 'serial': cert_serial + } + + classify_cert(expire_check_result, now, time_remaining, expire_window, kubeconfigs) + + ###################################################################### + # /Check service Kubeconfigs + ###################################################################### + + ###################################################################### + # Check etcd certs + # + # Two things to check: 'external' etcd, and embedded etcd. + ###################################################################### + # FIRST: The 'external' etcd + # + # Some values may be duplicated, make this a set for now so we + # unique them all + etcd_certs_to_check = set([]) + etcd_certs = [] + etcd_cert_params.append('dne') + try: + with io.open('/etc/etcd/etcd.conf', 'r', encoding='utf-8') as fp: + # Add dummy header section. + config = io.StringIO() + config.write(u'[ETCD]\n') + config.write(fp.read().replace('%', '%%')) + config.seek(0, os.SEEK_SET) + + etcd_config = configparser.ConfigParser() + etcd_config.readfp(config) + + for param in etcd_cert_params: + try: + etcd_certs_to_check.add(etcd_config.get('ETCD', param)) + except configparser.NoOptionError: + # That parameter does not exist, oh well... + pass + except IOError: + # No etcd to see here, move along + pass + + for etcd_cert in filter_paths(etcd_certs_to_check): + with io.open(etcd_cert, 'r', encoding='utf-8') as fp: + c = fp.read() + (cert_subject, + cert_expiry_date, + time_remaining, + cert_serial) = load_and_handle_cert(c, now, ans_module=module) + + expire_check_result = { + 'cert_cn': cert_subject, + 'path': fp.name, + 'expiry': cert_expiry_date, + 'days_remaining': time_remaining.days, + 'health': None, + 'serial': cert_serial + } + + classify_cert(expire_check_result, now, time_remaining, expire_window, etcd_certs) + + ###################################################################### + # Now the embedded etcd + ###################################################################### + try: + with io.open('/etc/origin/master/master-config.yaml', 'r', encoding='utf-8') as fp: + cfg = yaml.load(fp) + except IOError: + # Not present + pass + else: + if cfg.get('etcdConfig', {}).get('servingInfo', {}).get('certFile', None) is not None: + # This is embedded + etcd_crt_name = cfg['etcdConfig']['servingInfo']['certFile'] + else: + # Not embedded + etcd_crt_name = None + + if etcd_crt_name is not None: + # etcd_crt_name is relative to the location of the + # master-config.yaml file + cfg_path = os.path.dirname(fp.name) + etcd_cert = os.path.join(cfg_path, etcd_crt_name) + with open(etcd_cert, 'r') as etcd_fp: + (cert_subject, + cert_expiry_date, + time_remaining, + cert_serial) = load_and_handle_cert(etcd_fp.read(), now, ans_module=module) + + expire_check_result = { + 'cert_cn': cert_subject, + 'path': etcd_fp.name, + 'expiry': cert_expiry_date, + 'days_remaining': time_remaining.days, + 'health': None, + 'serial': cert_serial + } + + classify_cert(expire_check_result, now, time_remaining, expire_window, etcd_certs) + + ###################################################################### + # /Check etcd certs + ###################################################################### + + ###################################################################### + # Check router/registry certs + # + # These are saved as secrets in etcd. That means that we can not + # simply read a file to grab the data. Instead we're going to + # subprocess out to the 'oc get' command. On non-masters this + # command will fail, that is expected so we catch that exception. + ###################################################################### + router_certs = [] + registry_certs = [] + + ###################################################################### + # First the router certs + try: + router_secrets_raw = subprocess.Popen('oc get -n default secret router-certs -o yaml'.split(), + stdout=subprocess.PIPE) + router_ds = yaml.load(router_secrets_raw.communicate()[0]) + router_c = router_ds['data']['tls.crt'] + router_path = router_ds['metadata']['selfLink'] + except TypeError: + # YAML couldn't load the result, this is not a master + pass + except OSError: + # The OC command doesn't exist here. Move along. + pass + else: + (cert_subject, + cert_expiry_date, + time_remaining, + cert_serial) = load_and_handle_cert(router_c, now, base64decode=True, ans_module=module) + + expire_check_result = { + 'cert_cn': cert_subject, + 'path': router_path, + 'expiry': cert_expiry_date, + 'days_remaining': time_remaining.days, + 'health': None, + 'serial': cert_serial + } + + classify_cert(expire_check_result, now, time_remaining, expire_window, router_certs) + + ###################################################################### + # Now for registry + try: + registry_secrets_raw = subprocess.Popen('oc get -n default secret registry-certificates -o yaml'.split(), + stdout=subprocess.PIPE) + registry_ds = yaml.load(registry_secrets_raw.communicate()[0]) + registry_c = registry_ds['data']['registry.crt'] + registry_path = registry_ds['metadata']['selfLink'] + except TypeError: + # YAML couldn't load the result, this is not a master + pass + except OSError: + # The OC command doesn't exist here. Move along. + pass + else: + (cert_subject, + cert_expiry_date, + time_remaining, + cert_serial) = load_and_handle_cert(registry_c, now, base64decode=True, ans_module=module) + + expire_check_result = { + 'cert_cn': cert_subject, + 'path': registry_path, + 'expiry': cert_expiry_date, + 'days_remaining': time_remaining.days, + 'health': None, + 'serial': cert_serial + } + + classify_cert(expire_check_result, now, time_remaining, expire_window, registry_certs) + + ###################################################################### + # /Check router/registry certs + ###################################################################### + + res = tabulate_summary(ocp_certs, kubeconfigs, etcd_certs, router_certs, registry_certs) + + msg = "Checked {count} total certificates. Expired/Warning/OK: {exp}/{warn}/{ok}. Warning window: {window} days".format( + count=res['total'], + exp=res['expired'], + warn=res['warning'], + ok=res['ok'], + window=int(module.params['warning_days']), + ) + + # By default we only return detailed information about expired or + # warning certificates. If show_all is true then we will print all + # the certificates examined. + if not module.params['show_all']: + check_results['ocp_certs'] = [crt for crt in ocp_certs if crt['health'] in ['expired', 'warning']] + check_results['kubeconfigs'] = [crt for crt in kubeconfigs if crt['health'] in ['expired', 'warning']] + check_results['etcd'] = [crt for crt in etcd_certs if crt['health'] in ['expired', 'warning']] + check_results['registry'] = [crt for crt in registry_certs if crt['health'] in ['expired', 'warning']] + check_results['router'] = [crt for crt in router_certs if crt['health'] in ['expired', 'warning']] + else: + check_results['ocp_certs'] = ocp_certs + check_results['kubeconfigs'] = kubeconfigs + check_results['etcd'] = etcd_certs + check_results['registry'] = registry_certs + check_results['router'] = router_certs + + # Sort the final results to report in order of ascending safety + # time. That is to say, the certificates which will expire sooner + # will be at the front of the list and certificates which will + # expire later are at the end. Router and registry certs should be + # limited to just 1 result, so don't bother sorting those. + def cert_key(item): + ''' return the days_remaining key ''' + return item['days_remaining'] + + check_results['ocp_certs'] = sorted(check_results['ocp_certs'], key=cert_key) + check_results['kubeconfigs'] = sorted(check_results['kubeconfigs'], key=cert_key) + check_results['etcd'] = sorted(check_results['etcd'], key=cert_key) + + # This module will never change anything, but we might want to + # change the return code parameter if there is some catastrophic + # error we noticed earlier + module.exit_json( + check_results=check_results, + summary=res, + msg=msg, + rc=0, + changed=False + ) + + +if __name__ == '__main__': + main() diff --git a/roles/lib_utils/library/openshift_container_binary_sync.py b/roles/lib_utils/library/openshift_container_binary_sync.py new file mode 100644 index 000000000..440b8ec28 --- /dev/null +++ b/roles/lib_utils/library/openshift_container_binary_sync.py @@ -0,0 +1,205 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# pylint: disable=missing-docstring,invalid-name + +import random +import tempfile +import shutil +import os.path + +# pylint: disable=redefined-builtin,wildcard-import,unused-wildcard-import +from ansible.module_utils.basic import * # noqa: F403 + + +DOCUMENTATION = ''' +--- +module: openshift_container_binary_sync +short_description: Copies OpenShift binaries out of the given image tag to host system. +''' + + +class BinarySyncError(Exception): + def __init__(self, msg): + super(BinarySyncError, self).__init__(msg) + self.msg = msg + + +# pylint: disable=too-few-public-methods,too-many-instance-attributes +class BinarySyncer(object): + """ + Syncs the openshift, oc, and kubectl binaries/symlinks out of + a container onto the host system. + """ + + def __init__(self, module, image, tag, backend): + self.module = module + self.changed = False + self.output = [] + self.bin_dir = '/usr/local/bin' + self._image = image + self.tag = tag + self.backend = backend + self.temp_dir = None # TBD + + def sync(self): + if self.backend == 'atomic': + return self._sync_atomic() + + return self._sync_docker() + + def _sync_atomic(self): + self.temp_dir = tempfile.mkdtemp() + temp_dir_mount = tempfile.mkdtemp() + try: + image_spec = '%s:%s' % (self.image, self.tag) + rc, stdout, stderr = self.module.run_command(['atomic', 'mount', + '--storage', "ostree", + image_spec, temp_dir_mount]) + if rc: + raise BinarySyncError("Error mounting image. stdout=%s, stderr=%s" % + (stdout, stderr)) + for i in ["openshift", "oc"]: + src_file = os.path.join(temp_dir_mount, "usr/bin", i) + shutil.copy(src_file, self.temp_dir) + + self._sync_binaries() + finally: + self.module.run_command(['atomic', 'umount', temp_dir_mount]) + shutil.rmtree(temp_dir_mount) + shutil.rmtree(self.temp_dir) + + def _sync_docker(self): + container_name = "openshift-cli-%s" % random.randint(1, 100000) + rc, stdout, stderr = self.module.run_command(['docker', 'create', '--name', + container_name, '%s:%s' % (self.image, self.tag)]) + if rc: + raise BinarySyncError("Error creating temporary docker container. stdout=%s, stderr=%s" % + (stdout, stderr)) + self.output.append(stdout) + try: + self.temp_dir = tempfile.mkdtemp() + self.output.append("Using temp dir: %s" % self.temp_dir) + + rc, stdout, stderr = self.module.run_command(['docker', 'cp', "%s:/usr/bin/openshift" % container_name, + self.temp_dir]) + if rc: + raise BinarySyncError("Error copying file from docker container: stdout=%s, stderr=%s" % + (stdout, stderr)) + + rc, stdout, stderr = self.module.run_command(['docker', 'cp', "%s:/usr/bin/oc" % container_name, + self.temp_dir]) + if rc: + raise BinarySyncError("Error copying file from docker container: stdout=%s, stderr=%s" % + (stdout, stderr)) + + self._sync_binaries() + finally: + shutil.rmtree(self.temp_dir) + self.module.run_command(['docker', 'rm', container_name]) + + def _sync_binaries(self): + self._sync_binary('openshift') + + # In older versions, oc was a symlink to openshift: + if os.path.islink(os.path.join(self.temp_dir, 'oc')): + self._sync_symlink('oc', 'openshift') + else: + self._sync_binary('oc') + + # Ensure correct symlinks created: + self._sync_symlink('kubectl', 'openshift') + + # Remove old oadm binary + if os.path.exists(os.path.join(self.bin_dir, 'oadm')): + os.remove(os.path.join(self.bin_dir, 'oadm')) + + def _sync_symlink(self, binary_name, link_to): + """ Ensure the given binary name exists and links to the expected binary. """ + + # The symlink we are creating: + link_path = os.path.join(self.bin_dir, binary_name) + + # The expected file we should be linking to: + link_dest = os.path.join(self.bin_dir, link_to) + + if not os.path.exists(link_path) or \ + not os.path.islink(link_path) or \ + os.path.realpath(link_path) != os.path.realpath(link_dest): + if os.path.exists(link_path): + os.remove(link_path) + os.symlink(link_to, os.path.join(self.bin_dir, binary_name)) + self.output.append("Symlinked %s to %s." % (link_path, link_dest)) + self.changed = True + + def _sync_binary(self, binary_name): + src_path = os.path.join(self.temp_dir, binary_name) + dest_path = os.path.join(self.bin_dir, binary_name) + incoming_checksum = self.module.run_command(['sha256sum', src_path])[1] + if not os.path.exists(dest_path) or self.module.run_command(['sha256sum', dest_path])[1] != incoming_checksum: + + # See: https://github.com/openshift/openshift-ansible/issues/4965 + if os.path.islink(dest_path): + os.unlink(dest_path) + self.output.append('Removed old symlink {} before copying binary.'.format(dest_path)) + shutil.move(src_path, dest_path) + self.output.append("Moved %s to %s." % (src_path, dest_path)) + self.changed = True + + @property + def raw_image(self): + """ + Returns the image as it was originally passed in to the instance. + + .. note:: + This image string will only work directly with the atomic command. + + :returns: The original image passed in. + :rtype: str + """ + return self._image + + @property + def image(self): + """ + Returns the image without atomic prefixes used to map to skopeo args. + + :returns: The image string without prefixes + :rtype: str + """ + image = self._image + for remove in ('oci:', 'http:', 'https:'): + if image.startswith(remove): + image = image.replace(remove, '') + return image + + +def main(): + module = AnsibleModule( # noqa: F405 + argument_spec=dict( + image=dict(required=True), + tag=dict(required=True), + backend=dict(required=True), + ), + supports_check_mode=True + ) + + image = module.params['image'] + tag = module.params['tag'] + backend = module.params['backend'] + + if backend not in ["docker", "atomic"]: + module.fail_json(msg="unknown backend") + + binary_syncer = BinarySyncer(module, image, tag, backend) + + try: + binary_syncer.sync() + except BinarySyncError as ex: + module.fail_json(msg=ex.msg) + + return module.exit_json(changed=binary_syncer.changed, + output=binary_syncer.output) + + +if __name__ == '__main__': + main() diff --git a/roles/lib_utils/lookup_plugins/openshift_master_facts_default_predicates.py b/roles/lib_utils/lookup_plugins/openshift_master_facts_default_predicates.py new file mode 100644 index 000000000..4858c5ec6 --- /dev/null +++ b/roles/lib_utils/lookup_plugins/openshift_master_facts_default_predicates.py @@ -0,0 +1,143 @@ +# pylint: disable=missing-docstring + +import re +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase + + +class LookupModule(LookupBase): + # pylint: disable=too-many-branches,too-many-statements,too-many-arguments + + def run(self, terms, variables=None, regions_enabled=True, short_version=None, + deployment_type=None, **kwargs): + + predicates = [] + + if short_version is None or deployment_type is None: + if 'openshift' not in variables: + raise AnsibleError("This lookup module requires openshift_facts to be run prior to use") + + if deployment_type is None: + if 'common' not in variables['openshift'] or 'deployment_type' not in variables['openshift']['common']: + raise AnsibleError("This lookup module requires that the deployment_type be set") + + deployment_type = variables['openshift']['common']['deployment_type'] + + if short_version is None: + if 'short_version' in variables['openshift']['common']: + short_version = variables['openshift']['common']['short_version'] + elif 'openshift_release' in variables: + release = variables['openshift_release'] + if release.startswith('v'): + short_version = release[1:] + else: + short_version = release + short_version = '.'.join(short_version.split('.')[0:2]) + elif 'openshift_version' in variables: + version = variables['openshift_version'] + short_version = '.'.join(version.split('.')[0:2]) + else: + # pylint: disable=line-too-long + raise AnsibleError("Either OpenShift needs to be installed or openshift_release needs to be specified") + if deployment_type == 'origin': + if short_version not in ['1.1', '1.2', '1.3', '1.4', '1.5', '3.6', '3.7', '3.8', '3.9', 'latest']: + raise AnsibleError("Unknown short_version %s" % short_version) + elif deployment_type == 'openshift-enterprise': + if short_version not in ['3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', 'latest']: + raise AnsibleError("Unknown short_version %s" % short_version) + else: + raise AnsibleError("Unknown deployment_type %s" % deployment_type) + + if deployment_type == 'origin': + # convert short_version to enterprise short_version + short_version = re.sub('^1.', '3.', short_version) + + if short_version == 'latest': + short_version = '3.9' + + # Predicates ordered according to OpenShift Origin source: + # origin/vendor/k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go + + if short_version == '3.1': + predicates.extend([ + {'name': 'PodFitsHostPorts'}, + {'name': 'PodFitsResources'}, + {'name': 'NoDiskConflict'}, + {'name': 'MatchNodeSelector'}, + ]) + + if short_version == '3.2': + predicates.extend([ + {'name': 'PodFitsHostPorts'}, + {'name': 'PodFitsResources'}, + {'name': 'NoDiskConflict'}, + {'name': 'NoVolumeZoneConflict'}, + {'name': 'MatchNodeSelector'}, + {'name': 'MaxEBSVolumeCount'}, + {'name': 'MaxGCEPDVolumeCount'} + ]) + + if short_version == '3.3': + predicates.extend([ + {'name': 'NoDiskConflict'}, + {'name': 'NoVolumeZoneConflict'}, + {'name': 'MaxEBSVolumeCount'}, + {'name': 'MaxGCEPDVolumeCount'}, + {'name': 'GeneralPredicates'}, + {'name': 'PodToleratesNodeTaints'}, + {'name': 'CheckNodeMemoryPressure'} + ]) + + if short_version == '3.4': + predicates.extend([ + {'name': 'NoDiskConflict'}, + {'name': 'NoVolumeZoneConflict'}, + {'name': 'MaxEBSVolumeCount'}, + {'name': 'MaxGCEPDVolumeCount'}, + {'name': 'GeneralPredicates'}, + {'name': 'PodToleratesNodeTaints'}, + {'name': 'CheckNodeMemoryPressure'}, + {'name': 'CheckNodeDiskPressure'}, + {'name': 'MatchInterPodAffinity'} + ]) + + if short_version in ['3.5', '3.6']: + predicates.extend([ + {'name': 'NoVolumeZoneConflict'}, + {'name': 'MaxEBSVolumeCount'}, + {'name': 'MaxGCEPDVolumeCount'}, + {'name': 'MatchInterPodAffinity'}, + {'name': 'NoDiskConflict'}, + {'name': 'GeneralPredicates'}, + {'name': 'PodToleratesNodeTaints'}, + {'name': 'CheckNodeMemoryPressure'}, + {'name': 'CheckNodeDiskPressure'}, + ]) + + if short_version in ['3.7', '3.8', '3.9']: + predicates.extend([ + {'name': 'NoVolumeZoneConflict'}, + {'name': 'MaxEBSVolumeCount'}, + {'name': 'MaxGCEPDVolumeCount'}, + {'name': 'MaxAzureDiskVolumeCount'}, + {'name': 'MatchInterPodAffinity'}, + {'name': 'NoDiskConflict'}, + {'name': 'GeneralPredicates'}, + {'name': 'PodToleratesNodeTaints'}, + {'name': 'CheckNodeMemoryPressure'}, + {'name': 'CheckNodeDiskPressure'}, + {'name': 'NoVolumeNodeConflict'}, + ]) + + if regions_enabled: + region_predicate = { + 'name': 'Region', + 'argument': { + 'serviceAffinity': { + 'labels': ['region'] + } + } + } + predicates.append(region_predicate) + + return predicates diff --git a/roles/lib_utils/lookup_plugins/openshift_master_facts_default_priorities.py b/roles/lib_utils/lookup_plugins/openshift_master_facts_default_priorities.py new file mode 100644 index 000000000..18e1b2e0c --- /dev/null +++ b/roles/lib_utils/lookup_plugins/openshift_master_facts_default_priorities.py @@ -0,0 +1,117 @@ +# pylint: disable=missing-docstring + +import re +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase + + +class LookupModule(LookupBase): + # pylint: disable=too-many-branches,too-many-statements,too-many-arguments + + def run(self, terms, variables=None, zones_enabled=True, short_version=None, + deployment_type=None, **kwargs): + + priorities = [] + + if short_version is None or deployment_type is None: + if 'openshift' not in variables: + raise AnsibleError("This lookup module requires openshift_facts to be run prior to use") + + if deployment_type is None: + if 'common' not in variables['openshift'] or 'deployment_type' not in variables['openshift']['common']: + raise AnsibleError("This lookup module requires that the deployment_type be set") + + deployment_type = variables['openshift']['common']['deployment_type'] + + if short_version is None: + if 'short_version' in variables['openshift']['common']: + short_version = variables['openshift']['common']['short_version'] + elif 'openshift_release' in variables: + release = variables['openshift_release'] + if release.startswith('v'): + short_version = release[1:] + else: + short_version = release + short_version = '.'.join(short_version.split('.')[0:2]) + elif 'openshift_version' in variables: + version = variables['openshift_version'] + short_version = '.'.join(version.split('.')[0:2]) + else: + # pylint: disable=line-too-long + raise AnsibleError("Either OpenShift needs to be installed or openshift_release needs to be specified") + + if deployment_type == 'origin': + if short_version not in ['1.1', '1.2', '1.3', '1.4', '1.5', '3.6', '3.7', '3.8', '3.9', 'latest']: + raise AnsibleError("Unknown short_version %s" % short_version) + elif deployment_type == 'openshift-enterprise': + if short_version not in ['3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9', 'latest']: + raise AnsibleError("Unknown short_version %s" % short_version) + else: + raise AnsibleError("Unknown deployment_type %s" % deployment_type) + + if deployment_type == 'origin': + # convert short_version to origin short_version + short_version = re.sub('^1.', '3.', short_version) + + if short_version == 'latest': + short_version = '3.9' + + if short_version == '3.1': + priorities.extend([ + {'name': 'LeastRequestedPriority', 'weight': 1}, + {'name': 'BalancedResourceAllocation', 'weight': 1}, + {'name': 'SelectorSpreadPriority', 'weight': 1} + ]) + + if short_version == '3.2': + priorities.extend([ + {'name': 'LeastRequestedPriority', 'weight': 1}, + {'name': 'BalancedResourceAllocation', 'weight': 1}, + {'name': 'SelectorSpreadPriority', 'weight': 1}, + {'name': 'NodeAffinityPriority', 'weight': 1} + ]) + + if short_version == '3.3': + priorities.extend([ + {'name': 'LeastRequestedPriority', 'weight': 1}, + {'name': 'BalancedResourceAllocation', 'weight': 1}, + {'name': 'SelectorSpreadPriority', 'weight': 1}, + {'name': 'NodeAffinityPriority', 'weight': 1}, + {'name': 'TaintTolerationPriority', 'weight': 1} + ]) + + if short_version == '3.4': + priorities.extend([ + {'name': 'LeastRequestedPriority', 'weight': 1}, + {'name': 'BalancedResourceAllocation', 'weight': 1}, + {'name': 'SelectorSpreadPriority', 'weight': 1}, + {'name': 'NodePreferAvoidPodsPriority', 'weight': 10000}, + {'name': 'NodeAffinityPriority', 'weight': 1}, + {'name': 'TaintTolerationPriority', 'weight': 1}, + {'name': 'InterPodAffinityPriority', 'weight': 1} + ]) + + if short_version in ['3.5', '3.6', '3.7', '3.8', '3.9']: + priorities.extend([ + {'name': 'SelectorSpreadPriority', 'weight': 1}, + {'name': 'InterPodAffinityPriority', 'weight': 1}, + {'name': 'LeastRequestedPriority', 'weight': 1}, + {'name': 'BalancedResourceAllocation', 'weight': 1}, + {'name': 'NodePreferAvoidPodsPriority', 'weight': 10000}, + {'name': 'NodeAffinityPriority', 'weight': 1}, + {'name': 'TaintTolerationPriority', 'weight': 1} + ]) + + if zones_enabled: + zone_priority = { + 'name': 'Zone', + 'argument': { + 'serviceAntiAffinity': { + 'label': 'zone' + } + }, + 'weight': 2 + } + priorities.append(zone_priority) + + return priorities diff --git a/roles/lib_utils/test/conftest.py b/roles/lib_utils/test/conftest.py new file mode 100644 index 000000000..aabdd4fa1 --- /dev/null +++ b/roles/lib_utils/test/conftest.py @@ -0,0 +1,172 @@ +# pylint: disable=missing-docstring,invalid-name,redefined-outer-name +import os +import pytest +import sys + +from OpenSSL import crypto + +sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, "lookup_plugins")) + +from openshift_master_facts_default_predicates import LookupModule as PredicatesLookupModule # noqa: E402 +from openshift_master_facts_default_priorities import LookupModule as PrioritiesLookupModule # noqa: E402 + +# Parameter list for valid_cert fixture +VALID_CERTIFICATE_PARAMS = [ + { + 'short_name': 'client', + 'cn': 'client.example.com', + 'serial': 4, + 'uses': b'clientAuth', + 'dns': [], + 'ip': [], + }, + { + 'short_name': 'server', + 'cn': 'server.example.com', + 'serial': 5, + 'uses': b'serverAuth', + 'dns': ['kubernetes', 'openshift'], + 'ip': ['10.0.0.1', '192.168.0.1'] + }, + { + 'short_name': 'combined', + 'cn': 'combined.example.com', + # Verify that HUGE serials parse correctly. + # Frobs PARSING_HEX_SERIAL in _parse_cert + # See https://bugzilla.redhat.com/show_bug.cgi?id=1464240 + 'serial': 14449739080294792594019643629255165375, + 'uses': b'clientAuth, serverAuth', + 'dns': ['etcd'], + 'ip': ['10.0.0.2', '192.168.0.2'] + } +] + +# Extract the short_name from VALID_CERTIFICATE_PARAMS to provide +# friendly naming for the valid_cert fixture +VALID_CERTIFICATE_IDS = [param['short_name'] for param in VALID_CERTIFICATE_PARAMS] + + +@pytest.fixture(scope='session') +def ca(tmpdir_factory): + ca_dir = tmpdir_factory.mktemp('ca') + + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, 2048) + + cert = crypto.X509() + cert.set_version(3) + cert.set_serial_number(1) + cert.get_subject().commonName = 'test-signer' + cert.gmtime_adj_notBefore(0) + cert.gmtime_adj_notAfter(24 * 60 * 60) + cert.set_issuer(cert.get_subject()) + cert.set_pubkey(key) + cert.add_extensions([ + crypto.X509Extension(b'basicConstraints', True, b'CA:TRUE, pathlen:0'), + crypto.X509Extension(b'keyUsage', True, + b'digitalSignature, keyEncipherment, keyCertSign, cRLSign'), + crypto.X509Extension(b'subjectKeyIdentifier', False, b'hash', subject=cert) + ]) + cert.add_extensions([ + crypto.X509Extension(b'authorityKeyIdentifier', False, b'keyid:always', issuer=cert) + ]) + cert.sign(key, 'sha256') + + return { + 'dir': ca_dir, + 'key': key, + 'cert': cert, + } + + +@pytest.fixture(scope='session', + ids=VALID_CERTIFICATE_IDS, + params=VALID_CERTIFICATE_PARAMS) +def valid_cert(request, ca): + common_name = request.param['cn'] + + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, 2048) + + cert = crypto.X509() + cert.set_serial_number(request.param['serial']) + cert.gmtime_adj_notBefore(0) + cert.gmtime_adj_notAfter(24 * 60 * 60) + cert.set_issuer(ca['cert'].get_subject()) + cert.set_pubkey(key) + cert.set_version(3) + cert.get_subject().commonName = common_name + cert.add_extensions([ + crypto.X509Extension(b'basicConstraints', True, b'CA:FALSE'), + crypto.X509Extension(b'keyUsage', True, b'digitalSignature, keyEncipherment'), + crypto.X509Extension(b'extendedKeyUsage', False, request.param['uses']), + ]) + + if request.param['dns'] or request.param['ip']: + san_list = ['DNS:{}'.format(common_name)] + san_list.extend(['DNS:{}'.format(x) for x in request.param['dns']]) + san_list.extend(['IP:{}'.format(x) for x in request.param['ip']]) + + cert.add_extensions([ + crypto.X509Extension(b'subjectAltName', False, ', '.join(san_list).encode('utf8')) + ]) + cert.sign(ca['key'], 'sha256') + + cert_contents = crypto.dump_certificate(crypto.FILETYPE_PEM, cert) + cert_file = ca['dir'].join('{}.crt'.format(common_name)) + cert_file.write_binary(cert_contents) + + return { + 'common_name': common_name, + 'serial': request.param['serial'], + 'dns': request.param['dns'], + 'ip': request.param['ip'], + 'uses': request.param['uses'], + 'cert_file': cert_file, + 'cert': cert + } + + +@pytest.fixture() +def predicates_lookup(): + return PredicatesLookupModule() + + +@pytest.fixture() +def priorities_lookup(): + return PrioritiesLookupModule() + + +@pytest.fixture() +def facts(): + return { + 'openshift': { + 'common': {} + } + } + + +@pytest.fixture(params=[True, False]) +def regions_enabled(request): + return request.param + + +@pytest.fixture(params=[True, False]) +def zones_enabled(request): + return request.param + + +def v_prefix(release): + """Prefix a release number with 'v'.""" + return "v" + release + + +def minor(release): + """Add a suffix to release, making 'X.Y' become 'X.Y.Z'.""" + return release + ".1" + + +@pytest.fixture(params=[str, v_prefix, minor]) +def release_mod(request): + """Modifies a release string to alternative valid values.""" + return request.param diff --git a/roles/lib_utils/test/openshift_master_facts_bad_input_tests.py b/roles/lib_utils/test/openshift_master_facts_bad_input_tests.py new file mode 100644 index 000000000..e8da1e04a --- /dev/null +++ b/roles/lib_utils/test/openshift_master_facts_bad_input_tests.py @@ -0,0 +1,57 @@ +import copy +import os +import sys + +from ansible.errors import AnsibleError +import pytest + +sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, "lookup_plugins")) + +from openshift_master_facts_default_predicates import LookupModule # noqa: E402 + + +class TestOpenShiftMasterFactsBadInput(object): + lookup = LookupModule() + default_facts = { + 'openshift': { + 'common': {} + } + } + + def test_missing_openshift_facts(self): + with pytest.raises(AnsibleError): + facts = {} + self.lookup.run(None, variables=facts) + + def test_missing_deployment_type(self): + with pytest.raises(AnsibleError): + facts = copy.deepcopy(self.default_facts) + facts['openshift']['common']['short_version'] = '10.10' + self.lookup.run(None, variables=facts) + + def test_missing_short_version_and_missing_openshift_release(self): + with pytest.raises(AnsibleError): + facts = copy.deepcopy(self.default_facts) + facts['openshift']['common']['deployment_type'] = 'origin' + self.lookup.run(None, variables=facts) + + def test_unknown_deployment_types(self): + with pytest.raises(AnsibleError): + facts = copy.deepcopy(self.default_facts) + facts['openshift']['common']['short_version'] = '1.1' + facts['openshift']['common']['deployment_type'] = 'bogus' + self.lookup.run(None, variables=facts) + + def test_unknown_origin_version(self): + with pytest.raises(AnsibleError): + facts = copy.deepcopy(self.default_facts) + facts['openshift']['common']['short_version'] = '0.1' + facts['openshift']['common']['deployment_type'] = 'origin' + self.lookup.run(None, variables=facts) + + def test_unknown_ocp_version(self): + with pytest.raises(AnsibleError): + facts = copy.deepcopy(self.default_facts) + facts['openshift']['common']['short_version'] = '0.1' + facts['openshift']['common']['deployment_type'] = 'openshift-enterprise' + self.lookup.run(None, variables=facts) diff --git a/roles/lib_utils/test/openshift_master_facts_conftest.py b/roles/lib_utils/test/openshift_master_facts_conftest.py new file mode 100644 index 000000000..140cced73 --- /dev/null +++ b/roles/lib_utils/test/openshift_master_facts_conftest.py @@ -0,0 +1,54 @@ +import os +import sys + +import pytest + +sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, "lookup_plugins")) + +from openshift_master_facts_default_predicates import LookupModule as PredicatesLookupModule # noqa: E402 +from openshift_master_facts_default_priorities import LookupModule as PrioritiesLookupModule # noqa: E402 + + +@pytest.fixture() +def predicates_lookup(): + return PredicatesLookupModule() + + +@pytest.fixture() +def priorities_lookup(): + return PrioritiesLookupModule() + + +@pytest.fixture() +def facts(): + return { + 'openshift': { + 'common': {} + } + } + + +@pytest.fixture(params=[True, False]) +def regions_enabled(request): + return request.param + + +@pytest.fixture(params=[True, False]) +def zones_enabled(request): + return request.param + + +def v_prefix(release): + """Prefix a release number with 'v'.""" + return "v" + release + + +def minor(release): + """Add a suffix to release, making 'X.Y' become 'X.Y.Z'.""" + return release + ".1" + + +@pytest.fixture(params=[str, v_prefix, minor]) +def release_mod(request): + """Modifies a release string to alternative valid values.""" + return request.param diff --git a/roles/lib_utils/test/openshift_master_facts_default_predicates_tests.py b/roles/lib_utils/test/openshift_master_facts_default_predicates_tests.py new file mode 100644 index 000000000..11aad9f03 --- /dev/null +++ b/roles/lib_utils/test/openshift_master_facts_default_predicates_tests.py @@ -0,0 +1,193 @@ +import pytest + + +# Predicates ordered according to OpenShift Origin source: +# origin/vendor/k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go + +DEFAULT_PREDICATES_1_1 = [ + {'name': 'PodFitsHostPorts'}, + {'name': 'PodFitsResources'}, + {'name': 'NoDiskConflict'}, + {'name': 'MatchNodeSelector'}, +] + +DEFAULT_PREDICATES_1_2 = [ + {'name': 'PodFitsHostPorts'}, + {'name': 'PodFitsResources'}, + {'name': 'NoDiskConflict'}, + {'name': 'NoVolumeZoneConflict'}, + {'name': 'MatchNodeSelector'}, + {'name': 'MaxEBSVolumeCount'}, + {'name': 'MaxGCEPDVolumeCount'} +] + +DEFAULT_PREDICATES_1_3 = [ + {'name': 'NoDiskConflict'}, + {'name': 'NoVolumeZoneConflict'}, + {'name': 'MaxEBSVolumeCount'}, + {'name': 'MaxGCEPDVolumeCount'}, + {'name': 'GeneralPredicates'}, + {'name': 'PodToleratesNodeTaints'}, + {'name': 'CheckNodeMemoryPressure'} +] + +DEFAULT_PREDICATES_1_4 = [ + {'name': 'NoDiskConflict'}, + {'name': 'NoVolumeZoneConflict'}, + {'name': 'MaxEBSVolumeCount'}, + {'name': 'MaxGCEPDVolumeCount'}, + {'name': 'GeneralPredicates'}, + {'name': 'PodToleratesNodeTaints'}, + {'name': 'CheckNodeMemoryPressure'}, + {'name': 'CheckNodeDiskPressure'}, + {'name': 'MatchInterPodAffinity'} +] + +DEFAULT_PREDICATES_1_5 = [ + {'name': 'NoVolumeZoneConflict'}, + {'name': 'MaxEBSVolumeCount'}, + {'name': 'MaxGCEPDVolumeCount'}, + {'name': 'MatchInterPodAffinity'}, + {'name': 'NoDiskConflict'}, + {'name': 'GeneralPredicates'}, + {'name': 'PodToleratesNodeTaints'}, + {'name': 'CheckNodeMemoryPressure'}, + {'name': 'CheckNodeDiskPressure'}, +] + +DEFAULT_PREDICATES_3_6 = DEFAULT_PREDICATES_1_5 + +DEFAULT_PREDICATES_3_7 = [ + {'name': 'NoVolumeZoneConflict'}, + {'name': 'MaxEBSVolumeCount'}, + {'name': 'MaxGCEPDVolumeCount'}, + {'name': 'MaxAzureDiskVolumeCount'}, + {'name': 'MatchInterPodAffinity'}, + {'name': 'NoDiskConflict'}, + {'name': 'GeneralPredicates'}, + {'name': 'PodToleratesNodeTaints'}, + {'name': 'CheckNodeMemoryPressure'}, + {'name': 'CheckNodeDiskPressure'}, + {'name': 'NoVolumeNodeConflict'}, +] + +DEFAULT_PREDICATES_3_9 = DEFAULT_PREDICATES_3_8 = DEFAULT_PREDICATES_3_7 + +REGION_PREDICATE = { + 'name': 'Region', + 'argument': { + 'serviceAffinity': { + 'labels': ['region'] + } + } +} + +TEST_VARS = [ + ('1.1', 'origin', DEFAULT_PREDICATES_1_1), + ('3.1', 'openshift-enterprise', DEFAULT_PREDICATES_1_1), + ('1.2', 'origin', DEFAULT_PREDICATES_1_2), + ('3.2', 'openshift-enterprise', DEFAULT_PREDICATES_1_2), + ('1.3', 'origin', DEFAULT_PREDICATES_1_3), + ('3.3', 'openshift-enterprise', DEFAULT_PREDICATES_1_3), + ('1.4', 'origin', DEFAULT_PREDICATES_1_4), + ('3.4', 'openshift-enterprise', DEFAULT_PREDICATES_1_4), + ('1.5', 'origin', DEFAULT_PREDICATES_1_5), + ('3.5', 'openshift-enterprise', DEFAULT_PREDICATES_1_5), + ('3.6', 'origin', DEFAULT_PREDICATES_3_6), + ('3.6', 'openshift-enterprise', DEFAULT_PREDICATES_3_6), + ('3.7', 'origin', DEFAULT_PREDICATES_3_7), + ('3.7', 'openshift-enterprise', DEFAULT_PREDICATES_3_7), + ('3.8', 'origin', DEFAULT_PREDICATES_3_8), + ('3.8', 'openshift-enterprise', DEFAULT_PREDICATES_3_8), + ('3.9', 'origin', DEFAULT_PREDICATES_3_9), + ('3.9', 'openshift-enterprise', DEFAULT_PREDICATES_3_9), +] + + +def assert_ok(predicates_lookup, default_predicates, regions_enabled, **kwargs): + results = predicates_lookup.run(None, regions_enabled=regions_enabled, **kwargs) + if regions_enabled: + assert results == default_predicates + [REGION_PREDICATE] + else: + assert results == default_predicates + + +def test_openshift_version(predicates_lookup, openshift_version_fixture, regions_enabled): + facts, default_predicates = openshift_version_fixture + assert_ok(predicates_lookup, default_predicates, variables=facts, regions_enabled=regions_enabled) + + +@pytest.fixture(params=TEST_VARS) +def openshift_version_fixture(request, facts): + version, deployment_type, default_predicates = request.param + version += '.1' + facts['openshift_version'] = version + facts['openshift']['common']['deployment_type'] = deployment_type + return facts, default_predicates + + +def test_openshift_release(predicates_lookup, openshift_release_fixture, regions_enabled): + facts, default_predicates = openshift_release_fixture + assert_ok(predicates_lookup, default_predicates, variables=facts, regions_enabled=regions_enabled) + + +@pytest.fixture(params=TEST_VARS) +def openshift_release_fixture(request, facts, release_mod): + release, deployment_type, default_predicates = request.param + facts['openshift_release'] = release_mod(release) + facts['openshift']['common']['deployment_type'] = deployment_type + return facts, default_predicates + + +def test_short_version(predicates_lookup, short_version_fixture, regions_enabled): + facts, default_predicates = short_version_fixture + assert_ok(predicates_lookup, default_predicates, variables=facts, regions_enabled=regions_enabled) + + +@pytest.fixture(params=TEST_VARS) +def short_version_fixture(request, facts): + short_version, deployment_type, default_predicates = request.param + facts['openshift']['common']['short_version'] = short_version + facts['openshift']['common']['deployment_type'] = deployment_type + return facts, default_predicates + + +def test_short_version_kwarg(predicates_lookup, short_version_kwarg_fixture, regions_enabled): + facts, short_version, default_predicates = short_version_kwarg_fixture + assert_ok( + predicates_lookup, default_predicates, variables=facts, + regions_enabled=regions_enabled, short_version=short_version) + + +@pytest.fixture(params=TEST_VARS) +def short_version_kwarg_fixture(request, facts): + short_version, deployment_type, default_predicates = request.param + facts['openshift']['common']['deployment_type'] = deployment_type + return facts, short_version, default_predicates + + +def test_deployment_type_kwarg(predicates_lookup, deployment_type_kwarg_fixture, regions_enabled): + facts, deployment_type, default_predicates = deployment_type_kwarg_fixture + assert_ok( + predicates_lookup, default_predicates, variables=facts, + regions_enabled=regions_enabled, deployment_type=deployment_type) + + +@pytest.fixture(params=TEST_VARS) +def deployment_type_kwarg_fixture(request, facts): + short_version, deployment_type, default_predicates = request.param + facts['openshift']['common']['short_version'] = short_version + return facts, deployment_type, default_predicates + + +def test_short_version_deployment_type_kwargs( + predicates_lookup, short_version_deployment_type_kwargs_fixture, regions_enabled): + short_version, deployment_type, default_predicates = short_version_deployment_type_kwargs_fixture + assert_ok( + predicates_lookup, default_predicates, regions_enabled=regions_enabled, + short_version=short_version, deployment_type=deployment_type) + + +@pytest.fixture(params=TEST_VARS) +def short_version_deployment_type_kwargs_fixture(request): + return request.param diff --git a/roles/lib_utils/test/openshift_master_facts_default_priorities_tests.py b/roles/lib_utils/test/openshift_master_facts_default_priorities_tests.py new file mode 100644 index 000000000..527fc9ff4 --- /dev/null +++ b/roles/lib_utils/test/openshift_master_facts_default_priorities_tests.py @@ -0,0 +1,167 @@ +import pytest + + +DEFAULT_PRIORITIES_1_1 = [ + {'name': 'LeastRequestedPriority', 'weight': 1}, + {'name': 'BalancedResourceAllocation', 'weight': 1}, + {'name': 'SelectorSpreadPriority', 'weight': 1} +] + +DEFAULT_PRIORITIES_1_2 = [ + {'name': 'LeastRequestedPriority', 'weight': 1}, + {'name': 'BalancedResourceAllocation', 'weight': 1}, + {'name': 'SelectorSpreadPriority', 'weight': 1}, + {'name': 'NodeAffinityPriority', 'weight': 1} +] + +DEFAULT_PRIORITIES_1_3 = [ + {'name': 'LeastRequestedPriority', 'weight': 1}, + {'name': 'BalancedResourceAllocation', 'weight': 1}, + {'name': 'SelectorSpreadPriority', 'weight': 1}, + {'name': 'NodeAffinityPriority', 'weight': 1}, + {'name': 'TaintTolerationPriority', 'weight': 1} +] + +DEFAULT_PRIORITIES_1_4 = [ + {'name': 'LeastRequestedPriority', 'weight': 1}, + {'name': 'BalancedResourceAllocation', 'weight': 1}, + {'name': 'SelectorSpreadPriority', 'weight': 1}, + {'name': 'NodePreferAvoidPodsPriority', 'weight': 10000}, + {'name': 'NodeAffinityPriority', 'weight': 1}, + {'name': 'TaintTolerationPriority', 'weight': 1}, + {'name': 'InterPodAffinityPriority', 'weight': 1} +] + +DEFAULT_PRIORITIES_1_5 = [ + {'name': 'SelectorSpreadPriority', 'weight': 1}, + {'name': 'InterPodAffinityPriority', 'weight': 1}, + {'name': 'LeastRequestedPriority', 'weight': 1}, + {'name': 'BalancedResourceAllocation', 'weight': 1}, + {'name': 'NodePreferAvoidPodsPriority', 'weight': 10000}, + {'name': 'NodeAffinityPriority', 'weight': 1}, + {'name': 'TaintTolerationPriority', 'weight': 1} +] + +DEFAULT_PRIORITIES_3_6 = DEFAULT_PRIORITIES_1_5 + +DEFAULT_PRIORITIES_3_9 = DEFAULT_PRIORITIES_3_8 = DEFAULT_PRIORITIES_3_7 = DEFAULT_PRIORITIES_3_6 + +ZONE_PRIORITY = { + 'name': 'Zone', + 'argument': { + 'serviceAntiAffinity': { + 'label': 'zone' + } + }, + 'weight': 2 +} + +TEST_VARS = [ + ('1.1', 'origin', DEFAULT_PRIORITIES_1_1), + ('3.1', 'openshift-enterprise', DEFAULT_PRIORITIES_1_1), + ('1.2', 'origin', DEFAULT_PRIORITIES_1_2), + ('3.2', 'openshift-enterprise', DEFAULT_PRIORITIES_1_2), + ('1.3', 'origin', DEFAULT_PRIORITIES_1_3), + ('3.3', 'openshift-enterprise', DEFAULT_PRIORITIES_1_3), + ('1.4', 'origin', DEFAULT_PRIORITIES_1_4), + ('3.4', 'openshift-enterprise', DEFAULT_PRIORITIES_1_4), + ('1.5', 'origin', DEFAULT_PRIORITIES_1_5), + ('3.5', 'openshift-enterprise', DEFAULT_PRIORITIES_1_5), + ('3.6', 'origin', DEFAULT_PRIORITIES_3_6), + ('3.6', 'openshift-enterprise', DEFAULT_PRIORITIES_3_6), + ('3.7', 'origin', DEFAULT_PRIORITIES_3_7), + ('3.7', 'openshift-enterprise', DEFAULT_PRIORITIES_3_7), + ('3.8', 'origin', DEFAULT_PRIORITIES_3_8), + ('3.8', 'openshift-enterprise', DEFAULT_PRIORITIES_3_8), + ('3.9', 'origin', DEFAULT_PRIORITIES_3_9), + ('3.9', 'openshift-enterprise', DEFAULT_PRIORITIES_3_9), +] + + +def assert_ok(priorities_lookup, default_priorities, zones_enabled, **kwargs): + results = priorities_lookup.run(None, zones_enabled=zones_enabled, **kwargs) + if zones_enabled: + assert results == default_priorities + [ZONE_PRIORITY] + else: + assert results == default_priorities + + +def test_openshift_version(priorities_lookup, openshift_version_fixture, zones_enabled): + facts, default_priorities = openshift_version_fixture + assert_ok(priorities_lookup, default_priorities, variables=facts, zones_enabled=zones_enabled) + + +@pytest.fixture(params=TEST_VARS) +def openshift_version_fixture(request, facts): + version, deployment_type, default_priorities = request.param + version += '.1' + facts['openshift_version'] = version + facts['openshift']['common']['deployment_type'] = deployment_type + return facts, default_priorities + + +def test_openshift_release(priorities_lookup, openshift_release_fixture, zones_enabled): + facts, default_priorities = openshift_release_fixture + assert_ok(priorities_lookup, default_priorities, variables=facts, zones_enabled=zones_enabled) + + +@pytest.fixture(params=TEST_VARS) +def openshift_release_fixture(request, facts, release_mod): + release, deployment_type, default_priorities = request.param + facts['openshift_release'] = release_mod(release) + facts['openshift']['common']['deployment_type'] = deployment_type + return facts, default_priorities + + +def test_short_version(priorities_lookup, short_version_fixture, zones_enabled): + facts, default_priorities = short_version_fixture + assert_ok(priorities_lookup, default_priorities, variables=facts, zones_enabled=zones_enabled) + + +@pytest.fixture(params=TEST_VARS) +def short_version_fixture(request, facts): + short_version, deployment_type, default_priorities = request.param + facts['openshift']['common']['short_version'] = short_version + facts['openshift']['common']['deployment_type'] = deployment_type + return facts, default_priorities + + +def test_short_version_kwarg(priorities_lookup, short_version_kwarg_fixture, zones_enabled): + facts, short_version, default_priorities = short_version_kwarg_fixture + assert_ok( + priorities_lookup, default_priorities, variables=facts, + zones_enabled=zones_enabled, short_version=short_version) + + +@pytest.fixture(params=TEST_VARS) +def short_version_kwarg_fixture(request, facts): + short_version, deployment_type, default_priorities = request.param + facts['openshift']['common']['deployment_type'] = deployment_type + return facts, short_version, default_priorities + + +def test_deployment_type_kwarg(priorities_lookup, deployment_type_kwarg_fixture, zones_enabled): + facts, deployment_type, default_priorities = deployment_type_kwarg_fixture + assert_ok( + priorities_lookup, default_priorities, variables=facts, + zones_enabled=zones_enabled, deployment_type=deployment_type) + + +@pytest.fixture(params=TEST_VARS) +def deployment_type_kwarg_fixture(request, facts): + short_version, deployment_type, default_priorities = request.param + facts['openshift']['common']['short_version'] = short_version + return facts, deployment_type, default_priorities + + +def test_short_version_deployment_type_kwargs( + priorities_lookup, short_version_deployment_type_kwargs_fixture, zones_enabled): + short_version, deployment_type, default_priorities = short_version_deployment_type_kwargs_fixture + assert_ok( + priorities_lookup, default_priorities, zones_enabled=zones_enabled, + short_version=short_version, deployment_type=deployment_type) + + +@pytest.fixture(params=TEST_VARS) +def short_version_deployment_type_kwargs_fixture(request): + return request.param diff --git a/roles/lib_utils/test/test_fakeopensslclasses.py b/roles/lib_utils/test/test_fakeopensslclasses.py new file mode 100644 index 000000000..8a521a765 --- /dev/null +++ b/roles/lib_utils/test/test_fakeopensslclasses.py @@ -0,0 +1,90 @@ +''' + Unit tests for the FakeOpenSSL classes +''' +import os +import subprocess +import sys + +import pytest + +MODULE_PATH = os.path.realpath(os.path.join(__file__, os.pardir, os.pardir, 'library')) +sys.path.insert(1, MODULE_PATH) + +# pylint: disable=import-error,wrong-import-position,missing-docstring +# pylint: disable=invalid-name,redefined-outer-name +from openshift_cert_expiry import FakeOpenSSLCertificate # noqa: E402 + + +@pytest.fixture(scope='module') +def fake_valid_cert(valid_cert): + cmd = ['openssl', 'x509', '-in', str(valid_cert['cert_file']), '-text', + '-nameopt', 'oneline'] + cert = subprocess.check_output(cmd) + return FakeOpenSSLCertificate(cert.decode('utf8')) + + +def test_not_after(valid_cert, fake_valid_cert): + ''' Validate value returned back from get_notAfter() ''' + real_cert = valid_cert['cert'] + + # Internal representation of pyOpenSSL is bytes, while FakeOpenSSLCertificate + # is text, so decode the result from pyOpenSSL prior to comparing + assert real_cert.get_notAfter().decode('utf8') == fake_valid_cert.get_notAfter() + + +def test_serial(valid_cert, fake_valid_cert): + ''' Validate value returned back form get_serialnumber() ''' + real_cert = valid_cert['cert'] + assert real_cert.get_serial_number() == fake_valid_cert.get_serial_number() + + +def test_get_subject(valid_cert, fake_valid_cert): + ''' Validate the certificate subject ''' + + # Gather the subject components and create a list of colon separated strings. + # Since the internal representation of pyOpenSSL uses bytes, we need to decode + # the results before comparing. + c_subjects = valid_cert['cert'].get_subject().get_components() + c_subj = ', '.join(['{}:{}'.format(x.decode('utf8'), y.decode('utf8')) for x, y in c_subjects]) + f_subjects = fake_valid_cert.get_subject().get_components() + f_subj = ', '.join(['{}:{}'.format(x, y) for x, y in f_subjects]) + assert c_subj == f_subj + + +def get_san_extension(cert): + # Internal representation of pyOpenSSL is bytes, while FakeOpenSSLCertificate + # is text, so we need to set the value to search for accordingly. + if isinstance(cert, FakeOpenSSLCertificate): + san_short_name = 'subjectAltName' + else: + san_short_name = b'subjectAltName' + + for i in range(cert.get_extension_count()): + ext = cert.get_extension(i) + if ext.get_short_name() == san_short_name: + # return the string representation to compare the actual SAN + # values instead of the data types + return str(ext) + + return None + + +def test_subject_alt_names(valid_cert, fake_valid_cert): + real_cert = valid_cert['cert'] + + san = get_san_extension(real_cert) + f_san = get_san_extension(fake_valid_cert) + + assert san == f_san + + # If there are either dns or ip sans defined, verify common_name present + if valid_cert['ip'] or valid_cert['dns']: + assert 'DNS:' + valid_cert['common_name'] in f_san + + # Verify all ip sans are present + for ip in valid_cert['ip']: + assert 'IP Address:' + ip in f_san + + # Verify all dns sans are present + for name in valid_cert['dns']: + assert 'DNS:' + name in f_san diff --git a/roles/lib_utils/test/test_load_and_handle_cert.py b/roles/lib_utils/test/test_load_and_handle_cert.py new file mode 100644 index 000000000..98792e2ee --- /dev/null +++ b/roles/lib_utils/test/test_load_and_handle_cert.py @@ -0,0 +1,67 @@ +''' + Unit tests for the load_and_handle_cert method +''' +import datetime +import os +import sys + +import pytest + +MODULE_PATH = os.path.realpath(os.path.join(__file__, os.pardir, os.pardir, 'library')) +sys.path.insert(1, MODULE_PATH) + +# pylint: disable=import-error,wrong-import-position,missing-docstring +# pylint: disable=invalid-name,redefined-outer-name +import openshift_cert_expiry # noqa: E402 + +# TODO: More testing on the results of the load_and_handle_cert function +# could be implemented here as well, such as verifying subjects +# match up. + + +@pytest.fixture(params=['OpenSSLCertificate', 'FakeOpenSSLCertificate']) +def loaded_cert(request, valid_cert): + """ parameterized fixture to provide load_and_handle_cert results + for both OpenSSL and FakeOpenSSL parsed certificates + """ + now = datetime.datetime.now() + + openshift_cert_expiry.HAS_OPENSSL = request.param == 'OpenSSLCertificate' + + # valid_cert['cert_file'] is a `py.path.LocalPath` object and + # provides a read_text() method for reading the file contents. + cert_string = valid_cert['cert_file'].read_text('utf8') + + (subject, + expiry_date, + time_remaining, + serial) = openshift_cert_expiry.load_and_handle_cert(cert_string, now) + + return { + 'now': now, + 'subject': subject, + 'expiry_date': expiry_date, + 'time_remaining': time_remaining, + 'serial': serial, + } + + +def test_serial(loaded_cert, valid_cert): + """Params: + + * `loaded_cert` comes from the `loaded_cert` fixture in this file + * `valid_cert` comes from the 'valid_cert' fixture in conftest.py + """ + valid_cert_serial = valid_cert['cert'].get_serial_number() + assert loaded_cert['serial'] == valid_cert_serial + + +def test_expiry(loaded_cert): + """Params: + + * `loaded_cert` comes from the `loaded_cert` fixture in this file + """ + expiry_date = loaded_cert['expiry_date'] + time_remaining = loaded_cert['time_remaining'] + now = loaded_cert['now'] + assert expiry_date == now + time_remaining diff --git a/roles/openshift_aws/defaults/main.yml b/roles/openshift_aws/defaults/main.yml index 71de24339..8c8227b5e 100644 --- a/roles/openshift_aws/defaults/main.yml +++ b/roles/openshift_aws/defaults/main.yml @@ -109,6 +109,7 @@ openshift_aws_node_group_config_node_volumes: device_type: gp2 delete_on_termination: True +# build_instance_tags is a custom filter in role lib_utils openshift_aws_node_group_config_tags: "{{ openshift_aws_clusterid | build_instance_tags }}" openshift_aws_node_group_termination_policy: Default openshift_aws_node_group_replace_instances: [] @@ -201,6 +202,7 @@ openshift_aws_node_group_config: openshift_aws_elb_tags: "{{ openshift_aws_kube_tags }}" openshift_aws_elb_az_load_balancing: False +# build_instance_tags is a custom filter in role lib_utils openshift_aws_kube_tags: "{{ openshift_aws_clusterid | build_instance_tags }}" openshift_aws_elb_security_groups: "{{ openshift_aws_launch_config_security_groups }}" diff --git a/roles/openshift_aws/filter_plugins/openshift_aws_filters.py b/roles/openshift_aws/filter_plugins/openshift_aws_filters.py deleted file mode 100644 index dfcb11da3..000000000 --- a/roles/openshift_aws/filter_plugins/openshift_aws_filters.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -''' -Custom filters for use in openshift_aws -''' - -from ansible import errors - - -class FilterModule(object): - ''' Custom ansible filters for use by openshift_aws role''' - - @staticmethod - def scale_groups_serial(scale_group_info, upgrade=False): - ''' This function will determine what the deployment serial should be and return it - - Search through the tags and find the deployment_serial tag. Once found, - determine if an increment is needed during an upgrade. - if upgrade is true then increment the serial and return it - else return the serial - ''' - if scale_group_info == []: - return 1 - - scale_group_info = scale_group_info[0] - - if not isinstance(scale_group_info, dict): - raise errors.AnsibleFilterError("|filter plugin failed: Expected scale_group_info to be a dict") - - serial = None - - for tag in scale_group_info['tags']: - if tag['key'] == 'deployment_serial': - serial = int(tag['value']) - if upgrade: - serial += 1 - break - else: - raise errors.AnsibleFilterError("|filter plugin failed: deployment_serial tag was not found") - - return serial - - @staticmethod - def scale_groups_match_capacity(scale_group_info): - ''' This function will verify that the scale group instance count matches - the scale group desired capacity - - ''' - for scale_group in scale_group_info: - if scale_group['desired_capacity'] != len(scale_group['instances']): - return False - - return True - - @staticmethod - def build_instance_tags(clusterid): - ''' This function will return a dictionary of the instance tags. - - The main desire to have this inside of a filter_plugin is that we - need to build the following key. - - {"kubernetes.io/cluster/{{ openshift_aws_clusterid }}": "{{ openshift_aws_clusterid}}"} - - ''' - tags = {'clusterid': clusterid, - 'kubernetes.io/cluster/{}'.format(clusterid): clusterid} - - return tags - - def filters(self): - ''' returns a mapping of filters to methods ''' - return {'build_instance_tags': self.build_instance_tags, - 'scale_groups_match_capacity': self.scale_groups_match_capacity, - 'scale_groups_serial': self.scale_groups_serial} diff --git a/roles/openshift_aws/tasks/build_node_group.yml b/roles/openshift_aws/tasks/build_node_group.yml index 9485cc3ac..a9f9cc3c4 100644 --- a/roles/openshift_aws/tasks/build_node_group.yml +++ b/roles/openshift_aws/tasks/build_node_group.yml @@ -43,6 +43,7 @@ - name: set the value for the deployment_serial and the current asgs set_fact: + # scale_groups_serial is a custom filter in role lib_utils l_deployment_serial: "{{ openshift_aws_node_group_deployment_serial if openshift_aws_node_group_deployment_serial is defined else asgs.results | scale_groups_serial(openshift_aws_node_group_upgrade) }}" openshift_aws_current_asgs: "{{ asgs.results | map(attribute='auto_scaling_group_name') | list | union(openshift_aws_current_asgs) }}" diff --git a/roles/openshift_aws/tasks/wait_for_groups.yml b/roles/openshift_aws/tasks/wait_for_groups.yml index 1f4ef3e1c..3ad876e37 100644 --- a/roles/openshift_aws/tasks/wait_for_groups.yml +++ b/roles/openshift_aws/tasks/wait_for_groups.yml @@ -8,6 +8,7 @@ tags: "{{ {'kubernetes.io/cluster/' ~ openshift_aws_clusterid: openshift_aws_clusterid } }}" register: qasg + # scale_groups_match_capacity is a custom filter in role lib_utils until: qasg | json_query('results[*]') | scale_groups_match_capacity | bool delay: 10 retries: 60 diff --git a/roles/openshift_certificate_expiry/filter_plugins/oo_cert_expiry.py b/roles/openshift_certificate_expiry/filter_plugins/oo_cert_expiry.py deleted file mode 100644 index 58b228fee..000000000 --- a/roles/openshift_certificate_expiry/filter_plugins/oo_cert_expiry.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -""" -Custom filters for use in openshift-ansible -""" - - -# Disabling too-many-public-methods, since filter methods are necessarily -# public -# pylint: disable=too-many-public-methods -class FilterModule(object): - """ Custom ansible filters """ - - @staticmethod - def oo_cert_expiry_results_to_json(hostvars, play_hosts): - """Takes results (`hostvars`) from the openshift_cert_expiry role -check and serializes them into proper machine-readable JSON -output. This filter parameter **MUST** be the playbook `hostvars` -variable. The `play_hosts` parameter is so we know what to loop over -when we're extrating the values. - -Returns: - -Results are collected into two top-level keys under the `json_results` -dict: - -* `json_results.data` [dict] - Each individual host check result, keys are hostnames -* `json_results.summary` [dict] - Summary of number of `warning` and `expired` -certificates - -Example playbook usage: - - - name: Generate expiration results JSON - run_once: yes - delegate_to: localhost - when: openshift_certificate_expiry_save_json_results|bool - copy: - content: "{{ hostvars|oo_cert_expiry_results_to_json() }}" - dest: "{{ openshift_certificate_expiry_json_results_path }}" - - """ - json_result = { - 'data': {}, - 'summary': {}, - } - - for host in play_hosts: - json_result['data'][host] = hostvars[host]['check_results']['check_results'] - - total_warnings = sum([hostvars[h]['check_results']['summary']['warning'] for h in play_hosts]) - total_expired = sum([hostvars[h]['check_results']['summary']['expired'] for h in play_hosts]) - total_ok = sum([hostvars[h]['check_results']['summary']['ok'] for h in play_hosts]) - total_total = sum([hostvars[h]['check_results']['summary']['total'] for h in play_hosts]) - - json_result['summary']['warning'] = total_warnings - json_result['summary']['expired'] = total_expired - json_result['summary']['ok'] = total_ok - json_result['summary']['total'] = total_total - - return json_result - - def filters(self): - """ returns a mapping of filters to methods """ - return { - "oo_cert_expiry_results_to_json": self.oo_cert_expiry_results_to_json, - } diff --git a/roles/openshift_certificate_expiry/library/openshift_cert_expiry.py b/roles/openshift_certificate_expiry/library/openshift_cert_expiry.py deleted file mode 100644 index e355266b0..000000000 --- a/roles/openshift_certificate_expiry/library/openshift_cert_expiry.py +++ /dev/null @@ -1,839 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# pylint: disable=line-too-long,invalid-name - -"""For details on this module see DOCUMENTATION (below)""" - -import base64 -import datetime -import io -import os -import subprocess -import yaml - -# pylint import-error disabled because pylint cannot find the package -# when installed in a virtualenv -from ansible.module_utils.six.moves import configparser # pylint: disable=import-error -from ansible.module_utils.basic import AnsibleModule - -try: - # You can comment this import out and include a 'pass' in this - # block if you're manually testing this module on a NON-ATOMIC - # HOST (or any host that just doesn't have PyOpenSSL - # available). That will force the `load_and_handle_cert` function - # to use the Fake OpenSSL classes. - import OpenSSL.crypto - HAS_OPENSSL = True -except ImportError: - # Some platforms (such as RHEL Atomic) may not have the Python - # OpenSSL library installed. In this case we will use a manual - # work-around to parse each certificate. - # - # Check for 'OpenSSL.crypto' in `sys.modules` later. - HAS_OPENSSL = False - -DOCUMENTATION = ''' ---- -module: openshift_cert_expiry -short_description: Check OpenShift Container Platform (OCP) and Kube certificate expirations on a cluster -description: - - The M(openshift_cert_expiry) module has two basic functions: to flag certificates which will expire in a set window of time from now, and to notify you about certificates which have already expired. - - When the module finishes, a summary of the examination is returned. Each certificate in the summary has a C(health) key with a value of one of the following: - - C(ok) - not expired, and outside of the expiration C(warning_days) window. - - C(warning) - not expired, but will expire between now and the C(warning_days) window. - - C(expired) - an expired certificate. - - Certificate flagging follow this logic: - - If the expiration date is before now then the certificate is classified as C(expired). - - The certificates time to live (expiration date - now) is calculated, if that time window is less than C(warning_days) the certificate is classified as C(warning). - - All other conditions are classified as C(ok). - - The following keys are ALSO present in the certificate summary: - - C(cert_cn) - The common name of the certificate (additional CNs present in SAN extensions are omitted) - - C(days_remaining) - The number of days until the certificate expires. - - C(expiry) - The date the certificate expires on. - - C(path) - The full path to the certificate on the examined host. -version_added: "1.0" -options: - config_base: - description: - - Base path to OCP system settings. - required: false - default: /etc/origin - warning_days: - description: - - Flag certificates which will expire in C(warning_days) days from now. - required: false - default: 30 - show_all: - description: - - Enable this option to show analysis of ALL certificates examined by this module. - - By default only certificates which have expired, or will expire within the C(warning_days) window will be reported. - required: false - default: false - -author: "Tim Bielawa (@tbielawa) " -''' - -EXAMPLES = ''' -# Default invocation, only notify about expired certificates or certificates which will expire within 30 days from now -- openshift_cert_expiry: - -# Expand the warning window to show certificates expiring within a year from now -- openshift_cert_expiry: warning_days=365 - -# Show expired, soon to expire (now + 30 days), and all other certificates examined -- openshift_cert_expiry: show_all=true -''' - - -class FakeOpenSSLCertificate(object): - """This provides a rough mock of what you get from -`OpenSSL.crypto.load_certificate()`. This is a work-around for -platforms missing the Python OpenSSL library. - """ - def __init__(self, cert_string): - """`cert_string` is a certificate in the form you get from running a -.crt through 'openssl x509 -in CERT.cert -text'""" - self.cert_string = cert_string - self.serial = None - self.subject = None - self.extensions = [] - self.not_after = None - self._parse_cert() - - def _parse_cert(self): - """Manually parse the certificate line by line""" - self.extensions = [] - - PARSING_ALT_NAMES = False - PARSING_HEX_SERIAL = False - for line in self.cert_string.split('\n'): - l = line.strip() - if PARSING_ALT_NAMES: - # We're parsing a 'Subject Alternative Name' line - self.extensions.append( - FakeOpenSSLCertificateSANExtension(l)) - - PARSING_ALT_NAMES = False - continue - - if PARSING_HEX_SERIAL: - # Hex serials arrive colon-delimited - serial_raw = l.replace(':', '') - # Convert to decimal - self.serial = int('0x' + serial_raw, base=16) - PARSING_HEX_SERIAL = False - continue - - # parse out the bits that we can - if l.startswith('Serial Number:'): - # Decimal format: - # Serial Number: 11 (0xb) - # => 11 - # Hex Format (large serials): - # Serial Number: - # 0a:de:eb:24:04:75:ab:56:39:14:e9:5a:22:e2:85:bf - # => 14449739080294792594019643629255165375 - if l.endswith(':'): - PARSING_HEX_SERIAL = True - continue - self.serial = int(l.split()[-2]) - - elif l.startswith('Not After :'): - # Not After : Feb 7 18:19:35 2019 GMT - # => strptime(str, '%b %d %H:%M:%S %Y %Z') - # => strftime('%Y%m%d%H%M%SZ') - # => 20190207181935Z - not_after_raw = l.partition(' : ')[-1] - # Last item: ('Not After', ' : ', 'Feb 7 18:19:35 2019 GMT') - not_after_parsed = datetime.datetime.strptime(not_after_raw, '%b %d %H:%M:%S %Y %Z') - self.not_after = not_after_parsed.strftime('%Y%m%d%H%M%SZ') - - elif l.startswith('X509v3 Subject Alternative Name:'): - PARSING_ALT_NAMES = True - continue - - elif l.startswith('Subject:'): - # O = system:nodes, CN = system:node:m01.example.com - self.subject = FakeOpenSSLCertificateSubjects(l.partition(': ')[-1]) - - def get_serial_number(self): - """Return the serial number of the cert""" - return self.serial - - def get_subject(self): - """Subjects must implement get_components() and return dicts or -tuples. An 'openssl x509 -in CERT.cert -text' with 'Subject': - - Subject: Subject: O=system:nodes, CN=system:node:m01.example.com - -might return: [('O=system', 'nodes'), ('CN=system', 'node:m01.example.com')] - """ - return self.subject - - def get_extension(self, i): - """Extensions must implement get_short_name() and return the string -'subjectAltName'""" - return self.extensions[i] - - def get_extension_count(self): - """ get_extension_count """ - return len(self.extensions) - - def get_notAfter(self): - """Returns a date stamp as a string in the form -'20180922170439Z'. strptime the result with format param: -'%Y%m%d%H%M%SZ'.""" - return self.not_after - - -class FakeOpenSSLCertificateSANExtension(object): # pylint: disable=too-few-public-methods - """Mocks what happens when `get_extension` is called on a certificate -object""" - - def __init__(self, san_string): - """With `san_string` as you get from: - - $ openssl x509 -in certificate.crt -text - """ - self.san_string = san_string - self.short_name = 'subjectAltName' - - def get_short_name(self): - """Return the 'type' of this extension. It's always the same though -because we only care about subjectAltName's""" - return self.short_name - - def __str__(self): - """Return this extension and the value as a simple string""" - return self.san_string - - -# pylint: disable=too-few-public-methods -class FakeOpenSSLCertificateSubjects(object): - """Mocks what happens when `get_subject` is called on a certificate -object""" - - def __init__(self, subject_string): - """With `subject_string` as you get from: - - $ openssl x509 -in certificate.crt -text - """ - self.subjects = [] - for s in subject_string.split(', '): - name, _, value = s.partition(' = ') - self.subjects.append((name, value)) - - def get_components(self): - """Returns a list of tuples""" - return self.subjects - - -###################################################################### -def filter_paths(path_list): - """`path_list` - A list of file paths to check. Only files which exist -will be returned - """ - return [p for p in path_list if os.path.exists(os.path.realpath(p))] - - -# pylint: disable=too-many-locals,too-many-branches -# -# TODO: Break this function down into smaller chunks -def load_and_handle_cert(cert_string, now, base64decode=False, ans_module=None): - """Load a certificate, split off the good parts, and return some -useful data - -Params: - -- `cert_string` (string) - a certificate loaded into a string object -- `now` (datetime) - a datetime object of the time to calculate the certificate 'time_remaining' against -- `base64decode` (bool) - run base64.b64decode() on the input -- `ans_module` (AnsibleModule) - The AnsibleModule object for this module (so we can raise errors) - -Returns: -A tuple of the form: - (cert_subject, cert_expiry_date, time_remaining, cert_serial_number) - """ - if base64decode: - _cert_string = base64.b64decode(cert_string).decode('utf-8') - else: - _cert_string = cert_string - - # Disable this. We 'redefine' the type because we are working - # around a missing library on the target host. - # - # pylint: disable=redefined-variable-type - if HAS_OPENSSL: - # No work-around required - cert_loaded = OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, _cert_string) - else: - # Missing library, work-around required. Run the 'openssl' - # command on it to decode it - cmd = 'openssl x509 -text' - try: - openssl_proc = subprocess.Popen(cmd.split(), - stdout=subprocess.PIPE, - stdin=subprocess.PIPE) - except OSError: - ans_module.fail_json(msg="Error: The 'OpenSSL' python library and CLI command were not found on the target host. Unable to parse any certificates. This host will not be included in generated reports.") - else: - openssl_decoded = openssl_proc.communicate(_cert_string.encode('utf-8'))[0].decode('utf-8') - cert_loaded = FakeOpenSSLCertificate(openssl_decoded) - - ###################################################################### - # Read all possible names from the cert - cert_subjects = [] - for name, value in cert_loaded.get_subject().get_components(): - if isinstance(name, bytes) or isinstance(value, bytes): - name = name.decode('utf-8') - value = value.decode('utf-8') - cert_subjects.append('{}:{}'.format(name, value)) - - # To read SANs from a cert we must read the subjectAltName - # extension from the X509 Object. What makes this more difficult - # is that pyOpenSSL does not give extensions as an iterable - san = None - for i in range(cert_loaded.get_extension_count()): - ext = cert_loaded.get_extension(i) - if ext.get_short_name() == 'subjectAltName': - san = ext - - if san is not None: - # The X509Extension object for subjectAltName prints as a - # string with the alt names separated by a comma and a - # space. Split the string by ', ' and then add our new names - # to the list of existing names - cert_subjects.extend(str(san).split(', ')) - - cert_subject = ', '.join(cert_subjects) - ###################################################################### - - # Grab the expiration date - not_after = cert_loaded.get_notAfter() - # example get_notAfter() => 20180922170439Z - if isinstance(not_after, bytes): - not_after = not_after.decode('utf-8') - - cert_expiry_date = datetime.datetime.strptime( - not_after, - '%Y%m%d%H%M%SZ') - - time_remaining = cert_expiry_date - now - - return (cert_subject, cert_expiry_date, time_remaining, cert_loaded.get_serial_number()) - - -def classify_cert(cert_meta, now, time_remaining, expire_window, cert_list): - """Given metadata about a certificate under examination, classify it - into one of three categories, 'ok', 'warning', and 'expired'. - -Params: - -- `cert_meta` dict - A dict with certificate metadata. Required fields - include: 'cert_cn', 'path', 'expiry', 'days_remaining', 'health'. -- `now` (datetime) - a datetime object of the time to calculate the certificate 'time_remaining' against -- `time_remaining` (datetime.timedelta) - a timedelta for how long until the cert expires -- `expire_window` (datetime.timedelta) - a timedelta for how long the warning window is -- `cert_list` list - A list to shove the classified cert into - -Return: -- `cert_list` - The updated list of classified certificates - """ - expiry_str = str(cert_meta['expiry']) - # Categorization - if cert_meta['expiry'] < now: - # This already expired, must NOTIFY - cert_meta['health'] = 'expired' - elif time_remaining < expire_window: - # WARN about this upcoming expirations - cert_meta['health'] = 'warning' - else: - # Not expired or about to expire - cert_meta['health'] = 'ok' - - cert_meta['expiry'] = expiry_str - cert_meta['serial_hex'] = hex(int(cert_meta['serial'])) - cert_list.append(cert_meta) - return cert_list - - -def tabulate_summary(certificates, kubeconfigs, etcd_certs, router_certs, registry_certs): - """Calculate the summary text for when the module finishes -running. This includes counts of each classification and what have -you. - -Params: - -- `certificates` (list of dicts) - Processed `expire_check_result` - dicts with filled in `health` keys for system certificates. -- `kubeconfigs` - as above for kubeconfigs -- `etcd_certs` - as above for etcd certs - -Return: - -- `summary_results` (dict) - Counts of each cert type classification - and total items examined. - """ - items = certificates + kubeconfigs + etcd_certs + router_certs + registry_certs - - summary_results = { - 'system_certificates': len(certificates), - 'kubeconfig_certificates': len(kubeconfigs), - 'etcd_certificates': len(etcd_certs), - 'router_certs': len(router_certs), - 'registry_certs': len(registry_certs), - 'total': len(items), - 'ok': 0, - 'warning': 0, - 'expired': 0 - } - - summary_results['expired'] = len([c for c in items if c['health'] == 'expired']) - summary_results['warning'] = len([c for c in items if c['health'] == 'warning']) - summary_results['ok'] = len([c for c in items if c['health'] == 'ok']) - - return summary_results - - -###################################################################### -# This is our module MAIN function after all, so there's bound to be a -# lot of code bundled up into one block -# -# Reason: These checks are disabled because the issue was introduced -# during a period where the pylint checks weren't enabled for this file -# Status: temporarily disabled pending future refactoring -# pylint: disable=too-many-locals,too-many-statements,too-many-branches -def main(): - """This module examines certificates (in various forms) which compose -an OpenShift Container Platform cluster - """ - - module = AnsibleModule( - argument_spec=dict( - config_base=dict( - required=False, - default="/etc/origin", - type='str'), - warning_days=dict( - required=False, - default=30, - type='int'), - show_all=dict( - required=False, - default=False, - type='bool') - ), - supports_check_mode=True, - ) - - # Basic scaffolding for OpenShift specific certs - openshift_base_config_path = os.path.realpath(module.params['config_base']) - openshift_master_config_path = os.path.join(openshift_base_config_path, - "master", "master-config.yaml") - openshift_node_config_path = os.path.join(openshift_base_config_path, - "node", "node-config.yaml") - openshift_cert_check_paths = [ - openshift_master_config_path, - openshift_node_config_path, - ] - - # Paths for Kubeconfigs. Additional kubeconfigs are conditionally - # checked later in the code - master_kube_configs = ['admin', 'openshift-master', - 'openshift-node', 'openshift-router', - 'openshift-registry'] - - kubeconfig_paths = [] - for m_kube_config in master_kube_configs: - kubeconfig_paths.append( - os.path.join(openshift_base_config_path, "master", m_kube_config + ".kubeconfig") - ) - - # Validate some paths we have the ability to do ahead of time - openshift_cert_check_paths = filter_paths(openshift_cert_check_paths) - kubeconfig_paths = filter_paths(kubeconfig_paths) - - # etcd, where do you hide your certs? Used when parsing etcd.conf - etcd_cert_params = [ - "ETCD_CA_FILE", - "ETCD_CERT_FILE", - "ETCD_PEER_CA_FILE", - "ETCD_PEER_CERT_FILE", - ] - - # Expiry checking stuff - now = datetime.datetime.now() - # todo, catch exception for invalid input and return a fail_json - warning_days = int(module.params['warning_days']) - expire_window = datetime.timedelta(days=warning_days) - - # Module stuff - # - # The results of our cert checking to return from the task call - check_results = {} - check_results['meta'] = {} - check_results['meta']['warning_days'] = warning_days - check_results['meta']['checked_at_time'] = str(now) - check_results['meta']['warn_before_date'] = str(now + expire_window) - check_results['meta']['show_all'] = str(module.params['show_all']) - # All the analyzed certs accumulate here - ocp_certs = [] - - ###################################################################### - # Sure, why not? Let's enable check mode. - if module.check_mode: - check_results['ocp_certs'] = [] - module.exit_json( - check_results=check_results, - msg="Checked 0 total certificates. Expired/Warning/OK: 0/0/0. Warning window: %s days" % module.params['warning_days'], - rc=0, - changed=False - ) - - ###################################################################### - # Check for OpenShift Container Platform specific certs - ###################################################################### - for os_cert in filter_paths(openshift_cert_check_paths): - # Open up that config file and locate the cert and CA - with io.open(os_cert, 'r', encoding='utf-8') as fp: - cert_meta = {} - cfg = yaml.load(fp) - # cert files are specified in parsed `fp` as relative to the path - # of the original config file. 'master-config.yaml' with certFile - # = 'foo.crt' implies that 'foo.crt' is in the same - # directory. certFile = '../foo.crt' is in the parent directory. - cfg_path = os.path.dirname(fp.name) - cert_meta['certFile'] = os.path.join(cfg_path, cfg['servingInfo']['certFile']) - cert_meta['clientCA'] = os.path.join(cfg_path, cfg['servingInfo']['clientCA']) - - ###################################################################### - # Load the certificate and the CA, parse their expiration dates into - # datetime objects so we can manipulate them later - for v in cert_meta.values(): - with io.open(v, 'r', encoding='utf-8') as fp: - cert = fp.read() - (cert_subject, - cert_expiry_date, - time_remaining, - cert_serial) = load_and_handle_cert(cert, now, ans_module=module) - - expire_check_result = { - 'cert_cn': cert_subject, - 'path': fp.name, - 'expiry': cert_expiry_date, - 'days_remaining': time_remaining.days, - 'health': None, - 'serial': cert_serial - } - - classify_cert(expire_check_result, now, time_remaining, expire_window, ocp_certs) - - ###################################################################### - # /Check for OpenShift Container Platform specific certs - ###################################################################### - - ###################################################################### - # Check service Kubeconfigs - ###################################################################### - kubeconfigs = [] - - # There may be additional kubeconfigs to check, but their naming - # is less predictable than the ones we've already assembled. - - try: - # Try to read the standard 'node-config.yaml' file to check if - # this host is a node. - with io.open(openshift_node_config_path, 'r', encoding='utf-8') as fp: - cfg = yaml.load(fp) - - # OK, the config file exists, therefore this is a - # node. Nodes have their own kubeconfig files to - # communicate with the master API. Let's read the relative - # path to that file from the node config. - node_masterKubeConfig = cfg['masterKubeConfig'] - # As before, the path to the 'masterKubeConfig' file is - # relative to `fp` - cfg_path = os.path.dirname(fp.name) - node_kubeconfig = os.path.join(cfg_path, node_masterKubeConfig) - - with io.open(node_kubeconfig, 'r', encoding='utf8') as fp: - # Read in the nodes kubeconfig file and grab the good stuff - cfg = yaml.load(fp) - - c = cfg['users'][0]['user']['client-certificate-data'] - (cert_subject, - cert_expiry_date, - time_remaining, - cert_serial) = load_and_handle_cert(c, now, base64decode=True, ans_module=module) - - expire_check_result = { - 'cert_cn': cert_subject, - 'path': fp.name, - 'expiry': cert_expiry_date, - 'days_remaining': time_remaining.days, - 'health': None, - 'serial': cert_serial - } - - classify_cert(expire_check_result, now, time_remaining, expire_window, kubeconfigs) - except IOError: - # This is not a node - pass - - for kube in filter_paths(kubeconfig_paths): - with io.open(kube, 'r', encoding='utf-8') as fp: - # TODO: Maybe consider catching exceptions here? - cfg = yaml.load(fp) - - # Per conversation, "the kubeconfigs you care about: - # admin, router, registry should all be single - # value". Following that advice we only grab the data for - # the user at index 0 in the 'users' list. There should - # not be more than one user. - c = cfg['users'][0]['user']['client-certificate-data'] - (cert_subject, - cert_expiry_date, - time_remaining, - cert_serial) = load_and_handle_cert(c, now, base64decode=True, ans_module=module) - - expire_check_result = { - 'cert_cn': cert_subject, - 'path': fp.name, - 'expiry': cert_expiry_date, - 'days_remaining': time_remaining.days, - 'health': None, - 'serial': cert_serial - } - - classify_cert(expire_check_result, now, time_remaining, expire_window, kubeconfigs) - - ###################################################################### - # /Check service Kubeconfigs - ###################################################################### - - ###################################################################### - # Check etcd certs - # - # Two things to check: 'external' etcd, and embedded etcd. - ###################################################################### - # FIRST: The 'external' etcd - # - # Some values may be duplicated, make this a set for now so we - # unique them all - etcd_certs_to_check = set([]) - etcd_certs = [] - etcd_cert_params.append('dne') - try: - with io.open('/etc/etcd/etcd.conf', 'r', encoding='utf-8') as fp: - # Add dummy header section. - config = io.StringIO() - config.write(u'[ETCD]\n') - config.write(fp.read().replace('%', '%%')) - config.seek(0, os.SEEK_SET) - - etcd_config = configparser.ConfigParser() - etcd_config.readfp(config) - - for param in etcd_cert_params: - try: - etcd_certs_to_check.add(etcd_config.get('ETCD', param)) - except configparser.NoOptionError: - # That parameter does not exist, oh well... - pass - except IOError: - # No etcd to see here, move along - pass - - for etcd_cert in filter_paths(etcd_certs_to_check): - with io.open(etcd_cert, 'r', encoding='utf-8') as fp: - c = fp.read() - (cert_subject, - cert_expiry_date, - time_remaining, - cert_serial) = load_and_handle_cert(c, now, ans_module=module) - - expire_check_result = { - 'cert_cn': cert_subject, - 'path': fp.name, - 'expiry': cert_expiry_date, - 'days_remaining': time_remaining.days, - 'health': None, - 'serial': cert_serial - } - - classify_cert(expire_check_result, now, time_remaining, expire_window, etcd_certs) - - ###################################################################### - # Now the embedded etcd - ###################################################################### - try: - with io.open('/etc/origin/master/master-config.yaml', 'r', encoding='utf-8') as fp: - cfg = yaml.load(fp) - except IOError: - # Not present - pass - else: - if cfg.get('etcdConfig', {}).get('servingInfo', {}).get('certFile', None) is not None: - # This is embedded - etcd_crt_name = cfg['etcdConfig']['servingInfo']['certFile'] - else: - # Not embedded - etcd_crt_name = None - - if etcd_crt_name is not None: - # etcd_crt_name is relative to the location of the - # master-config.yaml file - cfg_path = os.path.dirname(fp.name) - etcd_cert = os.path.join(cfg_path, etcd_crt_name) - with open(etcd_cert, 'r') as etcd_fp: - (cert_subject, - cert_expiry_date, - time_remaining, - cert_serial) = load_and_handle_cert(etcd_fp.read(), now, ans_module=module) - - expire_check_result = { - 'cert_cn': cert_subject, - 'path': etcd_fp.name, - 'expiry': cert_expiry_date, - 'days_remaining': time_remaining.days, - 'health': None, - 'serial': cert_serial - } - - classify_cert(expire_check_result, now, time_remaining, expire_window, etcd_certs) - - ###################################################################### - # /Check etcd certs - ###################################################################### - - ###################################################################### - # Check router/registry certs - # - # These are saved as secrets in etcd. That means that we can not - # simply read a file to grab the data. Instead we're going to - # subprocess out to the 'oc get' command. On non-masters this - # command will fail, that is expected so we catch that exception. - ###################################################################### - router_certs = [] - registry_certs = [] - - ###################################################################### - # First the router certs - try: - router_secrets_raw = subprocess.Popen('oc get -n default secret router-certs -o yaml'.split(), - stdout=subprocess.PIPE) - router_ds = yaml.load(router_secrets_raw.communicate()[0]) - router_c = router_ds['data']['tls.crt'] - router_path = router_ds['metadata']['selfLink'] - except TypeError: - # YAML couldn't load the result, this is not a master - pass - except OSError: - # The OC command doesn't exist here. Move along. - pass - else: - (cert_subject, - cert_expiry_date, - time_remaining, - cert_serial) = load_and_handle_cert(router_c, now, base64decode=True, ans_module=module) - - expire_check_result = { - 'cert_cn': cert_subject, - 'path': router_path, - 'expiry': cert_expiry_date, - 'days_remaining': time_remaining.days, - 'health': None, - 'serial': cert_serial - } - - classify_cert(expire_check_result, now, time_remaining, expire_window, router_certs) - - ###################################################################### - # Now for registry - try: - registry_secrets_raw = subprocess.Popen('oc get -n default secret registry-certificates -o yaml'.split(), - stdout=subprocess.PIPE) - registry_ds = yaml.load(registry_secrets_raw.communicate()[0]) - registry_c = registry_ds['data']['registry.crt'] - registry_path = registry_ds['metadata']['selfLink'] - except TypeError: - # YAML couldn't load the result, this is not a master - pass - except OSError: - # The OC command doesn't exist here. Move along. - pass - else: - (cert_subject, - cert_expiry_date, - time_remaining, - cert_serial) = load_and_handle_cert(registry_c, now, base64decode=True, ans_module=module) - - expire_check_result = { - 'cert_cn': cert_subject, - 'path': registry_path, - 'expiry': cert_expiry_date, - 'days_remaining': time_remaining.days, - 'health': None, - 'serial': cert_serial - } - - classify_cert(expire_check_result, now, time_remaining, expire_window, registry_certs) - - ###################################################################### - # /Check router/registry certs - ###################################################################### - - res = tabulate_summary(ocp_certs, kubeconfigs, etcd_certs, router_certs, registry_certs) - - msg = "Checked {count} total certificates. Expired/Warning/OK: {exp}/{warn}/{ok}. Warning window: {window} days".format( - count=res['total'], - exp=res['expired'], - warn=res['warning'], - ok=res['ok'], - window=int(module.params['warning_days']), - ) - - # By default we only return detailed information about expired or - # warning certificates. If show_all is true then we will print all - # the certificates examined. - if not module.params['show_all']: - check_results['ocp_certs'] = [crt for crt in ocp_certs if crt['health'] in ['expired', 'warning']] - check_results['kubeconfigs'] = [crt for crt in kubeconfigs if crt['health'] in ['expired', 'warning']] - check_results['etcd'] = [crt for crt in etcd_certs if crt['health'] in ['expired', 'warning']] - check_results['registry'] = [crt for crt in registry_certs if crt['health'] in ['expired', 'warning']] - check_results['router'] = [crt for crt in router_certs if crt['health'] in ['expired', 'warning']] - else: - check_results['ocp_certs'] = ocp_certs - check_results['kubeconfigs'] = kubeconfigs - check_results['etcd'] = etcd_certs - check_results['registry'] = registry_certs - check_results['router'] = router_certs - - # Sort the final results to report in order of ascending safety - # time. That is to say, the certificates which will expire sooner - # will be at the front of the list and certificates which will - # expire later are at the end. Router and registry certs should be - # limited to just 1 result, so don't bother sorting those. - def cert_key(item): - ''' return the days_remaining key ''' - return item['days_remaining'] - - check_results['ocp_certs'] = sorted(check_results['ocp_certs'], key=cert_key) - check_results['kubeconfigs'] = sorted(check_results['kubeconfigs'], key=cert_key) - check_results['etcd'] = sorted(check_results['etcd'], key=cert_key) - - # This module will never change anything, but we might want to - # change the return code parameter if there is some catastrophic - # error we noticed earlier - module.exit_json( - check_results=check_results, - summary=res, - msg=msg, - rc=0, - changed=False - ) - - -if __name__ == '__main__': - main() diff --git a/roles/openshift_certificate_expiry/tasks/main.yml b/roles/openshift_certificate_expiry/tasks/main.yml index 8dea2c07f..7062b5060 100644 --- a/roles/openshift_certificate_expiry/tasks/main.yml +++ b/roles/openshift_certificate_expiry/tasks/main.yml @@ -16,7 +16,9 @@ - name: Generate the result JSON string run_once: yes - set_fact: json_result_string="{{ hostvars|oo_cert_expiry_results_to_json(play_hosts) }}" + set_fact: + # oo_cert_expiry_results_to_json is a custom filter in role lib_utils + json_result_string: "{{ hostvars|oo_cert_expiry_results_to_json(play_hosts) }}" when: openshift_certificate_expiry_save_json_results|bool - name: Generate results JSON file diff --git a/roles/openshift_certificate_expiry/test/conftest.py b/roles/openshift_certificate_expiry/test/conftest.py deleted file mode 100644 index df948fff0..000000000 --- a/roles/openshift_certificate_expiry/test/conftest.py +++ /dev/null @@ -1,119 +0,0 @@ -# pylint: disable=missing-docstring,invalid-name,redefined-outer-name -import pytest -from OpenSSL import crypto - -# Parameter list for valid_cert fixture -VALID_CERTIFICATE_PARAMS = [ - { - 'short_name': 'client', - 'cn': 'client.example.com', - 'serial': 4, - 'uses': b'clientAuth', - 'dns': [], - 'ip': [], - }, - { - 'short_name': 'server', - 'cn': 'server.example.com', - 'serial': 5, - 'uses': b'serverAuth', - 'dns': ['kubernetes', 'openshift'], - 'ip': ['10.0.0.1', '192.168.0.1'] - }, - { - 'short_name': 'combined', - 'cn': 'combined.example.com', - # Verify that HUGE serials parse correctly. - # Frobs PARSING_HEX_SERIAL in _parse_cert - # See https://bugzilla.redhat.com/show_bug.cgi?id=1464240 - 'serial': 14449739080294792594019643629255165375, - 'uses': b'clientAuth, serverAuth', - 'dns': ['etcd'], - 'ip': ['10.0.0.2', '192.168.0.2'] - } -] - -# Extract the short_name from VALID_CERTIFICATE_PARAMS to provide -# friendly naming for the valid_cert fixture -VALID_CERTIFICATE_IDS = [param['short_name'] for param in VALID_CERTIFICATE_PARAMS] - - -@pytest.fixture(scope='session') -def ca(tmpdir_factory): - ca_dir = tmpdir_factory.mktemp('ca') - - key = crypto.PKey() - key.generate_key(crypto.TYPE_RSA, 2048) - - cert = crypto.X509() - cert.set_version(3) - cert.set_serial_number(1) - cert.get_subject().commonName = 'test-signer' - cert.gmtime_adj_notBefore(0) - cert.gmtime_adj_notAfter(24 * 60 * 60) - cert.set_issuer(cert.get_subject()) - cert.set_pubkey(key) - cert.add_extensions([ - crypto.X509Extension(b'basicConstraints', True, b'CA:TRUE, pathlen:0'), - crypto.X509Extension(b'keyUsage', True, - b'digitalSignature, keyEncipherment, keyCertSign, cRLSign'), - crypto.X509Extension(b'subjectKeyIdentifier', False, b'hash', subject=cert) - ]) - cert.add_extensions([ - crypto.X509Extension(b'authorityKeyIdentifier', False, b'keyid:always', issuer=cert) - ]) - cert.sign(key, 'sha256') - - return { - 'dir': ca_dir, - 'key': key, - 'cert': cert, - } - - -@pytest.fixture(scope='session', - ids=VALID_CERTIFICATE_IDS, - params=VALID_CERTIFICATE_PARAMS) -def valid_cert(request, ca): - common_name = request.param['cn'] - - key = crypto.PKey() - key.generate_key(crypto.TYPE_RSA, 2048) - - cert = crypto.X509() - cert.set_serial_number(request.param['serial']) - cert.gmtime_adj_notBefore(0) - cert.gmtime_adj_notAfter(24 * 60 * 60) - cert.set_issuer(ca['cert'].get_subject()) - cert.set_pubkey(key) - cert.set_version(3) - cert.get_subject().commonName = common_name - cert.add_extensions([ - crypto.X509Extension(b'basicConstraints', True, b'CA:FALSE'), - crypto.X509Extension(b'keyUsage', True, b'digitalSignature, keyEncipherment'), - crypto.X509Extension(b'extendedKeyUsage', False, request.param['uses']), - ]) - - if request.param['dns'] or request.param['ip']: - san_list = ['DNS:{}'.format(common_name)] - san_list.extend(['DNS:{}'.format(x) for x in request.param['dns']]) - san_list.extend(['IP:{}'.format(x) for x in request.param['ip']]) - - cert.add_extensions([ - crypto.X509Extension(b'subjectAltName', False, ', '.join(san_list).encode('utf8')) - ]) - cert.sign(ca['key'], 'sha256') - - cert_contents = crypto.dump_certificate(crypto.FILETYPE_PEM, cert) - cert_file = ca['dir'].join('{}.crt'.format(common_name)) - cert_file.write_binary(cert_contents) - - return { - 'common_name': common_name, - 'serial': request.param['serial'], - 'dns': request.param['dns'], - 'ip': request.param['ip'], - 'uses': request.param['uses'], - 'cert_file': cert_file, - 'cert': cert - } diff --git a/roles/openshift_certificate_expiry/test/test_fakeopensslclasses.py b/roles/openshift_certificate_expiry/test/test_fakeopensslclasses.py deleted file mode 100644 index 8a521a765..000000000 --- a/roles/openshift_certificate_expiry/test/test_fakeopensslclasses.py +++ /dev/null @@ -1,90 +0,0 @@ -''' - Unit tests for the FakeOpenSSL classes -''' -import os -import subprocess -import sys - -import pytest - -MODULE_PATH = os.path.realpath(os.path.join(__file__, os.pardir, os.pardir, 'library')) -sys.path.insert(1, MODULE_PATH) - -# pylint: disable=import-error,wrong-import-position,missing-docstring -# pylint: disable=invalid-name,redefined-outer-name -from openshift_cert_expiry import FakeOpenSSLCertificate # noqa: E402 - - -@pytest.fixture(scope='module') -def fake_valid_cert(valid_cert): - cmd = ['openssl', 'x509', '-in', str(valid_cert['cert_file']), '-text', - '-nameopt', 'oneline'] - cert = subprocess.check_output(cmd) - return FakeOpenSSLCertificate(cert.decode('utf8')) - - -def test_not_after(valid_cert, fake_valid_cert): - ''' Validate value returned back from get_notAfter() ''' - real_cert = valid_cert['cert'] - - # Internal representation of pyOpenSSL is bytes, while FakeOpenSSLCertificate - # is text, so decode the result from pyOpenSSL prior to comparing - assert real_cert.get_notAfter().decode('utf8') == fake_valid_cert.get_notAfter() - - -def test_serial(valid_cert, fake_valid_cert): - ''' Validate value returned back form get_serialnumber() ''' - real_cert = valid_cert['cert'] - assert real_cert.get_serial_number() == fake_valid_cert.get_serial_number() - - -def test_get_subject(valid_cert, fake_valid_cert): - ''' Validate the certificate subject ''' - - # Gather the subject components and create a list of colon separated strings. - # Since the internal representation of pyOpenSSL uses bytes, we need to decode - # the results before comparing. - c_subjects = valid_cert['cert'].get_subject().get_components() - c_subj = ', '.join(['{}:{}'.format(x.decode('utf8'), y.decode('utf8')) for x, y in c_subjects]) - f_subjects = fake_valid_cert.get_subject().get_components() - f_subj = ', '.join(['{}:{}'.format(x, y) for x, y in f_subjects]) - assert c_subj == f_subj - - -def get_san_extension(cert): - # Internal representation of pyOpenSSL is bytes, while FakeOpenSSLCertificate - # is text, so we need to set the value to search for accordingly. - if isinstance(cert, FakeOpenSSLCertificate): - san_short_name = 'subjectAltName' - else: - san_short_name = b'subjectAltName' - - for i in range(cert.get_extension_count()): - ext = cert.get_extension(i) - if ext.get_short_name() == san_short_name: - # return the string representation to compare the actual SAN - # values instead of the data types - return str(ext) - - return None - - -def test_subject_alt_names(valid_cert, fake_valid_cert): - real_cert = valid_cert['cert'] - - san = get_san_extension(real_cert) - f_san = get_san_extension(fake_valid_cert) - - assert san == f_san - - # If there are either dns or ip sans defined, verify common_name present - if valid_cert['ip'] or valid_cert['dns']: - assert 'DNS:' + valid_cert['common_name'] in f_san - - # Verify all ip sans are present - for ip in valid_cert['ip']: - assert 'IP Address:' + ip in f_san - - # Verify all dns sans are present - for name in valid_cert['dns']: - assert 'DNS:' + name in f_san diff --git a/roles/openshift_certificate_expiry/test/test_load_and_handle_cert.py b/roles/openshift_certificate_expiry/test/test_load_and_handle_cert.py deleted file mode 100644 index 98792e2ee..000000000 --- a/roles/openshift_certificate_expiry/test/test_load_and_handle_cert.py +++ /dev/null @@ -1,67 +0,0 @@ -''' - Unit tests for the load_and_handle_cert method -''' -import datetime -import os -import sys - -import pytest - -MODULE_PATH = os.path.realpath(os.path.join(__file__, os.pardir, os.pardir, 'library')) -sys.path.insert(1, MODULE_PATH) - -# pylint: disable=import-error,wrong-import-position,missing-docstring -# pylint: disable=invalid-name,redefined-outer-name -import openshift_cert_expiry # noqa: E402 - -# TODO: More testing on the results of the load_and_handle_cert function -# could be implemented here as well, such as verifying subjects -# match up. - - -@pytest.fixture(params=['OpenSSLCertificate', 'FakeOpenSSLCertificate']) -def loaded_cert(request, valid_cert): - """ parameterized fixture to provide load_and_handle_cert results - for both OpenSSL and FakeOpenSSL parsed certificates - """ - now = datetime.datetime.now() - - openshift_cert_expiry.HAS_OPENSSL = request.param == 'OpenSSLCertificate' - - # valid_cert['cert_file'] is a `py.path.LocalPath` object and - # provides a read_text() method for reading the file contents. - cert_string = valid_cert['cert_file'].read_text('utf8') - - (subject, - expiry_date, - time_remaining, - serial) = openshift_cert_expiry.load_and_handle_cert(cert_string, now) - - return { - 'now': now, - 'subject': subject, - 'expiry_date': expiry_date, - 'time_remaining': time_remaining, - 'serial': serial, - } - - -def test_serial(loaded_cert, valid_cert): - """Params: - - * `loaded_cert` comes from the `loaded_cert` fixture in this file - * `valid_cert` comes from the 'valid_cert' fixture in conftest.py - """ - valid_cert_serial = valid_cert['cert'].get_serial_number() - assert loaded_cert['serial'] == valid_cert_serial - - -def test_expiry(loaded_cert): - """Params: - - * `loaded_cert` comes from the `loaded_cert` fixture in this file - """ - expiry_date = loaded_cert['expiry_date'] - time_remaining = loaded_cert['time_remaining'] - now = loaded_cert['now'] - assert expiry_date == now + time_remaining diff --git a/roles/openshift_cli/library/openshift_container_binary_sync.py b/roles/openshift_cli/library/openshift_container_binary_sync.py deleted file mode 100644 index 440b8ec28..000000000 --- a/roles/openshift_cli/library/openshift_container_binary_sync.py +++ /dev/null @@ -1,205 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# pylint: disable=missing-docstring,invalid-name - -import random -import tempfile -import shutil -import os.path - -# pylint: disable=redefined-builtin,wildcard-import,unused-wildcard-import -from ansible.module_utils.basic import * # noqa: F403 - - -DOCUMENTATION = ''' ---- -module: openshift_container_binary_sync -short_description: Copies OpenShift binaries out of the given image tag to host system. -''' - - -class BinarySyncError(Exception): - def __init__(self, msg): - super(BinarySyncError, self).__init__(msg) - self.msg = msg - - -# pylint: disable=too-few-public-methods,too-many-instance-attributes -class BinarySyncer(object): - """ - Syncs the openshift, oc, and kubectl binaries/symlinks out of - a container onto the host system. - """ - - def __init__(self, module, image, tag, backend): - self.module = module - self.changed = False - self.output = [] - self.bin_dir = '/usr/local/bin' - self._image = image - self.tag = tag - self.backend = backend - self.temp_dir = None # TBD - - def sync(self): - if self.backend == 'atomic': - return self._sync_atomic() - - return self._sync_docker() - - def _sync_atomic(self): - self.temp_dir = tempfile.mkdtemp() - temp_dir_mount = tempfile.mkdtemp() - try: - image_spec = '%s:%s' % (self.image, self.tag) - rc, stdout, stderr = self.module.run_command(['atomic', 'mount', - '--storage', "ostree", - image_spec, temp_dir_mount]) - if rc: - raise BinarySyncError("Error mounting image. stdout=%s, stderr=%s" % - (stdout, stderr)) - for i in ["openshift", "oc"]: - src_file = os.path.join(temp_dir_mount, "usr/bin", i) - shutil.copy(src_file, self.temp_dir) - - self._sync_binaries() - finally: - self.module.run_command(['atomic', 'umount', temp_dir_mount]) - shutil.rmtree(temp_dir_mount) - shutil.rmtree(self.temp_dir) - - def _sync_docker(self): - container_name = "openshift-cli-%s" % random.randint(1, 100000) - rc, stdout, stderr = self.module.run_command(['docker', 'create', '--name', - container_name, '%s:%s' % (self.image, self.tag)]) - if rc: - raise BinarySyncError("Error creating temporary docker container. stdout=%s, stderr=%s" % - (stdout, stderr)) - self.output.append(stdout) - try: - self.temp_dir = tempfile.mkdtemp() - self.output.append("Using temp dir: %s" % self.temp_dir) - - rc, stdout, stderr = self.module.run_command(['docker', 'cp', "%s:/usr/bin/openshift" % container_name, - self.temp_dir]) - if rc: - raise BinarySyncError("Error copying file from docker container: stdout=%s, stderr=%s" % - (stdout, stderr)) - - rc, stdout, stderr = self.module.run_command(['docker', 'cp', "%s:/usr/bin/oc" % container_name, - self.temp_dir]) - if rc: - raise BinarySyncError("Error copying file from docker container: stdout=%s, stderr=%s" % - (stdout, stderr)) - - self._sync_binaries() - finally: - shutil.rmtree(self.temp_dir) - self.module.run_command(['docker', 'rm', container_name]) - - def _sync_binaries(self): - self._sync_binary('openshift') - - # In older versions, oc was a symlink to openshift: - if os.path.islink(os.path.join(self.temp_dir, 'oc')): - self._sync_symlink('oc', 'openshift') - else: - self._sync_binary('oc') - - # Ensure correct symlinks created: - self._sync_symlink('kubectl', 'openshift') - - # Remove old oadm binary - if os.path.exists(os.path.join(self.bin_dir, 'oadm')): - os.remove(os.path.join(self.bin_dir, 'oadm')) - - def _sync_symlink(self, binary_name, link_to): - """ Ensure the given binary name exists and links to the expected binary. """ - - # The symlink we are creating: - link_path = os.path.join(self.bin_dir, binary_name) - - # The expected file we should be linking to: - link_dest = os.path.join(self.bin_dir, link_to) - - if not os.path.exists(link_path) or \ - not os.path.islink(link_path) or \ - os.path.realpath(link_path) != os.path.realpath(link_dest): - if os.path.exists(link_path): - os.remove(link_path) - os.symlink(link_to, os.path.join(self.bin_dir, binary_name)) - self.output.append("Symlinked %s to %s." % (link_path, link_dest)) - self.changed = True - - def _sync_binary(self, binary_name): - src_path = os.path.join(self.temp_dir, binary_name) - dest_path = os.path.join(self.bin_dir, binary_name) - incoming_checksum = self.module.run_command(['sha256sum', src_path])[1] - if not os.path.exists(dest_path) or self.module.run_command(['sha256sum', dest_path])[1] != incoming_checksum: - - # See: https://github.com/openshift/openshift-ansible/issues/4965 - if os.path.islink(dest_path): - os.unlink(dest_path) - self.output.append('Removed old symlink {} before copying binary.'.format(dest_path)) - shutil.move(src_path, dest_path) - self.output.append("Moved %s to %s." % (src_path, dest_path)) - self.changed = True - - @property - def raw_image(self): - """ - Returns the image as it was originally passed in to the instance. - - .. note:: - This image string will only work directly with the atomic command. - - :returns: The original image passed in. - :rtype: str - """ - return self._image - - @property - def image(self): - """ - Returns the image without atomic prefixes used to map to skopeo args. - - :returns: The image string without prefixes - :rtype: str - """ - image = self._image - for remove in ('oci:', 'http:', 'https:'): - if image.startswith(remove): - image = image.replace(remove, '') - return image - - -def main(): - module = AnsibleModule( # noqa: F405 - argument_spec=dict( - image=dict(required=True), - tag=dict(required=True), - backend=dict(required=True), - ), - supports_check_mode=True - ) - - image = module.params['image'] - tag = module.params['tag'] - backend = module.params['backend'] - - if backend not in ["docker", "atomic"]: - module.fail_json(msg="unknown backend") - - binary_syncer = BinarySyncer(module, image, tag, backend) - - try: - binary_syncer.sync() - except BinarySyncError as ex: - module.fail_json(msg=ex.msg) - - return module.exit_json(changed=binary_syncer.changed, - output=binary_syncer.output) - - -if __name__ == '__main__': - main() diff --git a/roles/openshift_cli/tasks/main.yml b/roles/openshift_cli/tasks/main.yml index 37bed9dbe..ae8d1ace0 100644 --- a/roles/openshift_cli/tasks/main.yml +++ b/roles/openshift_cli/tasks/main.yml @@ -12,6 +12,7 @@ register: pull_result changed_when: "'Downloaded newer image' in pull_result.stdout" + # openshift_container_binary_sync is a custom module in lib_utils - name: Copy client binaries/symlinks out of CLI image for use on the host openshift_container_binary_sync: image: "{{ openshift_cli_image }}" @@ -28,6 +29,7 @@ register: pull_result changed_when: "'Pulling layer' in pull_result.stdout" + # openshift_container_binary_sync is a custom module in lib_utils - name: Copy client binaries/symlinks out of CLI image for use on the host openshift_container_binary_sync: image: "{{ '' if system_images_registry == 'docker' else system_images_registry + '/' }}{{ openshift_cli_image }}" diff --git a/roles/openshift_health_checker/openshift_checks/disk_availability.py b/roles/openshift_health_checker/openshift_checks/disk_availability.py index 87e6146d4..6e30a8610 100644 --- a/roles/openshift_health_checker/openshift_checks/disk_availability.py +++ b/roles/openshift_health_checker/openshift_checks/disk_availability.py @@ -21,7 +21,7 @@ class DiskAvailability(OpenShiftCheck): 'oo_etcd_to_config': 20 * 10**9, }, # Used to copy client binaries into, - # see roles/openshift_cli/library/openshift_container_binary_sync.py. + # see roles/lib_utils/library/openshift_container_binary_sync.py. '/usr/local/bin': { 'oo_masters_to_config': 1 * 10**9, 'oo_nodes_to_config': 1 * 10**9, diff --git a/roles/openshift_hosted/filter_plugins/openshift_hosted_filters.py b/roles/openshift_hosted/filter_plugins/openshift_hosted_filters.py deleted file mode 100644 index 003ce5f9e..000000000 --- a/roles/openshift_hosted/filter_plugins/openshift_hosted_filters.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -''' -Custom filters for use in openshift_hosted -''' - - -class FilterModule(object): - ''' Custom ansible filters for use by openshift_hosted role''' - - @staticmethod - def get_router_replicas(replicas=None, router_nodes=None): - ''' This function will return the number of replicas - based on the results from the defined - openshift_hosted_router_replicas OR - the query from oc_obj on openshift nodes with a selector OR - default to 1 - - ''' - # We always use what they've specified if they've specified a value - if replicas is not None: - return replicas - - replicas = 1 - - # Ignore boolean expression limit of 5. - # pylint: disable=too-many-boolean-expressions - if (isinstance(router_nodes, dict) and - 'results' in router_nodes and - 'results' in router_nodes['results'] and - isinstance(router_nodes['results']['results'], list) and - len(router_nodes['results']['results']) > 0 and - 'items' in router_nodes['results']['results'][0]): - - if len(router_nodes['results']['results'][0]['items']) > 0: - replicas = len(router_nodes['results']['results'][0]['items']) - - return replicas - - def filters(self): - ''' returns a mapping of filters to methods ''' - return {'get_router_replicas': self.get_router_replicas} diff --git a/roles/openshift_hosted/tasks/router.yml b/roles/openshift_hosted/tasks/router.yml index 2dc9c98f6..c2be00d19 100644 --- a/roles/openshift_hosted/tasks/router.yml +++ b/roles/openshift_hosted/tasks/router.yml @@ -18,6 +18,7 @@ - name: set_fact replicas set_fact: + # get_router_replicas is a custom filter in role lib_utils replicas: "{{ openshift_hosted_router_replicas | default(None) | get_router_replicas(router_nodes) }}" - name: Get the certificate contents for router diff --git a/roles/openshift_logging/filter_plugins/openshift_logging.py b/roles/openshift_logging/filter_plugins/openshift_logging.py index ba412b5a6..247c7e4df 100644 --- a/roles/openshift_logging/filter_plugins/openshift_logging.py +++ b/roles/openshift_logging/filter_plugins/openshift_logging.py @@ -79,14 +79,6 @@ def entry_from_named_pair(register_pairs, key): raise RuntimeError("There was no entry found in the dict that had an item with a name that matched {}".format(key)) -def map_from_pairs(source, delim="="): - ''' Returns a dict given the source and delim delimited ''' - if source == '': - return dict() - - return dict(item.split(delim) for item in source.split(",")) - - def serviceaccount_name(qualified_sa): ''' Returns the simple name from a fully qualified name ''' return qualified_sa.split(":")[-1] @@ -134,7 +126,6 @@ class FilterModule(object): return { 'random_word': random_word, 'entry_from_named_pair': entry_from_named_pair, - 'map_from_pairs': map_from_pairs, 'min_cpu': min_cpu, 'es_storage': es_storage, 'serviceaccount_name': serviceaccount_name, diff --git a/roles/openshift_logging_fluentd/defaults/main.yml b/roles/openshift_logging_fluentd/defaults/main.yml index 9b58e4456..87b4204b5 100644 --- a/roles/openshift_logging_fluentd/defaults/main.yml +++ b/roles/openshift_logging_fluentd/defaults/main.yml @@ -5,6 +5,7 @@ openshift_logging_fluentd_master_url: "https://kubernetes.default.svc.{{ openshi openshift_logging_fluentd_namespace: logging ### Common settings +# map_from_pairs is a custom filter plugin in role lib_utils openshift_logging_fluentd_nodeselector: "{{ openshift_hosted_logging_fluentd_nodeselector_label | default('logging-infra-fluentd=true') | map_from_pairs }}" openshift_logging_fluentd_cpu_limit: null openshift_logging_fluentd_cpu_request: 100m diff --git a/roles/openshift_logging_mux/defaults/main.yml b/roles/openshift_logging_mux/defaults/main.yml index db6f23126..369ba86b3 100644 --- a/roles/openshift_logging_mux/defaults/main.yml +++ b/roles/openshift_logging_mux/defaults/main.yml @@ -6,6 +6,7 @@ openshift_logging_mux_master_public_url: "{{ openshift_hosted_logging_master_pub openshift_logging_mux_namespace: logging ### Common settings +# map_from_pairs is a custom filter plugin in role lib_utils openshift_logging_mux_nodeselector: "{{ openshift_hosted_logging_mux_nodeselector_label | default('') | map_from_pairs }}" openshift_logging_mux_cpu_limit: null openshift_logging_mux_cpu_request: 100m diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml index eea1401b8..b12a6b346 100644 --- a/roles/openshift_master/tasks/main.yml +++ b/roles/openshift_master/tasks/main.yml @@ -181,6 +181,7 @@ - restart master api - set_fact: + # translate_idps is a custom filter in role lib_utils translated_identity_providers: "{{ openshift.master.identity_providers | translate_idps('v1') }}" # TODO: add the validate parameter when there is a validation command to run diff --git a/roles/openshift_master/tasks/upgrade/upgrade_scheduler.yml b/roles/openshift_master/tasks/upgrade/upgrade_scheduler.yml index 8558bf3e9..995a5ab70 100644 --- a/roles/openshift_master/tasks/upgrade/upgrade_scheduler.yml +++ b/roles/openshift_master/tasks/upgrade/upgrade_scheduler.yml @@ -1,6 +1,8 @@ --- # Upgrade predicates - vars: + # openshift_master_facts_default_predicates is a custom lookup plugin in + # role lib_utils prev_predicates: "{{ lookup('openshift_master_facts_default_predicates', short_version=openshift_upgrade_min, deployment_type=openshift_deployment_type) }}" prev_predicates_no_region: "{{ lookup('openshift_master_facts_default_predicates', short_version=openshift_upgrade_min, deployment_type=openshift_deployment_type, regions_enabled=False) }}" default_predicates_no_region: "{{ lookup('openshift_master_facts_default_predicates', regions_enabled=False) }}" diff --git a/roles/openshift_master_certificates/tasks/main.yml b/roles/openshift_master_certificates/tasks/main.yml index 649a4bc5d..ce27e238f 100644 --- a/roles/openshift_master_certificates/tasks/main.yml +++ b/roles/openshift_master_certificates/tasks/main.yml @@ -101,6 +101,7 @@ state: hard force: true with_items: + # certificates_to_synchronize is a custom filter in lib_utils - "{{ hostvars[inventory_hostname] | certificates_to_synchronize }}" when: master_certs_missing | bool and inventory_hostname != openshift_ca_host delegate_to: "{{ openshift_ca_host }}" diff --git a/roles/openshift_master_facts/filter_plugins/openshift_master.py b/roles/openshift_master_facts/filter_plugins/openshift_master.py deleted file mode 100644 index ff15f693b..000000000 --- a/roles/openshift_master_facts/filter_plugins/openshift_master.py +++ /dev/null @@ -1,532 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -''' -Custom filters for use in openshift-master -''' -import copy -import sys - -from ansible import errors -from ansible.parsing.yaml.dumper import AnsibleDumper -from ansible.plugins.filter.core import to_bool as ansible_bool - -# ansible.compat.six goes away with Ansible 2.4 -try: - from ansible.compat.six import string_types, u -except ImportError: - from ansible.module_utils.six import string_types, u - -import yaml - - -class IdentityProviderBase(object): - """ IdentityProviderBase - - Attributes: - name (str): Identity provider Name - login (bool): Is this identity provider a login provider? - challenge (bool): Is this identity provider a challenge provider? - provider (dict): Provider specific config - _idp (dict): internal copy of the IDP dict passed in - _required (list): List of lists of strings for required attributes - _optional (list): List of lists of strings for optional attributes - _allow_additional (bool): Does this provider support attributes - not in _required and _optional - - Args: - api_version(str): OpenShift config version - idp (dict): idp config dict - - Raises: - AnsibleFilterError: - """ - # disabling this check since the number of instance attributes are - # necessary for this class - # pylint: disable=too-many-instance-attributes - def __init__(self, api_version, idp): - if api_version not in ['v1']: - raise errors.AnsibleFilterError("|failed api version {0} unknown".format(api_version)) - - self._idp = copy.deepcopy(idp) - - if 'name' not in self._idp: - raise errors.AnsibleFilterError("|failed identity provider missing a name") - - if 'kind' not in self._idp: - raise errors.AnsibleFilterError("|failed identity provider missing a kind") - - self.name = self._idp.pop('name') - self.login = ansible_bool(self._idp.pop('login', False)) - self.challenge = ansible_bool(self._idp.pop('challenge', False)) - self.provider = dict(apiVersion=api_version, kind=self._idp.pop('kind')) - - mm_keys = ('mappingMethod', 'mapping_method') - mapping_method = None - for key in mm_keys: - if key in self._idp: - mapping_method = self._idp.pop(key) - if mapping_method is None: - mapping_method = self.get_default('mappingMethod') - self.mapping_method = mapping_method - - valid_mapping_methods = ['add', 'claim', 'generate', 'lookup'] - if self.mapping_method not in valid_mapping_methods: - raise errors.AnsibleFilterError("|failed unknown mapping method " - "for provider {0}".format(self.__class__.__name__)) - self._required = [] - self._optional = [] - self._allow_additional = True - - @staticmethod - def validate_idp_list(idp_list): - ''' validates a list of idps ''' - names = [x.name for x in idp_list] - if len(set(names)) != len(names): - raise errors.AnsibleFilterError("|failed more than one provider configured with the same name") - - for idp in idp_list: - idp.validate() - - def validate(self): - ''' validate an instance of this idp class ''' - pass - - @staticmethod - def get_default(key): - ''' get a default value for a given key ''' - if key == 'mappingMethod': - return 'claim' - else: - return None - - def set_provider_item(self, items, required=False): - ''' set a provider item based on the list of item names provided. ''' - for item in items: - provider_key = items[0] - if item in self._idp: - self.provider[provider_key] = self._idp.pop(item) - break - else: - default = self.get_default(provider_key) - if default is not None: - self.provider[provider_key] = default - elif required: - raise errors.AnsibleFilterError("|failed provider {0} missing " - "required key {1}".format(self.__class__.__name__, provider_key)) - - def set_provider_items(self): - ''' set the provider items for this idp ''' - for items in self._required: - self.set_provider_item(items, True) - for items in self._optional: - self.set_provider_item(items) - if self._allow_additional: - for key in self._idp.keys(): - self.set_provider_item([key]) - else: - if len(self._idp) > 0: - raise errors.AnsibleFilterError("|failed provider {0} " - "contains unknown keys " - "{1}".format(self.__class__.__name__, ', '.join(self._idp.keys()))) - - def to_dict(self): - ''' translate this idp to a dictionary ''' - return dict(name=self.name, challenge=self.challenge, - login=self.login, mappingMethod=self.mapping_method, - provider=self.provider) - - -class LDAPPasswordIdentityProvider(IdentityProviderBase): - """ LDAPPasswordIdentityProvider - - Attributes: - - Args: - api_version(str): OpenShift config version - idp (dict): idp config dict - - Raises: - AnsibleFilterError: - """ - def __init__(self, api_version, idp): - super(LDAPPasswordIdentityProvider, self).__init__(api_version, idp) - self._allow_additional = False - self._required += [['attributes'], ['url'], ['insecure']] - self._optional += [['ca'], - ['bindDN', 'bind_dn'], - ['bindPassword', 'bind_password']] - - self._idp['insecure'] = ansible_bool(self._idp.pop('insecure', False)) - - if 'attributes' in self._idp and 'preferred_username' in self._idp['attributes']: - pref_user = self._idp['attributes'].pop('preferred_username') - self._idp['attributes']['preferredUsername'] = pref_user - - def validate(self): - ''' validate this idp instance ''' - if not isinstance(self.provider['attributes'], dict): - raise errors.AnsibleFilterError("|failed attributes for provider " - "{0} must be a dictionary".format(self.__class__.__name__)) - - attrs = ['id', 'email', 'name', 'preferredUsername'] - for attr in attrs: - if attr in self.provider['attributes'] and not isinstance(self.provider['attributes'][attr], list): - raise errors.AnsibleFilterError("|failed {0} attribute for " - "provider {1} must be a list".format(attr, self.__class__.__name__)) - - unknown_attrs = set(self.provider['attributes'].keys()) - set(attrs) - if len(unknown_attrs) > 0: - raise errors.AnsibleFilterError("|failed provider {0} has unknown " - "attributes: {1}".format(self.__class__.__name__, ', '.join(unknown_attrs))) - - -class KeystonePasswordIdentityProvider(IdentityProviderBase): - """ KeystoneIdentityProvider - - Attributes: - - Args: - api_version(str): OpenShift config version - idp (dict): idp config dict - - Raises: - AnsibleFilterError: - """ - def __init__(self, api_version, idp): - super(KeystonePasswordIdentityProvider, self).__init__(api_version, idp) - self._allow_additional = False - self._required += [['url'], ['domainName', 'domain_name']] - self._optional += [['ca'], ['certFile', 'cert_file'], ['keyFile', 'key_file']] - - -class RequestHeaderIdentityProvider(IdentityProviderBase): - """ RequestHeaderIdentityProvider - - Attributes: - - Args: - api_version(str): OpenShift config version - idp (dict): idp config dict - - Raises: - AnsibleFilterError: - """ - def __init__(self, api_version, idp): - super(RequestHeaderIdentityProvider, self).__init__(api_version, idp) - self._allow_additional = False - self._required += [['headers']] - self._optional += [['challengeURL', 'challenge_url'], - ['loginURL', 'login_url'], - ['clientCA', 'client_ca'], - ['clientCommonNames', 'client_common_names'], - ['emailHeaders', 'email_headers'], - ['nameHeaders', 'name_headers'], - ['preferredUsernameHeaders', 'preferred_username_headers']] - - def validate(self): - ''' validate this idp instance ''' - if not isinstance(self.provider['headers'], list): - raise errors.AnsibleFilterError("|failed headers for provider {0} " - "must be a list".format(self.__class__.__name__)) - - -class AllowAllPasswordIdentityProvider(IdentityProviderBase): - """ AllowAllPasswordIdentityProvider - - Attributes: - - Args: - api_version(str): OpenShift config version - idp (dict): idp config dict - - Raises: - AnsibleFilterError: - """ - def __init__(self, api_version, idp): - super(AllowAllPasswordIdentityProvider, self).__init__(api_version, idp) - self._allow_additional = False - - -class DenyAllPasswordIdentityProvider(IdentityProviderBase): - """ DenyAllPasswordIdentityProvider - - Attributes: - - Args: - api_version(str): OpenShift config version - idp (dict): idp config dict - - Raises: - AnsibleFilterError: - """ - def __init__(self, api_version, idp): - super(DenyAllPasswordIdentityProvider, self).__init__(api_version, idp) - self._allow_additional = False - - -class HTPasswdPasswordIdentityProvider(IdentityProviderBase): - """ HTPasswdPasswordIdentity - - Attributes: - - Args: - api_version(str): OpenShift config version - idp (dict): idp config dict - - Raises: - AnsibleFilterError: - """ - def __init__(self, api_version, idp): - super(HTPasswdPasswordIdentityProvider, self).__init__(api_version, idp) - self._allow_additional = False - self._required += [['file', 'filename', 'fileName', 'file_name']] - - @staticmethod - def get_default(key): - if key == 'file': - return '/etc/origin/htpasswd' - else: - return IdentityProviderBase.get_default(key) - - -class BasicAuthPasswordIdentityProvider(IdentityProviderBase): - """ BasicAuthPasswordIdentityProvider - - Attributes: - - Args: - api_version(str): OpenShift config version - idp (dict): idp config dict - - Raises: - AnsibleFilterError: - """ - def __init__(self, api_version, idp): - super(BasicAuthPasswordIdentityProvider, self).__init__(api_version, idp) - self._allow_additional = False - self._required += [['url']] - self._optional += [['ca'], ['certFile', 'cert_file'], ['keyFile', 'key_file']] - - -class IdentityProviderOauthBase(IdentityProviderBase): - """ IdentityProviderOauthBase - - Attributes: - - Args: - api_version(str): OpenShift config version - idp (dict): idp config dict - - Raises: - AnsibleFilterError: - """ - def __init__(self, api_version, idp): - super(IdentityProviderOauthBase, self).__init__(api_version, idp) - self._allow_additional = False - self._required += [['clientID', 'client_id'], ['clientSecret', 'client_secret']] - - def validate(self): - ''' validate an instance of this idp class ''' - pass - - -class OpenIDIdentityProvider(IdentityProviderOauthBase): - """ OpenIDIdentityProvider - - Attributes: - - Args: - api_version(str): OpenShift config version - idp (dict): idp config dict - - Raises: - AnsibleFilterError: - """ - def __init__(self, api_version, idp): - IdentityProviderOauthBase.__init__(self, api_version, idp) - self._required += [['claims'], ['urls']] - self._optional += [['ca'], - ['extraScopes'], - ['extraAuthorizeParameters']] - if 'claims' in self._idp and 'preferred_username' in self._idp['claims']: - pref_user = self._idp['claims'].pop('preferred_username') - self._idp['claims']['preferredUsername'] = pref_user - if 'urls' in self._idp and 'user_info' in self._idp['urls']: - user_info = self._idp['urls'].pop('user_info') - self._idp['urls']['userInfo'] = user_info - if 'extra_scopes' in self._idp: - self._idp['extraScopes'] = self._idp.pop('extra_scopes') - if 'extra_authorize_parameters' in self._idp: - self._idp['extraAuthorizeParameters'] = self._idp.pop('extra_authorize_parameters') - - def validate(self): - ''' validate this idp instance ''' - if not isinstance(self.provider['claims'], dict): - raise errors.AnsibleFilterError("|failed claims for provider {0} " - "must be a dictionary".format(self.__class__.__name__)) - - for var, var_type in (('extraScopes', list), ('extraAuthorizeParameters', dict)): - if var in self.provider and not isinstance(self.provider[var], var_type): - raise errors.AnsibleFilterError("|failed {1} for provider " - "{0} must be a {2}".format(self.__class__.__name__, - var, - var_type.__class__.__name__)) - - required_claims = ['id'] - optional_claims = ['email', 'name', 'preferredUsername'] - all_claims = required_claims + optional_claims - - for claim in required_claims: - if claim in required_claims and claim not in self.provider['claims']: - raise errors.AnsibleFilterError("|failed {0} claim missing " - "for provider {1}".format(claim, self.__class__.__name__)) - - for claim in all_claims: - if claim in self.provider['claims'] and not isinstance(self.provider['claims'][claim], list): - raise errors.AnsibleFilterError("|failed {0} claims for " - "provider {1} must be a list".format(claim, self.__class__.__name__)) - - unknown_claims = set(self.provider['claims'].keys()) - set(all_claims) - if len(unknown_claims) > 0: - raise errors.AnsibleFilterError("|failed provider {0} has unknown " - "claims: {1}".format(self.__class__.__name__, ', '.join(unknown_claims))) - - if not isinstance(self.provider['urls'], dict): - raise errors.AnsibleFilterError("|failed urls for provider {0} " - "must be a dictionary".format(self.__class__.__name__)) - - required_urls = ['authorize', 'token'] - optional_urls = ['userInfo'] - all_urls = required_urls + optional_urls - - for url in required_urls: - if url not in self.provider['urls']: - raise errors.AnsibleFilterError("|failed {0} url missing for " - "provider {1}".format(url, self.__class__.__name__)) - - unknown_urls = set(self.provider['urls'].keys()) - set(all_urls) - if len(unknown_urls) > 0: - raise errors.AnsibleFilterError("|failed provider {0} has unknown " - "urls: {1}".format(self.__class__.__name__, ', '.join(unknown_urls))) - - -class GoogleIdentityProvider(IdentityProviderOauthBase): - """ GoogleIdentityProvider - - Attributes: - - Args: - api_version(str): OpenShift config version - idp (dict): idp config dict - - Raises: - AnsibleFilterError: - """ - def __init__(self, api_version, idp): - IdentityProviderOauthBase.__init__(self, api_version, idp) - self._optional += [['hostedDomain', 'hosted_domain']] - - def validate(self): - ''' validate this idp instance ''' - if self.challenge: - raise errors.AnsibleFilterError("|failed provider {0} does not " - "allow challenge authentication".format(self.__class__.__name__)) - - -class GitHubIdentityProvider(IdentityProviderOauthBase): - """ GitHubIdentityProvider - - Attributes: - - Args: - api_version(str): OpenShift config version - idp (dict): idp config dict - - Raises: - AnsibleFilterError: - """ - def __init__(self, api_version, idp): - IdentityProviderOauthBase.__init__(self, api_version, idp) - self._optional += [['organizations'], - ['teams']] - - def validate(self): - ''' validate this idp instance ''' - if self.challenge: - raise errors.AnsibleFilterError("|failed provider {0} does not " - "allow challenge authentication".format(self.__class__.__name__)) - - -class FilterModule(object): - ''' Custom ansible filters for use by the openshift_master role''' - - @staticmethod - def translate_idps(idps, api_version): - ''' Translates a list of dictionaries into a valid identityProviders config ''' - idp_list = [] - - if not isinstance(idps, list): - raise errors.AnsibleFilterError("|failed expects to filter on a list of identity providers") - for idp in idps: - if not isinstance(idp, dict): - raise errors.AnsibleFilterError("|failed identity providers must be a list of dictionaries") - - cur_module = sys.modules[__name__] - idp_class = getattr(cur_module, idp['kind'], None) - idp_inst = idp_class(api_version, idp) if idp_class is not None else IdentityProviderBase(api_version, idp) - idp_inst.set_provider_items() - idp_list.append(idp_inst) - - IdentityProviderBase.validate_idp_list(idp_list) - return u(yaml.dump([idp.to_dict() for idp in idp_list], - allow_unicode=True, - default_flow_style=False, - width=float("inf"), - Dumper=AnsibleDumper)) - - @staticmethod - def certificates_to_synchronize(hostvars, include_keys=True, include_ca=True): - ''' Return certificates to synchronize based on facts. ''' - if not issubclass(type(hostvars), dict): - raise errors.AnsibleFilterError("|failed expects hostvars is a dict") - certs = ['admin.crt', - 'admin.key', - 'admin.kubeconfig', - 'master.kubelet-client.crt', - 'master.kubelet-client.key', - 'master.proxy-client.crt', - 'master.proxy-client.key', - 'service-signer.crt', - 'service-signer.key'] - if bool(include_ca): - certs += ['ca.crt', 'ca.key', 'ca-bundle.crt', 'client-ca-bundle.crt'] - if bool(include_keys): - certs += ['serviceaccounts.private.key', - 'serviceaccounts.public.key'] - return certs - - @staticmethod - def oo_htpasswd_users_from_file(file_contents): - ''' return a dictionary of htpasswd users from htpasswd file contents ''' - htpasswd_entries = {} - if not isinstance(file_contents, string_types): - raise errors.AnsibleFilterError("failed, expects to filter on a string") - for line in file_contents.splitlines(): - user = None - passwd = None - if len(line) == 0: - continue - if ':' in line: - user, passwd = line.split(':', 1) - - if user is None or len(user) == 0 or passwd is None or len(passwd) == 0: - error_msg = "failed, expects each line to be a colon separated string representing the user and passwd" - raise errors.AnsibleFilterError(error_msg) - htpasswd_entries[user] = passwd - return htpasswd_entries - - def filters(self): - ''' returns a mapping of filters to methods ''' - return {"translate_idps": self.translate_idps, - "certificates_to_synchronize": self.certificates_to_synchronize, - "oo_htpasswd_users_from_file": self.oo_htpasswd_users_from_file} diff --git a/roles/openshift_master_facts/tasks/main.yml b/roles/openshift_master_facts/tasks/main.yml index 85d0ac25c..f450c916a 100644 --- a/roles/openshift_master_facts/tasks/main.yml +++ b/roles/openshift_master_facts/tasks/main.yml @@ -57,6 +57,7 @@ access_token_max_seconds: "{{ openshift_master_access_token_max_seconds | default(None) }}" auth_token_max_seconds: "{{ openshift_master_auth_token_max_seconds | default(None) }}" identity_providers: "{{ openshift_master_identity_providers | default(None) }}" + # oo_htpasswd_users_from_file is a custom filter in role lib_utils htpasswd_users: "{{ openshift_master_htpasswd_users | default(lookup('file', openshift_master_htpasswd_file) | oo_htpasswd_users_from_file if openshift_master_htpasswd_file is defined else None) }}" manage_htpasswd: "{{ openshift_master_manage_htpasswd | default(true) }}" ldap_ca: "{{ openshift_master_ldap_ca | default(lookup('file', openshift_master_ldap_ca_file) if openshift_master_ldap_ca_file is defined else None) }}" @@ -90,6 +91,8 @@ - name: Set Default scheduler predicates and priorities set_fact: + # openshift_master_facts_default_predicates is a custom lookup plugin in + # role lib_utils openshift_master_scheduler_default_predicates: "{{ lookup('openshift_master_facts_default_predicates') }}" openshift_master_scheduler_default_priorities: "{{ lookup('openshift_master_facts_default_priorities') }}" diff --git a/roles/openshift_master_facts/test/conftest.py b/roles/openshift_master_facts/test/conftest.py deleted file mode 100644 index 140cced73..000000000 --- a/roles/openshift_master_facts/test/conftest.py +++ /dev/null @@ -1,54 +0,0 @@ -import os -import sys - -import pytest - -sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, "lookup_plugins")) - -from openshift_master_facts_default_predicates import LookupModule as PredicatesLookupModule # noqa: E402 -from openshift_master_facts_default_priorities import LookupModule as PrioritiesLookupModule # noqa: E402 - - -@pytest.fixture() -def predicates_lookup(): - return PredicatesLookupModule() - - -@pytest.fixture() -def priorities_lookup(): - return PrioritiesLookupModule() - - -@pytest.fixture() -def facts(): - return { - 'openshift': { - 'common': {} - } - } - - -@pytest.fixture(params=[True, False]) -def regions_enabled(request): - return request.param - - -@pytest.fixture(params=[True, False]) -def zones_enabled(request): - return request.param - - -def v_prefix(release): - """Prefix a release number with 'v'.""" - return "v" + release - - -def minor(release): - """Add a suffix to release, making 'X.Y' become 'X.Y.Z'.""" - return release + ".1" - - -@pytest.fixture(params=[str, v_prefix, minor]) -def release_mod(request): - """Modifies a release string to alternative valid values.""" - return request.param diff --git a/roles/openshift_master_facts/test/openshift_master_facts_bad_input_tests.py b/roles/openshift_master_facts/test/openshift_master_facts_bad_input_tests.py deleted file mode 100644 index e8da1e04a..000000000 --- a/roles/openshift_master_facts/test/openshift_master_facts_bad_input_tests.py +++ /dev/null @@ -1,57 +0,0 @@ -import copy -import os -import sys - -from ansible.errors import AnsibleError -import pytest - -sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, "lookup_plugins")) - -from openshift_master_facts_default_predicates import LookupModule # noqa: E402 - - -class TestOpenShiftMasterFactsBadInput(object): - lookup = LookupModule() - default_facts = { - 'openshift': { - 'common': {} - } - } - - def test_missing_openshift_facts(self): - with pytest.raises(AnsibleError): - facts = {} - self.lookup.run(None, variables=facts) - - def test_missing_deployment_type(self): - with pytest.raises(AnsibleError): - facts = copy.deepcopy(self.default_facts) - facts['openshift']['common']['short_version'] = '10.10' - self.lookup.run(None, variables=facts) - - def test_missing_short_version_and_missing_openshift_release(self): - with pytest.raises(AnsibleError): - facts = copy.deepcopy(self.default_facts) - facts['openshift']['common']['deployment_type'] = 'origin' - self.lookup.run(None, variables=facts) - - def test_unknown_deployment_types(self): - with pytest.raises(AnsibleError): - facts = copy.deepcopy(self.default_facts) - facts['openshift']['common']['short_version'] = '1.1' - facts['openshift']['common']['deployment_type'] = 'bogus' - self.lookup.run(None, variables=facts) - - def test_unknown_origin_version(self): - with pytest.raises(AnsibleError): - facts = copy.deepcopy(self.default_facts) - facts['openshift']['common']['short_version'] = '0.1' - facts['openshift']['common']['deployment_type'] = 'origin' - self.lookup.run(None, variables=facts) - - def test_unknown_ocp_version(self): - with pytest.raises(AnsibleError): - facts = copy.deepcopy(self.default_facts) - facts['openshift']['common']['short_version'] = '0.1' - facts['openshift']['common']['deployment_type'] = 'openshift-enterprise' - self.lookup.run(None, variables=facts) diff --git a/roles/openshift_master_facts/test/openshift_master_facts_default_predicates_tests.py b/roles/openshift_master_facts/test/openshift_master_facts_default_predicates_tests.py deleted file mode 100644 index 11aad9f03..000000000 --- a/roles/openshift_master_facts/test/openshift_master_facts_default_predicates_tests.py +++ /dev/null @@ -1,193 +0,0 @@ -import pytest - - -# Predicates ordered according to OpenShift Origin source: -# origin/vendor/k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go - -DEFAULT_PREDICATES_1_1 = [ - {'name': 'PodFitsHostPorts'}, - {'name': 'PodFitsResources'}, - {'name': 'NoDiskConflict'}, - {'name': 'MatchNodeSelector'}, -] - -DEFAULT_PREDICATES_1_2 = [ - {'name': 'PodFitsHostPorts'}, - {'name': 'PodFitsResources'}, - {'name': 'NoDiskConflict'}, - {'name': 'NoVolumeZoneConflict'}, - {'name': 'MatchNodeSelector'}, - {'name': 'MaxEBSVolumeCount'}, - {'name': 'MaxGCEPDVolumeCount'} -] - -DEFAULT_PREDICATES_1_3 = [ - {'name': 'NoDiskConflict'}, - {'name': 'NoVolumeZoneConflict'}, - {'name': 'MaxEBSVolumeCount'}, - {'name': 'MaxGCEPDVolumeCount'}, - {'name': 'GeneralPredicates'}, - {'name': 'PodToleratesNodeTaints'}, - {'name': 'CheckNodeMemoryPressure'} -] - -DEFAULT_PREDICATES_1_4 = [ - {'name': 'NoDiskConflict'}, - {'name': 'NoVolumeZoneConflict'}, - {'name': 'MaxEBSVolumeCount'}, - {'name': 'MaxGCEPDVolumeCount'}, - {'name': 'GeneralPredicates'}, - {'name': 'PodToleratesNodeTaints'}, - {'name': 'CheckNodeMemoryPressure'}, - {'name': 'CheckNodeDiskPressure'}, - {'name': 'MatchInterPodAffinity'} -] - -DEFAULT_PREDICATES_1_5 = [ - {'name': 'NoVolumeZoneConflict'}, - {'name': 'MaxEBSVolumeCount'}, - {'name': 'MaxGCEPDVolumeCount'}, - {'name': 'MatchInterPodAffinity'}, - {'name': 'NoDiskConflict'}, - {'name': 'GeneralPredicates'}, - {'name': 'PodToleratesNodeTaints'}, - {'name': 'CheckNodeMemoryPressure'}, - {'name': 'CheckNodeDiskPressure'}, -] - -DEFAULT_PREDICATES_3_6 = DEFAULT_PREDICATES_1_5 - -DEFAULT_PREDICATES_3_7 = [ - {'name': 'NoVolumeZoneConflict'}, - {'name': 'MaxEBSVolumeCount'}, - {'name': 'MaxGCEPDVolumeCount'}, - {'name': 'MaxAzureDiskVolumeCount'}, - {'name': 'MatchInterPodAffinity'}, - {'name': 'NoDiskConflict'}, - {'name': 'GeneralPredicates'}, - {'name': 'PodToleratesNodeTaints'}, - {'name': 'CheckNodeMemoryPressure'}, - {'name': 'CheckNodeDiskPressure'}, - {'name': 'NoVolumeNodeConflict'}, -] - -DEFAULT_PREDICATES_3_9 = DEFAULT_PREDICATES_3_8 = DEFAULT_PREDICATES_3_7 - -REGION_PREDICATE = { - 'name': 'Region', - 'argument': { - 'serviceAffinity': { - 'labels': ['region'] - } - } -} - -TEST_VARS = [ - ('1.1', 'origin', DEFAULT_PREDICATES_1_1), - ('3.1', 'openshift-enterprise', DEFAULT_PREDICATES_1_1), - ('1.2', 'origin', DEFAULT_PREDICATES_1_2), - ('3.2', 'openshift-enterprise', DEFAULT_PREDICATES_1_2), - ('1.3', 'origin', DEFAULT_PREDICATES_1_3), - ('3.3', 'openshift-enterprise', DEFAULT_PREDICATES_1_3), - ('1.4', 'origin', DEFAULT_PREDICATES_1_4), - ('3.4', 'openshift-enterprise', DEFAULT_PREDICATES_1_4), - ('1.5', 'origin', DEFAULT_PREDICATES_1_5), - ('3.5', 'openshift-enterprise', DEFAULT_PREDICATES_1_5), - ('3.6', 'origin', DEFAULT_PREDICATES_3_6), - ('3.6', 'openshift-enterprise', DEFAULT_PREDICATES_3_6), - ('3.7', 'origin', DEFAULT_PREDICATES_3_7), - ('3.7', 'openshift-enterprise', DEFAULT_PREDICATES_3_7), - ('3.8', 'origin', DEFAULT_PREDICATES_3_8), - ('3.8', 'openshift-enterprise', DEFAULT_PREDICATES_3_8), - ('3.9', 'origin', DEFAULT_PREDICATES_3_9), - ('3.9', 'openshift-enterprise', DEFAULT_PREDICATES_3_9), -] - - -def assert_ok(predicates_lookup, default_predicates, regions_enabled, **kwargs): - results = predicates_lookup.run(None, regions_enabled=regions_enabled, **kwargs) - if regions_enabled: - assert results == default_predicates + [REGION_PREDICATE] - else: - assert results == default_predicates - - -def test_openshift_version(predicates_lookup, openshift_version_fixture, regions_enabled): - facts, default_predicates = openshift_version_fixture - assert_ok(predicates_lookup, default_predicates, variables=facts, regions_enabled=regions_enabled) - - -@pytest.fixture(params=TEST_VARS) -def openshift_version_fixture(request, facts): - version, deployment_type, default_predicates = request.param - version += '.1' - facts['openshift_version'] = version - facts['openshift']['common']['deployment_type'] = deployment_type - return facts, default_predicates - - -def test_openshift_release(predicates_lookup, openshift_release_fixture, regions_enabled): - facts, default_predicates = openshift_release_fixture - assert_ok(predicates_lookup, default_predicates, variables=facts, regions_enabled=regions_enabled) - - -@pytest.fixture(params=TEST_VARS) -def openshift_release_fixture(request, facts, release_mod): - release, deployment_type, default_predicates = request.param - facts['openshift_release'] = release_mod(release) - facts['openshift']['common']['deployment_type'] = deployment_type - return facts, default_predicates - - -def test_short_version(predicates_lookup, short_version_fixture, regions_enabled): - facts, default_predicates = short_version_fixture - assert_ok(predicates_lookup, default_predicates, variables=facts, regions_enabled=regions_enabled) - - -@pytest.fixture(params=TEST_VARS) -def short_version_fixture(request, facts): - short_version, deployment_type, default_predicates = request.param - facts['openshift']['common']['short_version'] = short_version - facts['openshift']['common']['deployment_type'] = deployment_type - return facts, default_predicates - - -def test_short_version_kwarg(predicates_lookup, short_version_kwarg_fixture, regions_enabled): - facts, short_version, default_predicates = short_version_kwarg_fixture - assert_ok( - predicates_lookup, default_predicates, variables=facts, - regions_enabled=regions_enabled, short_version=short_version) - - -@pytest.fixture(params=TEST_VARS) -def short_version_kwarg_fixture(request, facts): - short_version, deployment_type, default_predicates = request.param - facts['openshift']['common']['deployment_type'] = deployment_type - return facts, short_version, default_predicates - - -def test_deployment_type_kwarg(predicates_lookup, deployment_type_kwarg_fixture, regions_enabled): - facts, deployment_type, default_predicates = deployment_type_kwarg_fixture - assert_ok( - predicates_lookup, default_predicates, variables=facts, - regions_enabled=regions_enabled, deployment_type=deployment_type) - - -@pytest.fixture(params=TEST_VARS) -def deployment_type_kwarg_fixture(request, facts): - short_version, deployment_type, default_predicates = request.param - facts['openshift']['common']['short_version'] = short_version - return facts, deployment_type, default_predicates - - -def test_short_version_deployment_type_kwargs( - predicates_lookup, short_version_deployment_type_kwargs_fixture, regions_enabled): - short_version, deployment_type, default_predicates = short_version_deployment_type_kwargs_fixture - assert_ok( - predicates_lookup, default_predicates, regions_enabled=regions_enabled, - short_version=short_version, deployment_type=deployment_type) - - -@pytest.fixture(params=TEST_VARS) -def short_version_deployment_type_kwargs_fixture(request): - return request.param diff --git a/roles/openshift_master_facts/test/openshift_master_facts_default_priorities_tests.py b/roles/openshift_master_facts/test/openshift_master_facts_default_priorities_tests.py deleted file mode 100644 index 527fc9ff4..000000000 --- a/roles/openshift_master_facts/test/openshift_master_facts_default_priorities_tests.py +++ /dev/null @@ -1,167 +0,0 @@ -import pytest - - -DEFAULT_PRIORITIES_1_1 = [ - {'name': 'LeastRequestedPriority', 'weight': 1}, - {'name': 'BalancedResourceAllocation', 'weight': 1}, - {'name': 'SelectorSpreadPriority', 'weight': 1} -] - -DEFAULT_PRIORITIES_1_2 = [ - {'name': 'LeastRequestedPriority', 'weight': 1}, - {'name': 'BalancedResourceAllocation', 'weight': 1}, - {'name': 'SelectorSpreadPriority', 'weight': 1}, - {'name': 'NodeAffinityPriority', 'weight': 1} -] - -DEFAULT_PRIORITIES_1_3 = [ - {'name': 'LeastRequestedPriority', 'weight': 1}, - {'name': 'BalancedResourceAllocation', 'weight': 1}, - {'name': 'SelectorSpreadPriority', 'weight': 1}, - {'name': 'NodeAffinityPriority', 'weight': 1}, - {'name': 'TaintTolerationPriority', 'weight': 1} -] - -DEFAULT_PRIORITIES_1_4 = [ - {'name': 'LeastRequestedPriority', 'weight': 1}, - {'name': 'BalancedResourceAllocation', 'weight': 1}, - {'name': 'SelectorSpreadPriority', 'weight': 1}, - {'name': 'NodePreferAvoidPodsPriority', 'weight': 10000}, - {'name': 'NodeAffinityPriority', 'weight': 1}, - {'name': 'TaintTolerationPriority', 'weight': 1}, - {'name': 'InterPodAffinityPriority', 'weight': 1} -] - -DEFAULT_PRIORITIES_1_5 = [ - {'name': 'SelectorSpreadPriority', 'weight': 1}, - {'name': 'InterPodAffinityPriority', 'weight': 1}, - {'name': 'LeastRequestedPriority', 'weight': 1}, - {'name': 'BalancedResourceAllocation', 'weight': 1}, - {'name': 'NodePreferAvoidPodsPriority', 'weight': 10000}, - {'name': 'NodeAffinityPriority', 'weight': 1}, - {'name': 'TaintTolerationPriority', 'weight': 1} -] - -DEFAULT_PRIORITIES_3_6 = DEFAULT_PRIORITIES_1_5 - -DEFAULT_PRIORITIES_3_9 = DEFAULT_PRIORITIES_3_8 = DEFAULT_PRIORITIES_3_7 = DEFAULT_PRIORITIES_3_6 - -ZONE_PRIORITY = { - 'name': 'Zone', - 'argument': { - 'serviceAntiAffinity': { - 'label': 'zone' - } - }, - 'weight': 2 -} - -TEST_VARS = [ - ('1.1', 'origin', DEFAULT_PRIORITIES_1_1), - ('3.1', 'openshift-enterprise', DEFAULT_PRIORITIES_1_1), - ('1.2', 'origin', DEFAULT_PRIORITIES_1_2), - ('3.2', 'openshift-enterprise', DEFAULT_PRIORITIES_1_2), - ('1.3', 'origin', DEFAULT_PRIORITIES_1_3), - ('3.3', 'openshift-enterprise', DEFAULT_PRIORITIES_1_3), - ('1.4', 'origin', DEFAULT_PRIORITIES_1_4), - ('3.4', 'openshift-enterprise', DEFAULT_PRIORITIES_1_4), - ('1.5', 'origin', DEFAULT_PRIORITIES_1_5), - ('3.5', 'openshift-enterprise', DEFAULT_PRIORITIES_1_5), - ('3.6', 'origin', DEFAULT_PRIORITIES_3_6), - ('3.6', 'openshift-enterprise', DEFAULT_PRIORITIES_3_6), - ('3.7', 'origin', DEFAULT_PRIORITIES_3_7), - ('3.7', 'openshift-enterprise', DEFAULT_PRIORITIES_3_7), - ('3.8', 'origin', DEFAULT_PRIORITIES_3_8), - ('3.8', 'openshift-enterprise', DEFAULT_PRIORITIES_3_8), - ('3.9', 'origin', DEFAULT_PRIORITIES_3_9), - ('3.9', 'openshift-enterprise', DEFAULT_PRIORITIES_3_9), -] - - -def assert_ok(priorities_lookup, default_priorities, zones_enabled, **kwargs): - results = priorities_lookup.run(None, zones_enabled=zones_enabled, **kwargs) - if zones_enabled: - assert results == default_priorities + [ZONE_PRIORITY] - else: - assert results == default_priorities - - -def test_openshift_version(priorities_lookup, openshift_version_fixture, zones_enabled): - facts, default_priorities = openshift_version_fixture - assert_ok(priorities_lookup, default_priorities, variables=facts, zones_enabled=zones_enabled) - - -@pytest.fixture(params=TEST_VARS) -def openshift_version_fixture(request, facts): - version, deployment_type, default_priorities = request.param - version += '.1' - facts['openshift_version'] = version - facts['openshift']['common']['deployment_type'] = deployment_type - return facts, default_priorities - - -def test_openshift_release(priorities_lookup, openshift_release_fixture, zones_enabled): - facts, default_priorities = openshift_release_fixture - assert_ok(priorities_lookup, default_priorities, variables=facts, zones_enabled=zones_enabled) - - -@pytest.fixture(params=TEST_VARS) -def openshift_release_fixture(request, facts, release_mod): - release, deployment_type, default_priorities = request.param - facts['openshift_release'] = release_mod(release) - facts['openshift']['common']['deployment_type'] = deployment_type - return facts, default_priorities - - -def test_short_version(priorities_lookup, short_version_fixture, zones_enabled): - facts, default_priorities = short_version_fixture - assert_ok(priorities_lookup, default_priorities, variables=facts, zones_enabled=zones_enabled) - - -@pytest.fixture(params=TEST_VARS) -def short_version_fixture(request, facts): - short_version, deployment_type, default_priorities = request.param - facts['openshift']['common']['short_version'] = short_version - facts['openshift']['common']['deployment_type'] = deployment_type - return facts, default_priorities - - -def test_short_version_kwarg(priorities_lookup, short_version_kwarg_fixture, zones_enabled): - facts, short_version, default_priorities = short_version_kwarg_fixture - assert_ok( - priorities_lookup, default_priorities, variables=facts, - zones_enabled=zones_enabled, short_version=short_version) - - -@pytest.fixture(params=TEST_VARS) -def short_version_kwarg_fixture(request, facts): - short_version, deployment_type, default_priorities = request.param - facts['openshift']['common']['deployment_type'] = deployment_type - return facts, short_version, default_priorities - - -def test_deployment_type_kwarg(priorities_lookup, deployment_type_kwarg_fixture, zones_enabled): - facts, deployment_type, default_priorities = deployment_type_kwarg_fixture - assert_ok( - priorities_lookup, default_priorities, variables=facts, - zones_enabled=zones_enabled, deployment_type=deployment_type) - - -@pytest.fixture(params=TEST_VARS) -def deployment_type_kwarg_fixture(request, facts): - short_version, deployment_type, default_priorities = request.param - facts['openshift']['common']['short_version'] = short_version - return facts, deployment_type, default_priorities - - -def test_short_version_deployment_type_kwargs( - priorities_lookup, short_version_deployment_type_kwargs_fixture, zones_enabled): - short_version, deployment_type, default_priorities = short_version_deployment_type_kwargs_fixture - assert_ok( - priorities_lookup, default_priorities, zones_enabled=zones_enabled, - short_version=short_version, deployment_type=deployment_type) - - -@pytest.fixture(params=TEST_VARS) -def short_version_deployment_type_kwargs_fixture(request): - return request.param diff --git a/roles/openshift_named_certificates/filter_plugins/openshift_named_certificates.py b/roles/openshift_named_certificates/filter_plugins/openshift_named_certificates.py deleted file mode 100644 index 6ed6d404c..000000000 --- a/roles/openshift_named_certificates/filter_plugins/openshift_named_certificates.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -''' -Custom filters for use with openshift named certificates -''' - - -class FilterModule(object): - ''' Custom ansible filters for use with openshift named certificates''' - - @staticmethod - def oo_named_certificates_list(named_certificates): - ''' Returns named certificates list with correct fields for the master - config file.''' - return [{'certFile': named_certificate['certfile'], - 'keyFile': named_certificate['keyfile'], - 'names': named_certificate['names']} for named_certificate in named_certificates] - - def filters(self): - ''' returns a mapping of filters to methods ''' - return {"oo_named_certificates_list": self.oo_named_certificates_list} diff --git a/roles/openshift_persistent_volumes/action_plugins/generate_pv_pvcs_list.py b/roles/openshift_persistent_volumes/action_plugins/generate_pv_pvcs_list.py deleted file mode 100644 index eb13a58ba..000000000 --- a/roles/openshift_persistent_volumes/action_plugins/generate_pv_pvcs_list.py +++ /dev/null @@ -1,157 +0,0 @@ -""" -Ansible action plugin to generate pv and pvc dictionaries lists -""" - -from ansible.plugins.action import ActionBase -from ansible import errors - - -class ActionModule(ActionBase): - """Action plugin to execute health checks.""" - - def get_templated(self, var_to_template): - """Return a properly templated ansible variable""" - return self._templar.template(self.task_vars.get(var_to_template)) - - def build_common(self, varname=None): - """Retrieve common variables for each pv and pvc type""" - volume = self.get_templated(str(varname) + '_volume_name') - size = self.get_templated(str(varname) + '_volume_size') - labels = self.task_vars.get(str(varname) + '_labels') - if labels: - labels = self._templar.template(labels) - else: - labels = dict() - access_modes = self.get_templated(str(varname) + '_access_modes') - return (volume, size, labels, access_modes) - - def build_pv_nfs(self, varname=None): - """Build pv dictionary for nfs storage type""" - host = self.task_vars.get(str(varname) + '_host') - if host: - self._templar.template(host) - elif host is None: - groups = self.task_vars.get('groups') - default_group_name = self.get_templated('openshift_persistent_volumes_default_nfs_group') - if groups and default_group_name and default_group_name in groups and len(groups[default_group_name]) > 0: - host = groups['oo_nfs_to_config'][0] - else: - raise errors.AnsibleModuleError("|failed no storage host detected") - volume, size, labels, access_modes = self.build_common(varname=varname) - directory = self.get_templated(str(varname) + '_nfs_directory') - path = directory + '/' + volume - return dict( - name="{0}-volume".format(volume), - capacity=size, - labels=labels, - access_modes=access_modes, - storage=dict( - nfs=dict( - server=host, - path=path))) - - def build_pv_openstack(self, varname=None): - """Build pv dictionary for openstack storage type""" - volume, size, labels, access_modes = self.build_common(varname=varname) - filesystem = self.get_templated(str(varname) + '_openstack_filesystem') - volume_id = self.get_templated(str(varname) + '_openstack_volumeID') - return dict( - name="{0}-volume".format(volume), - capacity=size, - labels=labels, - access_modes=access_modes, - storage=dict( - cinder=dict( - fsType=filesystem, - volumeID=volume_id))) - - def build_pv_glusterfs(self, varname=None): - """Build pv dictionary for glusterfs storage type""" - volume, size, labels, access_modes = self.build_common(varname=varname) - endpoints = self.get_templated(str(varname) + '_glusterfs_endpoints') - path = self.get_templated(str(varname) + '_glusterfs_path') - read_only = self.get_templated(str(varname) + '_glusterfs_readOnly') - return dict( - name="{0}-volume".format(volume), - capacity=size, - labels=labels, - access_modes=access_modes, - storage=dict( - glusterfs=dict( - endpoints=endpoints, - path=path, - readOnly=read_only))) - - def build_pv_dict(self, varname=None): - """Check for the existence of PV variables""" - kind = self.task_vars.get(str(varname) + '_kind') - if kind: - kind = self._templar.template(kind) - create_pv = self.task_vars.get(str(varname) + '_create_pv') - if create_pv and self._templar.template(create_pv): - if kind == 'nfs': - return self.build_pv_nfs(varname=varname) - - elif kind == 'openstack': - return self.build_pv_openstack(varname=varname) - - elif kind == 'glusterfs': - return self.build_pv_glusterfs(varname=varname) - - elif not (kind == 'object' or kind == 'dynamic'): - msg = "|failed invalid storage kind '{0}' for component '{1}'".format( - kind, - varname) - raise errors.AnsibleModuleError(msg) - return None - - def build_pvc_dict(self, varname=None): - """Check for the existence of PVC variables""" - kind = self.task_vars.get(str(varname) + '_kind') - if kind: - kind = self._templar.template(kind) - create_pv = self.task_vars.get(str(varname) + '_create_pv') - if create_pv: - create_pv = self._templar.template(create_pv) - create_pvc = self.task_vars.get(str(varname) + '_create_pvc') - if create_pvc: - create_pvc = self._templar.template(create_pvc) - if kind != 'object' and create_pv and create_pvc: - volume, size, _, access_modes = self.build_common(varname=varname) - return dict( - name="{0}-claim".format(volume), - capacity=size, - access_modes=access_modes) - return None - - def run(self, tmp=None, task_vars=None): - """Run generate_pv_pvcs_list action plugin""" - result = super(ActionModule, self).run(tmp, task_vars) - # Ignore settting self.task_vars outside of init. - # pylint: disable=W0201 - self.task_vars = task_vars or {} - - result["changed"] = False - result["failed"] = False - result["msg"] = "persistent_volumes list and persistent_volume_claims list created" - vars_to_check = ['openshift_hosted_registry_storage', - 'openshift_hosted_router_storage', - 'openshift_hosted_etcd_storage', - 'openshift_logging_storage', - 'openshift_loggingops_storage', - 'openshift_metrics_storage', - 'openshift_prometheus_storage', - 'openshift_prometheus_alertmanager_storage', - 'openshift_prometheus_alertbuffer_storage'] - persistent_volumes = [] - persistent_volume_claims = [] - for varname in vars_to_check: - pv_dict = self.build_pv_dict(varname) - if pv_dict: - persistent_volumes.append(pv_dict) - pvc_dict = self.build_pvc_dict(varname) - if pvc_dict: - persistent_volume_claims.append(pvc_dict) - result["persistent_volumes"] = persistent_volumes - result["persistent_volume_claims"] = persistent_volume_claims - return result diff --git a/roles/openshift_persistent_volumes/tasks/main.yml b/roles/openshift_persistent_volumes/tasks/main.yml index 0b4dd7d1f..b1d9c8cca 100644 --- a/roles/openshift_persistent_volumes/tasks/main.yml +++ b/roles/openshift_persistent_volumes/tasks/main.yml @@ -26,7 +26,8 @@ when: openshift_hosted_registry_storage_glusterfs_swap | default(False) - name: create standard pv and pvc lists - # generate_pv_pvcs_list is a custom action module defined in ../action_plugins + # generate_pv_pvcs_list is a custom action module defined in + # roles/lib_utils/action_plugins/generate_pv_pvcs_list.py generate_pv_pvcs_list: {} register: l_pv_pvcs_list diff --git a/roles/openshift_sanitize_inventory/filter_plugins/openshift_sanitize_inventory.py b/roles/openshift_sanitize_inventory/filter_plugins/openshift_sanitize_inventory.py index 72c47b8ee..14f1f72c2 100644 --- a/roles/openshift_sanitize_inventory/filter_plugins/openshift_sanitize_inventory.py +++ b/roles/openshift_sanitize_inventory/filter_plugins/openshift_sanitize_inventory.py @@ -6,15 +6,6 @@ import re -# This should be removed after map_from_pairs is no longer used in __deprecations_logging.yml -def map_from_pairs(source, delim="="): - ''' Returns a dict given the source and delim delimited ''' - if source == '': - return dict() - - return dict(item.split(delim) for item in source.split(",")) - - def vars_with_pattern(source, pattern=""): ''' Returns a list of variables whose name matches the given pattern ''' if source == '': @@ -39,6 +30,5 @@ class FilterModule(object): def filters(self): ''' Returns the names of the filters provided by this class ''' return { - 'map_from_pairs': map_from_pairs, 'vars_with_pattern': vars_with_pattern } diff --git a/roles/openshift_storage_glusterfs/filter_plugins/openshift_storage_glusterfs.py b/roles/openshift_storage_glusterfs/filter_plugins/openshift_storage_glusterfs.py deleted file mode 100644 index a86c96df7..000000000 --- a/roles/openshift_storage_glusterfs/filter_plugins/openshift_storage_glusterfs.py +++ /dev/null @@ -1,23 +0,0 @@ -''' - Openshift Storage GlusterFS class that provides useful filters used in GlusterFS -''' - - -def map_from_pairs(source, delim="="): - ''' Returns a dict given the source and delim delimited ''' - if source == '': - return dict() - - return dict(item.split(delim) for item in source.split(",")) - - -# pylint: disable=too-few-public-methods -class FilterModule(object): - ''' OpenShift Storage GlusterFS Filters ''' - - # pylint: disable=no-self-use, too-few-public-methods - def filters(self): - ''' Returns the names of the filters provided by this class ''' - return { - 'map_from_pairs': map_from_pairs - } diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml index 2ea7286f3..a374df0ce 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml @@ -4,6 +4,7 @@ glusterfs_namespace: "{{ openshift_storage_glusterfs_namespace }}" glusterfs_is_native: "{{ openshift_storage_glusterfs_is_native | bool }}" glusterfs_name: "{{ openshift_storage_glusterfs_name }}" + # map_from_pairs is a custom filter plugin in role lib_utils glusterfs_nodeselector: "{{ openshift_storage_glusterfs_nodeselector | default(['storagenode', openshift_storage_glusterfs_name] | join('=')) | map_from_pairs }}" glusterfs_use_default_selector: "{{ openshift_storage_glusterfs_use_default_selector }}" glusterfs_storageclass: "{{ openshift_storage_glusterfs_storageclass }}" diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml index b7cff6514..544a6f491 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml @@ -4,6 +4,7 @@ glusterfs_namespace: "{{ openshift_storage_glusterfs_registry_namespace }}" glusterfs_is_native: "{{ openshift_storage_glusterfs_registry_is_native | bool }}" glusterfs_name: "{{ openshift_storage_glusterfs_registry_name }}" + # map_from_pairs is a custom filter plugin in role lib_utils glusterfs_nodeselector: "{{ openshift_storage_glusterfs_registry_nodeselector | default(['storagenode', openshift_storage_glusterfs_registry_name] | join('=')) | map_from_pairs }}" glusterfs_use_default_selector: "{{ openshift_storage_glusterfs_registry_use_default_selector }}" glusterfs_storageclass: "{{ openshift_storage_glusterfs_registry_storageclass }}" -- cgit v1.2.3