diff options
Diffstat (limited to 'roles/lib_utils')
23 files changed, 4630 insertions, 0 deletions
diff --git a/roles/lib_utils/library/iam_cert23.py b/roles/lib_utils/library/iam_cert23.py new file mode 100644 index 000000000..07b3d3bdf --- /dev/null +++ b/roles/lib_utils/library/iam_cert23.py @@ -0,0 +1,314 @@ +#!/usr/bin/python +# pylint: skip-file +# flake8: noqa +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + + +DOCUMENTATION = ''' +--- +module: iam_cert +short_description: Manage server certificates for use on ELBs and CloudFront +description: + - Allows for the management of server certificates +version_added: "2.0" +options: + name: + description: + - Name of certificate to add, update or remove. + required: true + new_name: + description: + - When state is present, this will update the name of the cert. + - The cert, key and cert_chain parameters will be ignored if this is defined. + new_path: + description: + - When state is present, this will update the path of the cert. + - The cert, key and cert_chain parameters will be ignored if this is defined. + state: + description: + - Whether to create(or update) or delete certificate. + - If new_path or new_name is defined, specifying present will attempt to make an update these. + required: true + choices: [ "present", "absent" ] + path: + description: + - When creating or updating, specify the desired path of the certificate. + default: "/" + cert_chain: + description: + - The path to, or content of the CA certificate chain in PEM encoded format. + As of 2.4 content is accepted. If the parameter is not a file, it is assumed to be content. + cert: + description: + - The path to, or content of the certificate body in PEM encoded format. + As of 2.4 content is accepted. If the parameter is not a file, it is assumed to be content. + key: + description: + - The path to, or content of the private key in PEM encoded format. + As of 2.4 content is accepted. If the parameter is not a file, it is assumed to be content. + dup_ok: + description: + - By default the module will not upload a certificate that is already uploaded into AWS. + If set to True, it will upload the certificate as long as the name is unique. + default: False + + +requirements: [ "boto" ] +author: Jonathan I. Davila +extends_documentation_fragment: + - aws + - ec2 +''' + +EXAMPLES = ''' +# Basic server certificate upload from local file +- iam_cert: + name: very_ssl + state: present + cert: "{{ lookup('file', 'path/to/cert') }}" + key: "{{ lookup('file', 'path/to/key') }}" + cert_chain: "{{ lookup('file', 'path/to/certchain') }}" + +# Basic server certificate upload +- iam_cert: + name: very_ssl + state: present + cert: path/to/cert + key: path/to/key + cert_chain: path/to/certchain + +# Server certificate upload using key string +- iam_cert: + name: very_ssl + state: present + path: "/a/cert/path/" + cert: body_of_somecert + key: vault_body_of_privcertkey + cert_chain: body_of_myverytrustedchain + +# Basic rename of existing certificate +- iam_cert: + name: very_ssl + new_name: new_very_ssl + state: present + +''' +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.ec2 import ec2_argument_spec, get_aws_connection_info, connect_to_aws +import os + +try: + import boto + import boto.iam + import boto.ec2 + HAS_BOTO = True +except ImportError: + HAS_BOTO = False + + +def boto_exception(err): + '''generic error message handler''' + if hasattr(err, 'error_message'): + error = err.error_message + elif hasattr(err, 'message'): + error = err.message + else: + error = '%s: %s' % (Exception, err) + + return error + + +def cert_meta(iam, name): + certificate = iam.get_server_certificate(name).get_server_certificate_result.server_certificate + ocert = certificate.certificate_body + opath = certificate.server_certificate_metadata.path + ocert_id = certificate.server_certificate_metadata.server_certificate_id + upload_date = certificate.server_certificate_metadata.upload_date + exp = certificate.server_certificate_metadata.expiration + arn = certificate.server_certificate_metadata.arn + return opath, ocert, ocert_id, upload_date, exp, arn + + +def dup_check(module, iam, name, new_name, cert, orig_cert_names, orig_cert_bodies, dup_ok): + update = False + + # IAM cert names are case insensitive + names_lower = [n.lower() for n in [name, new_name] if n is not None] + orig_cert_names_lower = [ocn.lower() for ocn in orig_cert_names] + + if any(ct in orig_cert_names_lower for ct in names_lower): + for i_name in names_lower: + if cert is not None: + try: + c_index = orig_cert_names_lower.index(i_name) + except NameError: + continue + else: + # NOTE: remove the carriage return to strictly compare the cert bodies. + slug_cert = cert.replace('\r', '') + slug_orig_cert_bodies = orig_cert_bodies[c_index].replace('\r', '') + if slug_orig_cert_bodies == slug_cert: + update = True + break + elif slug_cert.startswith(slug_orig_cert_bodies): + update = True + break + elif slug_orig_cert_bodies != slug_cert: + module.fail_json(changed=False, msg='A cert with the name %s already exists and' + ' has a different certificate body associated' + ' with it. Certificates cannot have the same name' % orig_cert_names[c_index]) + else: + update = True + break + elif cert in orig_cert_bodies and not dup_ok: + for crt_name, crt_body in zip(orig_cert_names, orig_cert_bodies): + if crt_body == cert: + module.fail_json(changed=False, msg='This certificate already' + ' exists under the name %s' % crt_name) + + return update + + +def cert_action(module, iam, name, cpath, new_name, new_path, state, + cert, key, cert_chain, orig_cert_names, orig_cert_bodies, dup_ok): + if state == 'present': + update = dup_check(module, iam, name, new_name, cert, orig_cert_names, + orig_cert_bodies, dup_ok) + if update: + opath, ocert, ocert_id, upload_date, exp, arn = cert_meta(iam, name) + changed = True + if new_name and new_path: + iam.update_server_cert(name, new_cert_name=new_name, new_path=new_path) + module.exit_json(changed=changed, original_name=name, new_name=new_name, + original_path=opath, new_path=new_path, cert_body=ocert, + upload_date=upload_date, expiration_date=exp, arn=arn) + elif new_name and not new_path: + iam.update_server_cert(name, new_cert_name=new_name) + module.exit_json(changed=changed, original_name=name, new_name=new_name, + cert_path=opath, cert_body=ocert, + upload_date=upload_date, expiration_date=exp, arn=arn) + elif not new_name and new_path: + iam.update_server_cert(name, new_path=new_path) + module.exit_json(changed=changed, name=new_name, + original_path=opath, new_path=new_path, cert_body=ocert, + upload_date=upload_date, expiration_date=exp, arn=arn) + else: + changed = False + module.exit_json(changed=changed, name=name, cert_path=opath, cert_body=ocert, + upload_date=upload_date, expiration_date=exp, arn=arn, + msg='No new path or name specified. No changes made') + else: + changed = True + iam.upload_server_cert(name, cert, key, cert_chain=cert_chain, path=cpath) + opath, ocert, ocert_id, upload_date, exp, arn = cert_meta(iam, name) + module.exit_json(changed=changed, name=name, cert_path=opath, cert_body=ocert, + upload_date=upload_date, expiration_date=exp, arn=arn) + elif state == 'absent': + if name in orig_cert_names: + changed = True + iam.delete_server_cert(name) + module.exit_json(changed=changed, deleted_cert=name) + else: + changed = False + module.exit_json(changed=changed, msg='Certificate with the name %s already absent' % name) + + +def load_data(cert, key, cert_chain): + # if paths are provided rather than lookups read the files and return the contents + if cert and os.path.isfile(cert): + cert = open(cert, 'r').read().rstrip() + if key and os.path.isfile(key): + key = open(key, 'r').read().rstrip() + if cert_chain and os.path.isfile(cert_chain): + cert_chain = open(cert_chain, 'r').read() + return cert, key, cert_chain + + +def main(): + argument_spec = ec2_argument_spec() + argument_spec.update(dict( + state=dict(required=True, choices=['present', 'absent']), + name=dict(), + cert=dict(), + key=dict(no_log=True), + cert_chain=dict(), + new_name=dict(), + path=dict(default='/'), + new_path=dict(), + dup_ok=dict(type='bool') + ) + ) + + module = AnsibleModule( + argument_spec=argument_spec, + mutually_exclusive=[ + ['new_path', 'key'], + ['new_path', 'cert'], + ['new_path', 'cert_chain'], + ['new_name', 'key'], + ['new_name', 'cert'], + ['new_name', 'cert_chain'], + ], + ) + + if not HAS_BOTO: + module.fail_json(msg="Boto is required for this module") + + region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module) + + try: + if region: + iam = connect_to_aws(boto.iam, region, **aws_connect_kwargs) + else: + iam = boto.iam.connection.IAMConnection(**aws_connect_kwargs) + except boto.exception.NoAuthHandlerFound as e: + module.fail_json(msg=str(e)) + + state = module.params.get('state') + name = module.params.get('name') + path = module.params.get('path') + new_name = module.params.get('new_name') + new_path = module.params.get('new_path') + dup_ok = module.params.get('dup_ok') + if state == 'present' and not new_name and not new_path: + cert, key, cert_chain = load_data(cert=module.params.get('cert'), + key=module.params.get('key'), + cert_chain=module.params.get('cert_chain')) + else: + cert = key = cert_chain = None + + orig_cert_names = [ctb['server_certificate_name'] for ctb in + iam.get_all_server_certs().list_server_certificates_result.server_certificate_metadata_list] + orig_cert_bodies = [iam.get_server_certificate(thing).get_server_certificate_result.certificate_body + for thing in orig_cert_names] + if new_name == name: + new_name = None + if new_path == path: + new_path = None + + changed = False + try: + cert_action(module, iam, name, path, new_name, new_path, state, + cert, key, cert_chain, orig_cert_names, orig_cert_bodies, dup_ok) + except boto.exception.BotoServerError as err: + module.fail_json(changed=changed, msg=str(err), debug=[cert, key]) + + +if __name__ == '__main__': + main() diff --git a/roles/lib_utils/library/oo_iam_kms.py b/roles/lib_utils/library/oo_iam_kms.py new file mode 100644 index 000000000..c85745f01 --- /dev/null +++ b/roles/lib_utils/library/oo_iam_kms.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python +''' +ansible module for creating AWS IAM KMS keys +''' +# vim: expandtab:tabstop=4:shiftwidth=4 +# +# AWS IAM KMS ansible module +# +# +# Copyright 2016 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Jenkins environment doesn't have all the required libraries +# pylint: disable=import-error +import time +import boto3 +# Ansible modules need this wildcard import +# pylint: disable=unused-wildcard-import, wildcard-import, redefined-builtin +from ansible.module_utils.basic import AnsibleModule + +AWS_ALIAS_URL = "http://docs.aws.amazon.com/kms/latest/developerguide/programming-aliases.html" + + +class AwsIamKms(object): + ''' + ansible module for AWS IAM KMS + ''' + + def __init__(self): + ''' constructor ''' + self.module = None + self.kms_client = None + self.aliases = None + + @staticmethod + def valid_alias_name(user_alias): + ''' AWS KMS aliases must start with 'alias/' ''' + valid_start = 'alias/' + if user_alias.startswith(valid_start): + return True + + return False + + def get_all_kms_info(self): + '''fetch all kms info and return them + + list_keys doesn't have information regarding aliases + list_aliases doesn't have the full kms arn + + fetch both and join them on the targetKeyId + ''' + aliases = self.kms_client.list_aliases()['Aliases'] + keys = self.kms_client.list_keys()['Keys'] + + for alias in aliases: + for key in keys: + if 'TargetKeyId' in alias and 'KeyId' in key: + if alias['TargetKeyId'] == key['KeyId']: + alias.update(key) + + return aliases + + def get_kms_entry(self, user_alias, alias_list): + ''' return single alias details from list of aliases ''' + for alias in alias_list: + if user_alias == alias.get('AliasName', False): + return alias + + msg = "Did not find alias {}".format(user_alias) + self.module.exit_json(failed=True, results=msg) + + @staticmethod + def exists(user_alias, alias_list): + ''' Check if KMS alias already exists ''' + for alias in alias_list: + if user_alias == alias.get('AliasName'): + return True + + return False + + def main(self): + ''' entry point for module ''' + + self.module = AnsibleModule( + argument_spec=dict( + state=dict(default='list', choices=['list', 'present'], type='str'), + region=dict(default=None, required=True, type='str'), + alias=dict(default=None, type='str'), + # description default cannot be None + description=dict(default='', type='str'), + aws_access_key=dict(default=None, type='str'), + aws_secret_key=dict(default=None, type='str'), + ), + ) + + state = self.module.params['state'] + aws_access_key = self.module.params['aws_access_key'] + aws_secret_key = self.module.params['aws_secret_key'] + if aws_access_key and aws_secret_key: + boto3.setup_default_session(aws_access_key_id=aws_access_key, + aws_secret_access_key=aws_secret_key, + region_name=self.module.params['region']) + else: + boto3.setup_default_session(region_name=self.module.params['region']) + + self.kms_client = boto3.client('kms') + + aliases = self.get_all_kms_info() + + if state == 'list': + if self.module.params['alias'] is not None: + user_kms = self.get_kms_entry(self.module.params['alias'], + aliases) + self.module.exit_json(changed=False, results=user_kms, + state="list") + else: + self.module.exit_json(changed=False, results=aliases, + state="list") + + if state == 'present': + + # early sanity check to make sure the alias name conforms with + # AWS alias name requirements + if not self.valid_alias_name(self.module.params['alias']): + self.module.exit_json(failed=True, changed=False, + results="Alias must start with the prefix " + + "'alias/'. Please see " + AWS_ALIAS_URL, + state='present') + + if not self.exists(self.module.params['alias'], aliases): + # if we didn't find it, create it + response = self.kms_client.create_key(KeyUsage='ENCRYPT_DECRYPT', + Description=self.module.params['description']) + kid = response['KeyMetadata']['KeyId'] + response = self.kms_client.create_alias(AliasName=self.module.params['alias'], + TargetKeyId=kid) + # sleep for a bit so that the KMS data can be queried + time.sleep(10) + # get details for newly created KMS entry + new_alias_list = self.kms_client.list_aliases()['Aliases'] + user_kms = self.get_kms_entry(self.module.params['alias'], + new_alias_list) + + self.module.exit_json(changed=True, results=user_kms, + state='present') + + # already exists, normally we would check whether we need to update it + # but this module isn't written to allow changing the alias name + # or changing whether the key is enabled/disabled + user_kms = self.get_kms_entry(self.module.params['alias'], aliases) + self.module.exit_json(changed=False, results=user_kms, + state="present") + + self.module.exit_json(failed=True, + changed=False, + results='Unknown state passed. %s' % state, + state="unknown") + + +if __name__ == '__main__': + AwsIamKms().main() diff --git a/roles/lib_utils/library/repoquery.py b/roles/lib_utils/library/repoquery.py new file mode 100644 index 000000000..e5ac1f74f --- /dev/null +++ b/roles/lib_utils/library/repoquery.py @@ -0,0 +1,643 @@ +#!/usr/bin/env python +# pylint: disable=missing-docstring +# ___ ___ _ _ ___ ___ _ _____ ___ ___ +# / __| __| \| | __| _ \ /_\_ _| __| \ +# | (_ | _|| .` | _|| / / _ \| | | _|| |) | +# \___|___|_|\_|___|_|_\/_/_\_\_|_|___|___/_ _____ +# | \ / _ \ | \| |/ _ \_ _| | __| \_ _|_ _| +# | |) | (_) | | .` | (_) || | | _|| |) | | | | +# |___/ \___/ |_|\_|\___/ |_| |___|___/___| |_| +# +# Copyright 2016 Red Hat, Inc. and/or its affiliates +# and other contributors as indicated by the @author tags. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- -*- -*- Begin included fragment: lib/import.py -*- -*- -*- + +# pylint: disable=wrong-import-order,wrong-import-position,unused-import + +from __future__ import print_function # noqa: F401 +import copy # noqa: F401 +import json # noqa: F401 +import os # noqa: F401 +import re # noqa: F401 +import shutil # noqa: F401 +import tempfile # noqa: F401 +import time # noqa: F401 + +try: + import ruamel.yaml as yaml # noqa: F401 +except ImportError: + import yaml # noqa: F401 + +from ansible.module_utils.basic import AnsibleModule + +# -*- -*- -*- End included fragment: lib/import.py -*- -*- -*- + +# -*- -*- -*- Begin included fragment: doc/repoquery -*- -*- -*- + +DOCUMENTATION = ''' +--- +module: repoquery +short_description: Query package information from Yum repositories +description: + - Query package information from Yum repositories. +options: + state: + description: + - The expected state. Currently only supports list. + required: false + default: list + choices: ["list"] + aliases: [] + name: + description: + - The name of the package to query + required: true + default: None + aliases: [] + query_type: + description: + - Narrows the packages queried based off of this value. + - If repos, it narrows the query to repositories defined on the machine. + - If installed, it narrows the query to only packages installed on the machine. + - If available, it narrows the query to packages that are available to be installed. + - If recent, it narrows the query to only recently edited packages. + - If updates, it narrows the query to only packages that are updates to existing installed packages. + - If extras, it narrows the query to packages that are not present in any of the available repositories. + - If all, it queries all of the above. + required: false + default: repos + aliases: [] + verbose: + description: + - Shows more detail for the requested query. + required: false + default: false + aliases: [] + show_duplicates: + description: + - Shows multiple versions of a package. + required: false + default: false + aliases: [] + match_version: + description: + - Match the specific version given to the package. + required: false + default: None + aliases: [] +author: +- "Matt Woodson <mwoodson@redhat.com>" +extends_documentation_fragment: [] +''' + +EXAMPLES = ''' +# Example 1: Get bash versions + - name: Get bash version + repoquery: + name: bash + show_duplicates: True + register: bash_out + +# Results: +# ok: [localhost] => { +# "bash_out": { +# "changed": false, +# "results": { +# "cmd": "/usr/bin/repoquery --quiet --pkgnarrow=repos --queryformat=%{version}|%{release}|%{arch}|%{repo}|%{version}-%{release} --show-duplicates bash", +# "package_found": true, +# "package_name": "bash", +# "returncode": 0, +# "versions": { +# "available_versions": [ +# "4.2.45", +# "4.2.45", +# "4.2.45", +# "4.2.46", +# "4.2.46", +# "4.2.46", +# "4.2.46" +# ], +# "available_versions_full": [ +# "4.2.45-5.el7", +# "4.2.45-5.el7_0.2", +# "4.2.45-5.el7_0.4", +# "4.2.46-12.el7", +# "4.2.46-19.el7", +# "4.2.46-20.el7_2", +# "4.2.46-21.el7_3" +# ], +# "latest": "4.2.46", +# "latest_full": "4.2.46-21.el7_3" +# } +# }, +# "state": "present" +# } +# } + + + +# Example 2: Get bash versions verbosely + - name: Get bash versions verbosely + repoquery: + name: bash + show_duplicates: True + verbose: True + register: bash_out + +# Results: +# ok: [localhost] => { +# "bash_out": { +# "changed": false, +# "results": { +# "cmd": "/usr/bin/repoquery --quiet --pkgnarrow=repos --queryformat=%{version}|%{release}|%{arch}|%{repo}|%{version}-%{release} --show-duplicates bash", +# "package_found": true, +# "package_name": "bash", +# "raw_versions": { +# "4.2.45-5.el7": { +# "arch": "x86_64", +# "release": "5.el7", +# "repo": "rhel-7-server-rpms", +# "version": "4.2.45", +# "version_release": "4.2.45-5.el7" +# }, +# "4.2.45-5.el7_0.2": { +# "arch": "x86_64", +# "release": "5.el7_0.2", +# "repo": "rhel-7-server-rpms", +# "version": "4.2.45", +# "version_release": "4.2.45-5.el7_0.2" +# }, +# "4.2.45-5.el7_0.4": { +# "arch": "x86_64", +# "release": "5.el7_0.4", +# "repo": "rhel-7-server-rpms", +# "version": "4.2.45", +# "version_release": "4.2.45-5.el7_0.4" +# }, +# "4.2.46-12.el7": { +# "arch": "x86_64", +# "release": "12.el7", +# "repo": "rhel-7-server-rpms", +# "version": "4.2.46", +# "version_release": "4.2.46-12.el7" +# }, +# "4.2.46-19.el7": { +# "arch": "x86_64", +# "release": "19.el7", +# "repo": "rhel-7-server-rpms", +# "version": "4.2.46", +# "version_release": "4.2.46-19.el7" +# }, +# "4.2.46-20.el7_2": { +# "arch": "x86_64", +# "release": "20.el7_2", +# "repo": "rhel-7-server-rpms", +# "version": "4.2.46", +# "version_release": "4.2.46-20.el7_2" +# }, +# "4.2.46-21.el7_3": { +# "arch": "x86_64", +# "release": "21.el7_3", +# "repo": "rhel-7-server-rpms", +# "version": "4.2.46", +# "version_release": "4.2.46-21.el7_3" +# } +# }, +# "results": "4.2.45|5.el7|x86_64|rhel-7-server-rpms|4.2.45-5.el7\n4.2.45|5.el7_0.2|x86_64|rhel-7-server-rpms|4.2.45-5.el7_0.2\n4.2.45|5.el7_0.4|x86_64|rhel-7-server-rpms|4.2.45-5.el7_0.4\n4.2.46|12.el7|x86_64|rhel-7-server-rpms|4.2.46-12.el7\n4.2.46|19.el7|x86_64|rhel-7-server-rpms|4.2.46-19.el7\n4.2.46|20.el7_2|x86_64|rhel-7-server-rpms|4.2.46-20.el7_2\n4.2.46|21.el7_3|x86_64|rhel-7-server-rpms|4.2.46-21.el7_3\n", +# "returncode": 0, +# "versions": { +# "available_versions": [ +# "4.2.45", +# "4.2.45", +# "4.2.45", +# "4.2.46", +# "4.2.46", +# "4.2.46", +# "4.2.46" +# ], +# "available_versions_full": [ +# "4.2.45-5.el7", +# "4.2.45-5.el7_0.2", +# "4.2.45-5.el7_0.4", +# "4.2.46-12.el7", +# "4.2.46-19.el7", +# "4.2.46-20.el7_2", +# "4.2.46-21.el7_3" +# ], +# "latest": "4.2.46", +# "latest_full": "4.2.46-21.el7_3" +# } +# }, +# "state": "present" +# } +# } + +# Example 3: Match a specific version + - name: matched versions repoquery test + repoquery: + name: atomic-openshift + show_duplicates: True + match_version: 3.3 + register: openshift_out + +# Result: + +# ok: [localhost] => { +# "openshift_out": { +# "changed": false, +# "results": { +# "cmd": "/usr/bin/repoquery --quiet --pkgnarrow=repos --queryformat=%{version}|%{release}|%{arch}|%{repo}|%{version}-%{release} --show-duplicates atomic-openshift", +# "package_found": true, +# "package_name": "atomic-openshift", +# "returncode": 0, +# "versions": { +# "available_versions": [ +# "3.2.0.43", +# "3.2.1.23", +# "3.3.0.32", +# "3.3.0.34", +# "3.3.0.35", +# "3.3.1.3", +# "3.3.1.4", +# "3.3.1.5", +# "3.3.1.7", +# "3.4.0.39" +# ], +# "available_versions_full": [ +# "3.2.0.43-1.git.0.672599f.el7", +# "3.2.1.23-1.git.0.88a7a1d.el7", +# "3.3.0.32-1.git.0.37bd7ea.el7", +# "3.3.0.34-1.git.0.83f306f.el7", +# "3.3.0.35-1.git.0.d7bd9b6.el7", +# "3.3.1.3-1.git.0.86dc49a.el7", +# "3.3.1.4-1.git.0.7c8657c.el7", +# "3.3.1.5-1.git.0.62700af.el7", +# "3.3.1.7-1.git.0.0988966.el7", +# "3.4.0.39-1.git.0.5f32f06.el7" +# ], +# "latest": "3.4.0.39", +# "latest_full": "3.4.0.39-1.git.0.5f32f06.el7", +# "matched_version_found": true, +# "matched_version_full_latest": "3.3.1.7-1.git.0.0988966.el7", +# "matched_version_latest": "3.3.1.7", +# "matched_versions": [ +# "3.3.0.32", +# "3.3.0.34", +# "3.3.0.35", +# "3.3.1.3", +# "3.3.1.4", +# "3.3.1.5", +# "3.3.1.7" +# ], +# "matched_versions_full": [ +# "3.3.0.32-1.git.0.37bd7ea.el7", +# "3.3.0.34-1.git.0.83f306f.el7", +# "3.3.0.35-1.git.0.d7bd9b6.el7", +# "3.3.1.3-1.git.0.86dc49a.el7", +# "3.3.1.4-1.git.0.7c8657c.el7", +# "3.3.1.5-1.git.0.62700af.el7", +# "3.3.1.7-1.git.0.0988966.el7" +# ], +# "requested_match_version": "3.3" +# } +# }, +# "state": "present" +# } +# } + +''' + +# -*- -*- -*- End included fragment: doc/repoquery -*- -*- -*- + +# -*- -*- -*- Begin included fragment: lib/repoquery.py -*- -*- -*- + +''' + class that wraps the repoquery commands in a subprocess +''' + +# pylint: disable=too-many-lines,wrong-import-position,wrong-import-order + +from collections import defaultdict # noqa: E402 + + +# pylint: disable=no-name-in-module,import-error +# Reason: pylint errors with "No name 'version' in module 'distutils'". +# This is a bug: https://github.com/PyCQA/pylint/issues/73 +from distutils.version import LooseVersion # noqa: E402 + +import subprocess # noqa: E402 + + +class RepoqueryCLIError(Exception): + '''Exception class for repoquerycli''' + pass + + +def _run(cmds): + ''' Actually executes the command. This makes mocking easier. ''' + proc = subprocess.Popen(cmds, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + stdout, stderr = proc.communicate() + + return proc.returncode, stdout, stderr + + +# pylint: disable=too-few-public-methods +class RepoqueryCLI(object): + ''' Class to wrap the command line tools ''' + def __init__(self, + verbose=False): + ''' Constructor for RepoqueryCLI ''' + self.verbose = verbose + self.verbose = True + + def _repoquery_cmd(self, cmd, output=False, output_type='json'): + '''Base command for repoquery ''' + cmds = ['/usr/bin/repoquery', '--plugins', '--quiet'] + + cmds.extend(cmd) + + rval = {} + results = '' + err = None + + if self.verbose: + print(' '.join(cmds)) + + returncode, stdout, stderr = _run(cmds) + + rval = { + "returncode": returncode, + "results": results, + "cmd": ' '.join(cmds), + } + + if returncode == 0: + if output: + if output_type == 'raw': + rval['results'] = stdout + + if self.verbose: + print(stdout) + print(stderr) + + if err: + rval.update({ + "err": err, + "stderr": stderr, + "stdout": stdout, + "cmd": cmds + }) + + else: + rval.update({ + "stderr": stderr, + "stdout": stdout, + "results": {}, + }) + + return rval + +# -*- -*- -*- End included fragment: lib/repoquery.py -*- -*- -*- + +# -*- -*- -*- Begin included fragment: class/repoquery.py -*- -*- -*- + + +class Repoquery(RepoqueryCLI): + ''' Class to wrap the repoquery + ''' + # pylint: disable=too-many-arguments,too-many-instance-attributes + def __init__(self, name, query_type, show_duplicates, + match_version, ignore_excluders, verbose): + ''' Constructor for YumList ''' + super(Repoquery, self).__init__(None) + self.name = name + self.query_type = query_type + self.show_duplicates = show_duplicates + self.match_version = match_version + self.ignore_excluders = ignore_excluders + self.verbose = verbose + + if self.match_version: + self.show_duplicates = True + + self.query_format = "%{version}|%{release}|%{arch}|%{repo}|%{version}-%{release}" + + self.tmp_file = None + + def build_cmd(self): + ''' build the repoquery cmd options ''' + + repo_cmd = [] + + repo_cmd.append("--pkgnarrow=" + self.query_type) + repo_cmd.append("--queryformat=" + self.query_format) + + if self.show_duplicates: + repo_cmd.append('--show-duplicates') + + if self.ignore_excluders: + repo_cmd.append('--config=' + self.tmp_file.name) + + repo_cmd.append(self.name) + + return repo_cmd + + @staticmethod + def process_versions(query_output): + ''' format the package data into something that can be presented ''' + + version_dict = defaultdict(dict) + + for version in query_output.decode().split('\n'): + pkg_info = version.split("|") + + pkg_version = {} + pkg_version['version'] = pkg_info[0] + pkg_version['release'] = pkg_info[1] + pkg_version['arch'] = pkg_info[2] + pkg_version['repo'] = pkg_info[3] + pkg_version['version_release'] = pkg_info[4] + + version_dict[pkg_info[4]] = pkg_version + + return version_dict + + def format_versions(self, formatted_versions): + ''' Gather and present the versions of each package ''' + + versions_dict = {} + versions_dict['available_versions_full'] = list(formatted_versions.keys()) + + # set the match version, if called + if self.match_version: + versions_dict['matched_versions_full'] = [] + versions_dict['requested_match_version'] = self.match_version + versions_dict['matched_versions'] = [] + + # get the "full version (version - release) + versions_dict['available_versions_full'].sort(key=LooseVersion) + versions_dict['latest_full'] = versions_dict['available_versions_full'][-1] + + # get the "short version (version) + versions_dict['available_versions'] = [] + for version in versions_dict['available_versions_full']: + versions_dict['available_versions'].append(formatted_versions[version]['version']) + + if self.match_version: + if version.startswith(self.match_version): + versions_dict['matched_versions_full'].append(version) + versions_dict['matched_versions'].append(formatted_versions[version]['version']) + + versions_dict['available_versions'].sort(key=LooseVersion) + versions_dict['latest'] = versions_dict['available_versions'][-1] + + # finish up the matched version + if self.match_version: + if versions_dict['matched_versions_full']: + versions_dict['matched_version_found'] = True + versions_dict['matched_versions'].sort(key=LooseVersion) + versions_dict['matched_version_latest'] = versions_dict['matched_versions'][-1] + versions_dict['matched_version_full_latest'] = versions_dict['matched_versions_full'][-1] + else: + versions_dict['matched_version_found'] = False + versions_dict['matched_versions'] = [] + versions_dict['matched_version_latest'] = "" + versions_dict['matched_version_full_latest'] = "" + + return versions_dict + + def repoquery(self): + '''perform a repoquery ''' + + if self.ignore_excluders: + # Duplicate yum.conf and reset exclude= line to an empty string + # to clear a list of all excluded packages + self.tmp_file = tempfile.NamedTemporaryFile() + + with open("/etc/yum.conf", "r") as file_handler: + yum_conf_lines = file_handler.readlines() + + yum_conf_lines = ["exclude=" if l.startswith("exclude=") else l for l in yum_conf_lines] + + with open(self.tmp_file.name, "w") as file_handler: + file_handler.writelines(yum_conf_lines) + file_handler.flush() + + repoquery_cmd = self.build_cmd() + + rval = self._repoquery_cmd(repoquery_cmd, True, 'raw') + + # check to see if there are actual results + if rval['results']: + processed_versions = Repoquery.process_versions(rval['results'].strip()) + formatted_versions = self.format_versions(processed_versions) + + rval['package_found'] = True + rval['versions'] = formatted_versions + rval['package_name'] = self.name + + if self.verbose: + rval['raw_versions'] = processed_versions + else: + del rval['results'] + + # No packages found + else: + rval['package_found'] = False + + if self.ignore_excluders: + self.tmp_file.close() + + return rval + + @staticmethod + def run_ansible(params, check_mode): + '''run the ansible idempotent code''' + + repoquery = Repoquery( + params['name'], + params['query_type'], + params['show_duplicates'], + params['match_version'], + params['ignore_excluders'], + params['verbose'], + ) + + state = params['state'] + + if state == 'list': + results = repoquery.repoquery() + + if results['returncode'] != 0: + return {'failed': True, + 'msg': results} + + return {'changed': False, 'results': results, 'state': 'list', 'check_mode': check_mode} + + return {'failed': True, + 'changed': False, + 'msg': 'Unknown state passed. %s' % state, + 'state': 'unknown'} + +# -*- -*- -*- End included fragment: class/repoquery.py -*- -*- -*- + +# -*- -*- -*- Begin included fragment: ansible/repoquery.py -*- -*- -*- + + +def main(): + ''' + ansible repoquery module + ''' + module = AnsibleModule( + argument_spec=dict( + state=dict(default='list', type='str', choices=['list']), + name=dict(default=None, required=True, type='str'), + query_type=dict(default='repos', required=False, type='str', + choices=[ + 'installed', 'available', 'recent', + 'updates', 'extras', 'all', 'repos' + ]), + verbose=dict(default=False, required=False, type='bool'), + show_duplicates=dict(default=False, required=False, type='bool'), + match_version=dict(default=None, required=False, type='str'), + ignore_excluders=dict(default=False, required=False, type='bool'), + retries=dict(default=4, required=False, type='int'), + retry_interval=dict(default=5, required=False, type='int'), + ), + supports_check_mode=False, + required_if=[('show_duplicates', True, ['name'])], + ) + + tries = 1 + while True: + rval = Repoquery.run_ansible(module.params, module.check_mode) + if 'failed' not in rval: + module.exit_json(**rval) + elif tries > module.params['retries']: + module.fail_json(**rval) + tries += 1 + time.sleep(module.params['retry_interval']) + + +if __name__ == "__main__": + main() + +# -*- -*- -*- End included fragment: ansible/repoquery.py -*- -*- -*- diff --git a/roles/lib_utils/library/yedit.py b/roles/lib_utils/library/yedit.py new file mode 100644 index 000000000..cf5c2e423 --- /dev/null +++ b/roles/lib_utils/library/yedit.py @@ -0,0 +1,938 @@ +#!/usr/bin/env python +# pylint: disable=missing-docstring +# ___ ___ _ _ ___ ___ _ _____ ___ ___ +# / __| __| \| | __| _ \ /_\_ _| __| \ +# | (_ | _|| .` | _|| / / _ \| | | _|| |) | +# \___|___|_|\_|___|_|_\/_/_\_\_|_|___|___/_ _____ +# | \ / _ \ | \| |/ _ \_ _| | __| \_ _|_ _| +# | |) | (_) | | .` | (_) || | | _|| |) | | | | +# |___/ \___/ |_|\_|\___/ |_| |___|___/___| |_| +# +# Copyright 2016 Red Hat, Inc. and/or its affiliates +# and other contributors as indicated by the @author tags. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- -*- -*- Begin included fragment: lib/import.py -*- -*- -*- + +# pylint: disable=wrong-import-order,wrong-import-position,unused-import + +from __future__ import print_function # noqa: F401 +import copy # noqa: F401 +import json # noqa: F401 +import os # noqa: F401 +import re # noqa: F401 +import shutil # noqa: F401 +import tempfile # noqa: F401 +import time # noqa: F401 + +try: + import ruamel.yaml as yaml # noqa: F401 +except ImportError: + import yaml # noqa: F401 + +from ansible.module_utils.basic import AnsibleModule + +# -*- -*- -*- End included fragment: lib/import.py -*- -*- -*- + +# -*- -*- -*- Begin included fragment: doc/yedit -*- -*- -*- + +DOCUMENTATION = ''' +--- +module: yedit +short_description: Create, modify, and idempotently manage yaml files. +description: + - Modify yaml files programmatically. +options: + state: + description: + - State represents whether to create, modify, delete, or list yaml + required: true + default: present + choices: ["present", "absent", "list"] + aliases: [] + debug: + description: + - Turn on debug information. + required: false + default: false + aliases: [] + src: + description: + - The file that is the target of the modifications. + required: false + default: None + aliases: [] + content: + description: + - Content represents the yaml content you desire to work with. This + - could be the file contents to write or the inmemory data to modify. + required: false + default: None + aliases: [] + content_type: + description: + - The python type of the content parameter. + required: false + default: 'dict' + aliases: [] + key: + description: + - The path to the value you wish to modify. Emtpy string means the top of + - the document. + required: false + default: '' + aliases: [] + value: + description: + - The incoming value of parameter 'key'. + required: false + default: + aliases: [] + value_type: + description: + - The python type of the incoming value. + required: false + default: '' + aliases: [] + update: + description: + - Whether the update should be performed on a dict/hash or list/array + - object. + required: false + default: false + aliases: [] + append: + description: + - Whether to append to an array/list. When the key does not exist or is + - null, a new array is created. When the key is of a non-list type, + - nothing is done. + required: false + default: false + aliases: [] + index: + description: + - Used in conjunction with the update parameter. This will update a + - specific index in an array/list. + required: false + default: false + aliases: [] + curr_value: + description: + - Used in conjunction with the update parameter. This is the current + - value of 'key' in the yaml file. + required: false + default: false + aliases: [] + curr_value_format: + description: + - Format of the incoming current value. + choices: ["yaml", "json", "str"] + required: false + default: false + aliases: [] + backup: + description: + - Whether to make a backup copy of the current file when performing an + - edit. + required: false + default: true + aliases: [] + separator: + description: + - The separator being used when parsing strings. + required: false + default: '.' + aliases: [] +author: +- "Kenny Woodson <kwoodson@redhat.com>" +extends_documentation_fragment: [] +''' + +EXAMPLES = ''' +# Simple insert of key, value +- name: insert simple key, value + yedit: + src: somefile.yml + key: test + value: somevalue + state: present +# Results: +# test: somevalue + +# Multilevel insert of key, value +- name: insert simple key, value + yedit: + src: somefile.yml + key: a#b#c + value: d + state: present +# Results: +# a: +# b: +# c: d +# +# multiple edits at the same time +- name: perform multiple edits + yedit: + src: somefile.yml + edits: + - key: a#b#c + value: d + - key: a#b#c#d + value: e + state: present +# Results: +# a: +# b: +# c: +# d: e +''' + +# -*- -*- -*- End included fragment: doc/yedit -*- -*- -*- + +# -*- -*- -*- Begin included fragment: class/yedit.py -*- -*- -*- + + +class YeditException(Exception): + ''' Exception class for Yedit ''' + pass + + +# pylint: disable=too-many-public-methods +class Yedit(object): + ''' Class to modify yaml files ''' + re_valid_key = r"(((\[-?\d+\])|([0-9a-zA-Z%s/_-]+)).?)+$" + re_key = r"(?:\[(-?\d+)\])|([0-9a-zA-Z{}/_-]+)" + com_sep = set(['.', '#', '|', ':']) + + # pylint: disable=too-many-arguments + def __init__(self, + filename=None, + content=None, + content_type='yaml', + separator='.', + backup=False): + self.content = content + self._separator = separator + self.filename = filename + self.__yaml_dict = content + self.content_type = content_type + self.backup = backup + self.load(content_type=self.content_type) + if self.__yaml_dict is None: + self.__yaml_dict = {} + + @property + def separator(self): + ''' getter method for separator ''' + return self._separator + + @separator.setter + def separator(self, inc_sep): + ''' setter method for separator ''' + self._separator = inc_sep + + @property + def yaml_dict(self): + ''' getter method for yaml_dict ''' + return self.__yaml_dict + + @yaml_dict.setter + def yaml_dict(self, value): + ''' setter method for yaml_dict ''' + self.__yaml_dict = value + + @staticmethod + def parse_key(key, sep='.'): + '''parse the key allowing the appropriate separator''' + common_separators = list(Yedit.com_sep - set([sep])) + return re.findall(Yedit.re_key.format(''.join(common_separators)), key) + + @staticmethod + def valid_key(key, sep='.'): + '''validate the incoming key''' + common_separators = list(Yedit.com_sep - set([sep])) + if not re.match(Yedit.re_valid_key.format(''.join(common_separators)), key): + return False + + return True + + @staticmethod + def remove_entry(data, key, sep='.'): + ''' remove data at location key ''' + if key == '' and isinstance(data, dict): + data.clear() + return True + elif key == '' and isinstance(data, list): + del data[:] + return True + + if not (key and Yedit.valid_key(key, sep)) and \ + isinstance(data, (list, dict)): + return None + + key_indexes = Yedit.parse_key(key, sep) + for arr_ind, dict_key in key_indexes[:-1]: + if dict_key and isinstance(data, dict): + data = data.get(dict_key) + elif (arr_ind and isinstance(data, list) and + int(arr_ind) <= len(data) - 1): + data = data[int(arr_ind)] + else: + return None + + # process last index for remove + # expected list entry + if key_indexes[-1][0]: + if isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: # noqa: E501 + del data[int(key_indexes[-1][0])] + return True + + # expected dict entry + elif key_indexes[-1][1]: + if isinstance(data, dict): + del data[key_indexes[-1][1]] + return True + + @staticmethod + def add_entry(data, key, item=None, sep='.'): + ''' Get an item from a dictionary with key notation a.b.c + d = {'a': {'b': 'c'}}} + key = a#b + return c + ''' + if key == '': + pass + elif (not (key and Yedit.valid_key(key, sep)) and + isinstance(data, (list, dict))): + return None + + key_indexes = Yedit.parse_key(key, sep) + for arr_ind, dict_key in key_indexes[:-1]: + if dict_key: + if isinstance(data, dict) and dict_key in data and data[dict_key]: # noqa: E501 + data = data[dict_key] + continue + + elif data and not isinstance(data, dict): + raise YeditException("Unexpected item type found while going through key " + + "path: {} (at key: {})".format(key, dict_key)) + + data[dict_key] = {} + data = data[dict_key] + + elif (arr_ind and isinstance(data, list) and + int(arr_ind) <= len(data) - 1): + data = data[int(arr_ind)] + else: + raise YeditException("Unexpected item type found while going through key path: {}".format(key)) + + if key == '': + data = item + + # process last index for add + # expected list entry + elif key_indexes[-1][0] and isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: # noqa: E501 + data[int(key_indexes[-1][0])] = item + + # expected dict entry + elif key_indexes[-1][1] and isinstance(data, dict): + data[key_indexes[-1][1]] = item + + # didn't add/update to an existing list, nor add/update key to a dict + # so we must have been provided some syntax like a.b.c[<int>] = "data" for a + # non-existent array + else: + raise YeditException("Error adding to object at path: {}".format(key)) + + return data + + @staticmethod + def get_entry(data, key, sep='.'): + ''' Get an item from a dictionary with key notation a.b.c + d = {'a': {'b': 'c'}}} + key = a.b + return c + ''' + if key == '': + pass + elif (not (key and Yedit.valid_key(key, sep)) and + isinstance(data, (list, dict))): + return None + + key_indexes = Yedit.parse_key(key, sep) + for arr_ind, dict_key in key_indexes: + if dict_key and isinstance(data, dict): + data = data.get(dict_key) + elif (arr_ind and isinstance(data, list) and + int(arr_ind) <= len(data) - 1): + data = data[int(arr_ind)] + else: + return None + + return data + + @staticmethod + def _write(filename, contents): + ''' Actually write the file contents to disk. This helps with mocking. ''' + + tmp_filename = filename + '.yedit' + + with open(tmp_filename, 'w') as yfd: + yfd.write(contents) + + os.rename(tmp_filename, filename) + + def write(self): + ''' write to file ''' + if not self.filename: + raise YeditException('Please specify a filename.') + + if self.backup and self.file_exists(): + shutil.copy(self.filename, self.filename + '.orig') + + # Try to set format attributes if supported + try: + self.yaml_dict.fa.set_block_style() + except AttributeError: + pass + + # Try to use RoundTripDumper if supported. + try: + Yedit._write(self.filename, yaml.dump(self.yaml_dict, Dumper=yaml.RoundTripDumper)) + except AttributeError: + Yedit._write(self.filename, yaml.safe_dump(self.yaml_dict, default_flow_style=False)) + + return (True, self.yaml_dict) + + def read(self): + ''' read from file ''' + # check if it exists + if self.filename is None or not self.file_exists(): + return None + + contents = None + with open(self.filename) as yfd: + contents = yfd.read() + + return contents + + def file_exists(self): + ''' return whether file exists ''' + if os.path.exists(self.filename): + return True + + return False + + def load(self, content_type='yaml'): + ''' return yaml file ''' + contents = self.read() + + if not contents and not self.content: + return None + + if self.content: + if isinstance(self.content, dict): + self.yaml_dict = self.content + return self.yaml_dict + elif isinstance(self.content, str): + contents = self.content + + # check if it is yaml + try: + if content_type == 'yaml' and contents: + # Try to set format attributes if supported + try: + self.yaml_dict.fa.set_block_style() + except AttributeError: + pass + + # Try to use RoundTripLoader if supported. + try: + self.yaml_dict = yaml.safe_load(contents, yaml.RoundTripLoader) + except AttributeError: + self.yaml_dict = yaml.safe_load(contents) + + # Try to set format attributes if supported + try: + self.yaml_dict.fa.set_block_style() + except AttributeError: + pass + + elif content_type == 'json' and contents: + self.yaml_dict = json.loads(contents) + except yaml.YAMLError as err: + # Error loading yaml or json + raise YeditException('Problem with loading yaml file. {}'.format(err)) + + return self.yaml_dict + + def get(self, key): + ''' get a specified key''' + try: + entry = Yedit.get_entry(self.yaml_dict, key, self.separator) + except KeyError: + entry = None + + return entry + + def pop(self, path, key_or_item): + ''' remove a key, value pair from a dict or an item for a list''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry is None: + return (False, self.yaml_dict) + + if isinstance(entry, dict): + # AUDIT:maybe-no-member makes sense due to fuzzy types + # pylint: disable=maybe-no-member + if key_or_item in entry: + entry.pop(key_or_item) + return (True, self.yaml_dict) + return (False, self.yaml_dict) + + elif isinstance(entry, list): + # AUDIT:maybe-no-member makes sense due to fuzzy types + # pylint: disable=maybe-no-member + ind = None + try: + ind = entry.index(key_or_item) + except ValueError: + return (False, self.yaml_dict) + + entry.pop(ind) + return (True, self.yaml_dict) + + return (False, self.yaml_dict) + + def delete(self, path): + ''' remove path from a dict''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry is None: + return (False, self.yaml_dict) + + result = Yedit.remove_entry(self.yaml_dict, path, self.separator) + if not result: + return (False, self.yaml_dict) + + return (True, self.yaml_dict) + + def exists(self, path, value): + ''' check if value exists at path''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if isinstance(entry, list): + if value in entry: + return True + return False + + elif isinstance(entry, dict): + if isinstance(value, dict): + rval = False + for key, val in value.items(): + if entry[key] != val: + rval = False + break + else: + rval = True + return rval + + return value in entry + + return entry == value + + def append(self, path, value): + '''append value to a list''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry is None: + self.put(path, []) + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + if not isinstance(entry, list): + return (False, self.yaml_dict) + + # AUDIT:maybe-no-member makes sense due to loading data from + # a serialized format. + # pylint: disable=maybe-no-member + entry.append(value) + return (True, self.yaml_dict) + + # pylint: disable=too-many-arguments + def update(self, path, value, index=None, curr_value=None): + ''' put path, value into a dict ''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if isinstance(entry, dict): + # AUDIT:maybe-no-member makes sense due to fuzzy types + # pylint: disable=maybe-no-member + if not isinstance(value, dict): + raise YeditException('Cannot replace key, value entry in dict with non-dict type. ' + + 'value=[{}] type=[{}]'.format(value, type(value))) + + entry.update(value) + return (True, self.yaml_dict) + + elif isinstance(entry, list): + # AUDIT:maybe-no-member makes sense due to fuzzy types + # pylint: disable=maybe-no-member + ind = None + if curr_value: + try: + ind = entry.index(curr_value) + except ValueError: + return (False, self.yaml_dict) + + elif index is not None: + ind = index + + if ind is not None and entry[ind] != value: + entry[ind] = value + return (True, self.yaml_dict) + + # see if it exists in the list + try: + ind = entry.index(value) + except ValueError: + # doesn't exist, append it + entry.append(value) + return (True, self.yaml_dict) + + # already exists, return + if ind is not None: + return (False, self.yaml_dict) + return (False, self.yaml_dict) + + def put(self, path, value): + ''' put path, value into a dict ''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry == value: + return (False, self.yaml_dict) + + # deepcopy didn't work + # Try to use ruamel.yaml and fallback to pyyaml + try: + tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, + default_flow_style=False), + yaml.RoundTripLoader) + except AttributeError: + tmp_copy = copy.deepcopy(self.yaml_dict) + + # set the format attributes if available + try: + tmp_copy.fa.set_block_style() + except AttributeError: + pass + + result = Yedit.add_entry(tmp_copy, path, value, self.separator) + if result is None: + return (False, self.yaml_dict) + + # When path equals "" it is a special case. + # "" refers to the root of the document + # Only update the root path (entire document) when its a list or dict + if path == '': + if isinstance(result, list) or isinstance(result, dict): + self.yaml_dict = result + return (True, self.yaml_dict) + + return (False, self.yaml_dict) + + self.yaml_dict = tmp_copy + + return (True, self.yaml_dict) + + def create(self, path, value): + ''' create a yaml file ''' + if not self.file_exists(): + # deepcopy didn't work + # Try to use ruamel.yaml and fallback to pyyaml + try: + tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, + default_flow_style=False), + yaml.RoundTripLoader) + except AttributeError: + tmp_copy = copy.deepcopy(self.yaml_dict) + + # set the format attributes if available + try: + tmp_copy.fa.set_block_style() + except AttributeError: + pass + + result = Yedit.add_entry(tmp_copy, path, value, self.separator) + if result is not None: + self.yaml_dict = tmp_copy + return (True, self.yaml_dict) + + return (False, self.yaml_dict) + + @staticmethod + def get_curr_value(invalue, val_type): + '''return the current value''' + if invalue is None: + return None + + curr_value = invalue + if val_type == 'yaml': + curr_value = yaml.load(invalue) + elif val_type == 'json': + curr_value = json.loads(invalue) + + return curr_value + + @staticmethod + def parse_value(inc_value, vtype=''): + '''determine value type passed''' + true_bools = ['y', 'Y', 'yes', 'Yes', 'YES', 'true', 'True', 'TRUE', + 'on', 'On', 'ON', ] + false_bools = ['n', 'N', 'no', 'No', 'NO', 'false', 'False', 'FALSE', + 'off', 'Off', 'OFF'] + + # It came in as a string but you didn't specify value_type as string + # we will convert to bool if it matches any of the above cases + if isinstance(inc_value, str) and 'bool' in vtype: + if inc_value not in true_bools and inc_value not in false_bools: + raise YeditException('Not a boolean type. str=[{}] vtype=[{}]'.format(inc_value, vtype)) + elif isinstance(inc_value, bool) and 'str' in vtype: + inc_value = str(inc_value) + + # There is a special case where '' will turn into None after yaml loading it so skip + if isinstance(inc_value, str) and inc_value == '': + pass + # If vtype is not str then go ahead and attempt to yaml load it. + elif isinstance(inc_value, str) and 'str' not in vtype: + try: + inc_value = yaml.safe_load(inc_value) + except Exception: + raise YeditException('Could not determine type of incoming value. ' + + 'value=[{}] vtype=[{}]'.format(type(inc_value), vtype)) + + return inc_value + + @staticmethod + def process_edits(edits, yamlfile): + '''run through a list of edits and process them one-by-one''' + results = [] + for edit in edits: + value = Yedit.parse_value(edit['value'], edit.get('value_type', '')) + if edit.get('action') == 'update': + # pylint: disable=line-too-long + curr_value = Yedit.get_curr_value( + Yedit.parse_value(edit.get('curr_value')), + edit.get('curr_value_format')) + + rval = yamlfile.update(edit['key'], + value, + edit.get('index'), + curr_value) + + elif edit.get('action') == 'append': + rval = yamlfile.append(edit['key'], value) + + else: + rval = yamlfile.put(edit['key'], value) + + if rval[0]: + results.append({'key': edit['key'], 'edit': rval[1]}) + + return {'changed': len(results) > 0, 'results': results} + + # pylint: disable=too-many-return-statements,too-many-branches + @staticmethod + def run_ansible(params): + '''perform the idempotent crud operations''' + yamlfile = Yedit(filename=params['src'], + backup=params['backup'], + separator=params['separator']) + + state = params['state'] + + if params['src']: + rval = yamlfile.load() + + if yamlfile.yaml_dict is None and state != 'present': + return {'failed': True, + 'msg': 'Error opening file [{}]. Verify that the '.format(params['src']) + + 'file exists, that it is has correct permissions, and is valid yaml.'} + + if state == 'list': + if params['content']: + content = Yedit.parse_value(params['content'], params['content_type']) + yamlfile.yaml_dict = content + + if params['key']: + rval = yamlfile.get(params['key']) + + return {'changed': False, 'result': rval, 'state': state} + + elif state == 'absent': + if params['content']: + content = Yedit.parse_value(params['content'], params['content_type']) + yamlfile.yaml_dict = content + + if params['update']: + rval = yamlfile.pop(params['key'], params['value']) + else: + rval = yamlfile.delete(params['key']) + + if rval[0] and params['src']: + yamlfile.write() + + return {'changed': rval[0], 'result': rval[1], 'state': state} + + elif state == 'present': + # check if content is different than what is in the file + if params['content']: + content = Yedit.parse_value(params['content'], params['content_type']) + + # We had no edits to make and the contents are the same + if yamlfile.yaml_dict == content and \ + params['value'] is None: + return {'changed': False, 'result': yamlfile.yaml_dict, 'state': state} + + yamlfile.yaml_dict = content + + # If we were passed a key, value then + # we enapsulate it in a list and process it + # Key, Value passed to the module : Converted to Edits list # + edits = [] + _edit = {} + if params['value'] is not None: + _edit['value'] = params['value'] + _edit['value_type'] = params['value_type'] + _edit['key'] = params['key'] + + if params['update']: + _edit['action'] = 'update' + _edit['curr_value'] = params['curr_value'] + _edit['curr_value_format'] = params['curr_value_format'] + _edit['index'] = params['index'] + + elif params['append']: + _edit['action'] = 'append' + + edits.append(_edit) + + elif params['edits'] is not None: + edits = params['edits'] + + if edits: + results = Yedit.process_edits(edits, yamlfile) + + # if there were changes and a src provided to us we need to write + if results['changed'] and params['src']: + yamlfile.write() + + return {'changed': results['changed'], 'result': results['results'], 'state': state} + + # no edits to make + if params['src']: + # pylint: disable=redefined-variable-type + rval = yamlfile.write() + return {'changed': rval[0], + 'result': rval[1], + 'state': state} + + # We were passed content but no src, key or value, or edits. Return contents in memory + return {'changed': False, 'result': yamlfile.yaml_dict, 'state': state} + return {'failed': True, 'msg': 'Unkown state passed'} + +# -*- -*- -*- End included fragment: class/yedit.py -*- -*- -*- + +# -*- -*- -*- Begin included fragment: ansible/yedit.py -*- -*- -*- + + +# pylint: disable=too-many-branches +def main(): + ''' ansible oc module for secrets ''' + + module = AnsibleModule( + argument_spec=dict( + state=dict(default='present', type='str', + choices=['present', 'absent', 'list']), + debug=dict(default=False, type='bool'), + src=dict(default=None, type='str'), + content=dict(default=None), + content_type=dict(default='dict', choices=['dict']), + key=dict(default='', type='str'), + value=dict(), + value_type=dict(default='', type='str'), + update=dict(default=False, type='bool'), + append=dict(default=False, type='bool'), + index=dict(default=None, type='int'), + curr_value=dict(default=None, type='str'), + curr_value_format=dict(default='yaml', + choices=['yaml', 'json', 'str'], + type='str'), + backup=dict(default=True, type='bool'), + separator=dict(default='.', type='str'), + edits=dict(default=None, type='list'), + ), + mutually_exclusive=[["curr_value", "index"], ['update', "append"]], + required_one_of=[["content", "src"]], + ) + + # Verify we recieved either a valid key or edits with valid keys when receiving a src file. + # A valid key being not None or not ''. + if module.params['src'] is not None: + key_error = False + edit_error = False + + if module.params['key'] in [None, '']: + key_error = True + + if module.params['edits'] in [None, []]: + edit_error = True + + else: + for edit in module.params['edits']: + if edit.get('key') in [None, '']: + edit_error = True + break + + if key_error and edit_error: + module.fail_json(failed=True, msg='Empty value for parameter key not allowed.') + + rval = Yedit.run_ansible(module.params) + if 'failed' in rval and rval['failed']: + module.fail_json(**rval) + + module.exit_json(**rval) + + +if __name__ == '__main__': + main() + +# -*- -*- -*- End included fragment: ansible/yedit.py -*- -*- -*- diff --git a/roles/lib_utils/meta/main.yml b/roles/lib_utils/meta/main.yml new file mode 100644 index 000000000..cc18c453c --- /dev/null +++ b/roles/lib_utils/meta/main.yml @@ -0,0 +1,14 @@ +--- +galaxy_info: + author: TODO + description: OpenShift Repositories + company: Red Hat, Inc. + license: Apache License, Version 2.0 + min_ansible_version: 1.7 + platforms: + - name: EL + versions: + - 7 + categories: + - cloud +dependencies: [] diff --git a/roles/lib_utils/src/ansible/repoquery.py b/roles/lib_utils/src/ansible/repoquery.py new file mode 100644 index 000000000..5f5b93639 --- /dev/null +++ b/roles/lib_utils/src/ansible/repoquery.py @@ -0,0 +1,41 @@ +# pylint: skip-file +# flake8: noqa + + +def main(): + ''' + ansible repoquery module + ''' + module = AnsibleModule( + argument_spec=dict( + state=dict(default='list', type='str', choices=['list']), + name=dict(default=None, required=True, type='str'), + query_type=dict(default='repos', required=False, type='str', + choices=[ + 'installed', 'available', 'recent', + 'updates', 'extras', 'all', 'repos' + ]), + verbose=dict(default=False, required=False, type='bool'), + show_duplicates=dict(default=False, required=False, type='bool'), + match_version=dict(default=None, required=False, type='str'), + ignore_excluders=dict(default=False, required=False, type='bool'), + retries=dict(default=4, required=False, type='int'), + retry_interval=dict(default=5, required=False, type='int'), + ), + supports_check_mode=False, + required_if=[('show_duplicates', True, ['name'])], + ) + + tries = 1 + while True: + rval = Repoquery.run_ansible(module.params, module.check_mode) + if 'failed' not in rval: + module.exit_json(**rval) + elif tries > module.params['retries']: + module.fail_json(**rval) + tries += 1 + time.sleep(module.params['retry_interval']) + + +if __name__ == "__main__": + main() diff --git a/roles/lib_utils/src/ansible/yedit.py b/roles/lib_utils/src/ansible/yedit.py new file mode 100644 index 000000000..c4b818cf1 --- /dev/null +++ b/roles/lib_utils/src/ansible/yedit.py @@ -0,0 +1,64 @@ +# flake8: noqa +# pylint: skip-file + + +# pylint: disable=too-many-branches +def main(): + ''' ansible oc module for secrets ''' + + module = AnsibleModule( + argument_spec=dict( + state=dict(default='present', type='str', + choices=['present', 'absent', 'list']), + debug=dict(default=False, type='bool'), + src=dict(default=None, type='str'), + content=dict(default=None), + content_type=dict(default='dict', choices=['dict']), + key=dict(default='', type='str'), + value=dict(), + value_type=dict(default='', type='str'), + update=dict(default=False, type='bool'), + append=dict(default=False, type='bool'), + index=dict(default=None, type='int'), + curr_value=dict(default=None, type='str'), + curr_value_format=dict(default='yaml', + choices=['yaml', 'json', 'str'], + type='str'), + backup=dict(default=True, type='bool'), + separator=dict(default='.', type='str'), + edits=dict(default=None, type='list'), + ), + mutually_exclusive=[["curr_value", "index"], ['update', "append"]], + required_one_of=[["content", "src"]], + ) + + # Verify we recieved either a valid key or edits with valid keys when receiving a src file. + # A valid key being not None or not ''. + if module.params['src'] is not None: + key_error = False + edit_error = False + + if module.params['key'] in [None, '']: + key_error = True + + if module.params['edits'] in [None, []]: + edit_error = True + + else: + for edit in module.params['edits']: + if edit.get('key') in [None, '']: + edit_error = True + break + + if key_error and edit_error: + module.fail_json(failed=True, msg='Empty value for parameter key not allowed.') + + rval = Yedit.run_ansible(module.params) + if 'failed' in rval and rval['failed']: + module.fail_json(**rval) + + module.exit_json(**rval) + + +if __name__ == '__main__': + main() diff --git a/roles/lib_utils/src/class/repoquery.py b/roles/lib_utils/src/class/repoquery.py new file mode 100644 index 000000000..e997780ad --- /dev/null +++ b/roles/lib_utils/src/class/repoquery.py @@ -0,0 +1,180 @@ +# pylint: skip-file +# flake8: noqa + + +class Repoquery(RepoqueryCLI): + ''' Class to wrap the repoquery + ''' + # pylint: disable=too-many-arguments,too-many-instance-attributes + def __init__(self, name, query_type, show_duplicates, + match_version, ignore_excluders, verbose): + ''' Constructor for YumList ''' + super(Repoquery, self).__init__(None) + self.name = name + self.query_type = query_type + self.show_duplicates = show_duplicates + self.match_version = match_version + self.ignore_excluders = ignore_excluders + self.verbose = verbose + + if self.match_version: + self.show_duplicates = True + + self.query_format = "%{version}|%{release}|%{arch}|%{repo}|%{version}-%{release}" + + self.tmp_file = None + + def build_cmd(self): + ''' build the repoquery cmd options ''' + + repo_cmd = [] + + repo_cmd.append("--pkgnarrow=" + self.query_type) + repo_cmd.append("--queryformat=" + self.query_format) + + if self.show_duplicates: + repo_cmd.append('--show-duplicates') + + if self.ignore_excluders: + repo_cmd.append('--config=' + self.tmp_file.name) + + repo_cmd.append(self.name) + + return repo_cmd + + @staticmethod + def process_versions(query_output): + ''' format the package data into something that can be presented ''' + + version_dict = defaultdict(dict) + + for version in query_output.decode().split('\n'): + pkg_info = version.split("|") + + pkg_version = {} + pkg_version['version'] = pkg_info[0] + pkg_version['release'] = pkg_info[1] + pkg_version['arch'] = pkg_info[2] + pkg_version['repo'] = pkg_info[3] + pkg_version['version_release'] = pkg_info[4] + + version_dict[pkg_info[4]] = pkg_version + + return version_dict + + def format_versions(self, formatted_versions): + ''' Gather and present the versions of each package ''' + + versions_dict = {} + versions_dict['available_versions_full'] = list(formatted_versions.keys()) + + # set the match version, if called + if self.match_version: + versions_dict['matched_versions_full'] = [] + versions_dict['requested_match_version'] = self.match_version + versions_dict['matched_versions'] = [] + + # get the "full version (version - release) + versions_dict['available_versions_full'].sort(key=LooseVersion) + versions_dict['latest_full'] = versions_dict['available_versions_full'][-1] + + # get the "short version (version) + versions_dict['available_versions'] = [] + for version in versions_dict['available_versions_full']: + versions_dict['available_versions'].append(formatted_versions[version]['version']) + + if self.match_version: + if version.startswith(self.match_version): + versions_dict['matched_versions_full'].append(version) + versions_dict['matched_versions'].append(formatted_versions[version]['version']) + + versions_dict['available_versions'].sort(key=LooseVersion) + versions_dict['latest'] = versions_dict['available_versions'][-1] + + # finish up the matched version + if self.match_version: + if versions_dict['matched_versions_full']: + versions_dict['matched_version_found'] = True + versions_dict['matched_versions'].sort(key=LooseVersion) + versions_dict['matched_version_latest'] = versions_dict['matched_versions'][-1] + versions_dict['matched_version_full_latest'] = versions_dict['matched_versions_full'][-1] + else: + versions_dict['matched_version_found'] = False + versions_dict['matched_versions'] = [] + versions_dict['matched_version_latest'] = "" + versions_dict['matched_version_full_latest'] = "" + + return versions_dict + + def repoquery(self): + '''perform a repoquery ''' + + if self.ignore_excluders: + # Duplicate yum.conf and reset exclude= line to an empty string + # to clear a list of all excluded packages + self.tmp_file = tempfile.NamedTemporaryFile() + + with open("/etc/yum.conf", "r") as file_handler: + yum_conf_lines = file_handler.readlines() + + yum_conf_lines = ["exclude=" if l.startswith("exclude=") else l for l in yum_conf_lines] + + with open(self.tmp_file.name, "w") as file_handler: + file_handler.writelines(yum_conf_lines) + file_handler.flush() + + repoquery_cmd = self.build_cmd() + + rval = self._repoquery_cmd(repoquery_cmd, True, 'raw') + + # check to see if there are actual results + if rval['results']: + processed_versions = Repoquery.process_versions(rval['results'].strip()) + formatted_versions = self.format_versions(processed_versions) + + rval['package_found'] = True + rval['versions'] = formatted_versions + rval['package_name'] = self.name + + if self.verbose: + rval['raw_versions'] = processed_versions + else: + del rval['results'] + + # No packages found + else: + rval['package_found'] = False + + if self.ignore_excluders: + self.tmp_file.close() + + return rval + + @staticmethod + def run_ansible(params, check_mode): + '''run the ansible idempotent code''' + + repoquery = Repoquery( + params['name'], + params['query_type'], + params['show_duplicates'], + params['match_version'], + params['ignore_excluders'], + params['verbose'], + ) + + state = params['state'] + + if state == 'list': + results = repoquery.repoquery() + + if results['returncode'] != 0: + return {'failed': True, + 'msg': results} + + return {'changed': False, 'results': results, 'state': 'list', 'check_mode': check_mode} + + return {'failed': True, + 'changed': False, + 'msg': 'Unknown state passed. %s' % state, + 'state': 'unknown'} diff --git a/roles/lib_utils/src/class/yedit.py b/roles/lib_utils/src/class/yedit.py new file mode 100644 index 000000000..0a4fbe07a --- /dev/null +++ b/roles/lib_utils/src/class/yedit.py @@ -0,0 +1,667 @@ +# flake8: noqa +# pylint: skip-file + + +class YeditException(Exception): + ''' Exception class for Yedit ''' + pass + + +# pylint: disable=too-many-public-methods +class Yedit(object): + ''' Class to modify yaml files ''' + re_valid_key = r"(((\[-?\d+\])|([0-9a-zA-Z%s/_-]+)).?)+$" + re_key = r"(?:\[(-?\d+)\])|([0-9a-zA-Z{}/_-]+)" + com_sep = set(['.', '#', '|', ':']) + + # pylint: disable=too-many-arguments + def __init__(self, + filename=None, + content=None, + content_type='yaml', + separator='.', + backup=False): + self.content = content + self._separator = separator + self.filename = filename + self.__yaml_dict = content + self.content_type = content_type + self.backup = backup + self.load(content_type=self.content_type) + if self.__yaml_dict is None: + self.__yaml_dict = {} + + @property + def separator(self): + ''' getter method for separator ''' + return self._separator + + @separator.setter + def separator(self, inc_sep): + ''' setter method for separator ''' + self._separator = inc_sep + + @property + def yaml_dict(self): + ''' getter method for yaml_dict ''' + return self.__yaml_dict + + @yaml_dict.setter + def yaml_dict(self, value): + ''' setter method for yaml_dict ''' + self.__yaml_dict = value + + @staticmethod + def parse_key(key, sep='.'): + '''parse the key allowing the appropriate separator''' + common_separators = list(Yedit.com_sep - set([sep])) + return re.findall(Yedit.re_key.format(''.join(common_separators)), key) + + @staticmethod + def valid_key(key, sep='.'): + '''validate the incoming key''' + common_separators = list(Yedit.com_sep - set([sep])) + if not re.match(Yedit.re_valid_key.format(''.join(common_separators)), key): + return False + + return True + + @staticmethod + def remove_entry(data, key, sep='.'): + ''' remove data at location key ''' + if key == '' and isinstance(data, dict): + data.clear() + return True + elif key == '' and isinstance(data, list): + del data[:] + return True + + if not (key and Yedit.valid_key(key, sep)) and \ + isinstance(data, (list, dict)): + return None + + key_indexes = Yedit.parse_key(key, sep) + for arr_ind, dict_key in key_indexes[:-1]: + if dict_key and isinstance(data, dict): + data = data.get(dict_key) + elif (arr_ind and isinstance(data, list) and + int(arr_ind) <= len(data) - 1): + data = data[int(arr_ind)] + else: + return None + + # process last index for remove + # expected list entry + if key_indexes[-1][0]: + if isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: # noqa: E501 + del data[int(key_indexes[-1][0])] + return True + + # expected dict entry + elif key_indexes[-1][1]: + if isinstance(data, dict): + del data[key_indexes[-1][1]] + return True + + @staticmethod + def add_entry(data, key, item=None, sep='.'): + ''' Get an item from a dictionary with key notation a.b.c + d = {'a': {'b': 'c'}}} + key = a#b + return c + ''' + if key == '': + pass + elif (not (key and Yedit.valid_key(key, sep)) and + isinstance(data, (list, dict))): + return None + + key_indexes = Yedit.parse_key(key, sep) + for arr_ind, dict_key in key_indexes[:-1]: + if dict_key: + if isinstance(data, dict) and dict_key in data and data[dict_key]: # noqa: E501 + data = data[dict_key] + continue + + elif data and not isinstance(data, dict): + raise YeditException("Unexpected item type found while going through key " + + "path: {} (at key: {})".format(key, dict_key)) + + data[dict_key] = {} + data = data[dict_key] + + elif (arr_ind and isinstance(data, list) and + int(arr_ind) <= len(data) - 1): + data = data[int(arr_ind)] + else: + raise YeditException("Unexpected item type found while going through key path: {}".format(key)) + + if key == '': + data = item + + # process last index for add + # expected list entry + elif key_indexes[-1][0] and isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: # noqa: E501 + data[int(key_indexes[-1][0])] = item + + # expected dict entry + elif key_indexes[-1][1] and isinstance(data, dict): + data[key_indexes[-1][1]] = item + + # didn't add/update to an existing list, nor add/update key to a dict + # so we must have been provided some syntax like a.b.c[<int>] = "data" for a + # non-existent array + else: + raise YeditException("Error adding to object at path: {}".format(key)) + + return data + + @staticmethod + def get_entry(data, key, sep='.'): + ''' Get an item from a dictionary with key notation a.b.c + d = {'a': {'b': 'c'}}} + key = a.b + return c + ''' + if key == '': + pass + elif (not (key and Yedit.valid_key(key, sep)) and + isinstance(data, (list, dict))): + return None + + key_indexes = Yedit.parse_key(key, sep) + for arr_ind, dict_key in key_indexes: + if dict_key and isinstance(data, dict): + data = data.get(dict_key) + elif (arr_ind and isinstance(data, list) and + int(arr_ind) <= len(data) - 1): + data = data[int(arr_ind)] + else: + return None + + return data + + @staticmethod + def _write(filename, contents): + ''' Actually write the file contents to disk. This helps with mocking. ''' + + tmp_filename = filename + '.yedit' + + with open(tmp_filename, 'w') as yfd: + yfd.write(contents) + + os.rename(tmp_filename, filename) + + def write(self): + ''' write to file ''' + if not self.filename: + raise YeditException('Please specify a filename.') + + if self.backup and self.file_exists(): + shutil.copy(self.filename, self.filename + '.orig') + + # Try to set format attributes if supported + try: + self.yaml_dict.fa.set_block_style() + except AttributeError: + pass + + # Try to use RoundTripDumper if supported. + try: + Yedit._write(self.filename, yaml.dump(self.yaml_dict, Dumper=yaml.RoundTripDumper)) + except AttributeError: + Yedit._write(self.filename, yaml.safe_dump(self.yaml_dict, default_flow_style=False)) + + return (True, self.yaml_dict) + + def read(self): + ''' read from file ''' + # check if it exists + if self.filename is None or not self.file_exists(): + return None + + contents = None + with open(self.filename) as yfd: + contents = yfd.read() + + return contents + + def file_exists(self): + ''' return whether file exists ''' + if os.path.exists(self.filename): + return True + + return False + + def load(self, content_type='yaml'): + ''' return yaml file ''' + contents = self.read() + + if not contents and not self.content: + return None + + if self.content: + if isinstance(self.content, dict): + self.yaml_dict = self.content + return self.yaml_dict + elif isinstance(self.content, str): + contents = self.content + + # check if it is yaml + try: + if content_type == 'yaml' and contents: + # Try to set format attributes if supported + try: + self.yaml_dict.fa.set_block_style() + except AttributeError: + pass + + # Try to use RoundTripLoader if supported. + try: + self.yaml_dict = yaml.safe_load(contents, yaml.RoundTripLoader) + except AttributeError: + self.yaml_dict = yaml.safe_load(contents) + + # Try to set format attributes if supported + try: + self.yaml_dict.fa.set_block_style() + except AttributeError: + pass + + elif content_type == 'json' and contents: + self.yaml_dict = json.loads(contents) + except yaml.YAMLError as err: + # Error loading yaml or json + raise YeditException('Problem with loading yaml file. {}'.format(err)) + + return self.yaml_dict + + def get(self, key): + ''' get a specified key''' + try: + entry = Yedit.get_entry(self.yaml_dict, key, self.separator) + except KeyError: + entry = None + + return entry + + def pop(self, path, key_or_item): + ''' remove a key, value pair from a dict or an item for a list''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry is None: + return (False, self.yaml_dict) + + if isinstance(entry, dict): + # AUDIT:maybe-no-member makes sense due to fuzzy types + # pylint: disable=maybe-no-member + if key_or_item in entry: + entry.pop(key_or_item) + return (True, self.yaml_dict) + return (False, self.yaml_dict) + + elif isinstance(entry, list): + # AUDIT:maybe-no-member makes sense due to fuzzy types + # pylint: disable=maybe-no-member + ind = None + try: + ind = entry.index(key_or_item) + except ValueError: + return (False, self.yaml_dict) + + entry.pop(ind) + return (True, self.yaml_dict) + + return (False, self.yaml_dict) + + def delete(self, path): + ''' remove path from a dict''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry is None: + return (False, self.yaml_dict) + + result = Yedit.remove_entry(self.yaml_dict, path, self.separator) + if not result: + return (False, self.yaml_dict) + + return (True, self.yaml_dict) + + def exists(self, path, value): + ''' check if value exists at path''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if isinstance(entry, list): + if value in entry: + return True + return False + + elif isinstance(entry, dict): + if isinstance(value, dict): + rval = False + for key, val in value.items(): + if entry[key] != val: + rval = False + break + else: + rval = True + return rval + + return value in entry + + return entry == value + + def append(self, path, value): + '''append value to a list''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry is None: + self.put(path, []) + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + if not isinstance(entry, list): + return (False, self.yaml_dict) + + # AUDIT:maybe-no-member makes sense due to loading data from + # a serialized format. + # pylint: disable=maybe-no-member + entry.append(value) + return (True, self.yaml_dict) + + # pylint: disable=too-many-arguments + def update(self, path, value, index=None, curr_value=None): + ''' put path, value into a dict ''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if isinstance(entry, dict): + # AUDIT:maybe-no-member makes sense due to fuzzy types + # pylint: disable=maybe-no-member + if not isinstance(value, dict): + raise YeditException('Cannot replace key, value entry in dict with non-dict type. ' + + 'value=[{}] type=[{}]'.format(value, type(value))) + + entry.update(value) + return (True, self.yaml_dict) + + elif isinstance(entry, list): + # AUDIT:maybe-no-member makes sense due to fuzzy types + # pylint: disable=maybe-no-member + ind = None + if curr_value: + try: + ind = entry.index(curr_value) + except ValueError: + return (False, self.yaml_dict) + + elif index is not None: + ind = index + + if ind is not None and entry[ind] != value: + entry[ind] = value + return (True, self.yaml_dict) + + # see if it exists in the list + try: + ind = entry.index(value) + except ValueError: + # doesn't exist, append it + entry.append(value) + return (True, self.yaml_dict) + + # already exists, return + if ind is not None: + return (False, self.yaml_dict) + return (False, self.yaml_dict) + + def put(self, path, value): + ''' put path, value into a dict ''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry == value: + return (False, self.yaml_dict) + + # deepcopy didn't work + # Try to use ruamel.yaml and fallback to pyyaml + try: + tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, + default_flow_style=False), + yaml.RoundTripLoader) + except AttributeError: + tmp_copy = copy.deepcopy(self.yaml_dict) + + # set the format attributes if available + try: + tmp_copy.fa.set_block_style() + except AttributeError: + pass + + result = Yedit.add_entry(tmp_copy, path, value, self.separator) + if result is None: + return (False, self.yaml_dict) + + # When path equals "" it is a special case. + # "" refers to the root of the document + # Only update the root path (entire document) when its a list or dict + if path == '': + if isinstance(result, list) or isinstance(result, dict): + self.yaml_dict = result + return (True, self.yaml_dict) + + return (False, self.yaml_dict) + + self.yaml_dict = tmp_copy + + return (True, self.yaml_dict) + + def create(self, path, value): + ''' create a yaml file ''' + if not self.file_exists(): + # deepcopy didn't work + # Try to use ruamel.yaml and fallback to pyyaml + try: + tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, + default_flow_style=False), + yaml.RoundTripLoader) + except AttributeError: + tmp_copy = copy.deepcopy(self.yaml_dict) + + # set the format attributes if available + try: + tmp_copy.fa.set_block_style() + except AttributeError: + pass + + result = Yedit.add_entry(tmp_copy, path, value, self.separator) + if result is not None: + self.yaml_dict = tmp_copy + return (True, self.yaml_dict) + + return (False, self.yaml_dict) + + @staticmethod + def get_curr_value(invalue, val_type): + '''return the current value''' + if invalue is None: + return None + + curr_value = invalue + if val_type == 'yaml': + curr_value = yaml.load(invalue) + elif val_type == 'json': + curr_value = json.loads(invalue) + + return curr_value + + @staticmethod + def parse_value(inc_value, vtype=''): + '''determine value type passed''' + true_bools = ['y', 'Y', 'yes', 'Yes', 'YES', 'true', 'True', 'TRUE', + 'on', 'On', 'ON', ] + false_bools = ['n', 'N', 'no', 'No', 'NO', 'false', 'False', 'FALSE', + 'off', 'Off', 'OFF'] + + # It came in as a string but you didn't specify value_type as string + # we will convert to bool if it matches any of the above cases + if isinstance(inc_value, str) and 'bool' in vtype: + if inc_value not in true_bools and inc_value not in false_bools: + raise YeditException('Not a boolean type. str=[{}] vtype=[{}]'.format(inc_value, vtype)) + elif isinstance(inc_value, bool) and 'str' in vtype: + inc_value = str(inc_value) + + # There is a special case where '' will turn into None after yaml loading it so skip + if isinstance(inc_value, str) and inc_value == '': + pass + # If vtype is not str then go ahead and attempt to yaml load it. + elif isinstance(inc_value, str) and 'str' not in vtype: + try: + inc_value = yaml.safe_load(inc_value) + except Exception: + raise YeditException('Could not determine type of incoming value. ' + + 'value=[{}] vtype=[{}]'.format(type(inc_value), vtype)) + + return inc_value + + @staticmethod + def process_edits(edits, yamlfile): + '''run through a list of edits and process them one-by-one''' + results = [] + for edit in edits: + value = Yedit.parse_value(edit['value'], edit.get('value_type', '')) + if edit.get('action') == 'update': + # pylint: disable=line-too-long + curr_value = Yedit.get_curr_value( + Yedit.parse_value(edit.get('curr_value')), + edit.get('curr_value_format')) + + rval = yamlfile.update(edit['key'], + value, + edit.get('index'), + curr_value) + + elif edit.get('action') == 'append': + rval = yamlfile.append(edit['key'], value) + + else: + rval = yamlfile.put(edit['key'], value) + + if rval[0]: + results.append({'key': edit['key'], 'edit': rval[1]}) + + return {'changed': len(results) > 0, 'results': results} + + # pylint: disable=too-many-return-statements,too-many-branches + @staticmethod + def run_ansible(params): + '''perform the idempotent crud operations''' + yamlfile = Yedit(filename=params['src'], + backup=params['backup'], + separator=params['separator']) + + state = params['state'] + + if params['src']: + rval = yamlfile.load() + + if yamlfile.yaml_dict is None and state != 'present': + return {'failed': True, + 'msg': 'Error opening file [{}]. Verify that the '.format(params['src']) + + 'file exists, that it is has correct permissions, and is valid yaml.'} + + if state == 'list': + if params['content']: + content = Yedit.parse_value(params['content'], params['content_type']) + yamlfile.yaml_dict = content + + if params['key']: + rval = yamlfile.get(params['key']) + + return {'changed': False, 'result': rval, 'state': state} + + elif state == 'absent': + if params['content']: + content = Yedit.parse_value(params['content'], params['content_type']) + yamlfile.yaml_dict = content + + if params['update']: + rval = yamlfile.pop(params['key'], params['value']) + else: + rval = yamlfile.delete(params['key']) + + if rval[0] and params['src']: + yamlfile.write() + + return {'changed': rval[0], 'result': rval[1], 'state': state} + + elif state == 'present': + # check if content is different than what is in the file + if params['content']: + content = Yedit.parse_value(params['content'], params['content_type']) + + # We had no edits to make and the contents are the same + if yamlfile.yaml_dict == content and \ + params['value'] is None: + return {'changed': False, 'result': yamlfile.yaml_dict, 'state': state} + + yamlfile.yaml_dict = content + + # If we were passed a key, value then + # we enapsulate it in a list and process it + # Key, Value passed to the module : Converted to Edits list # + edits = [] + _edit = {} + if params['value'] is not None: + _edit['value'] = params['value'] + _edit['value_type'] = params['value_type'] + _edit['key'] = params['key'] + + if params['update']: + _edit['action'] = 'update' + _edit['curr_value'] = params['curr_value'] + _edit['curr_value_format'] = params['curr_value_format'] + _edit['index'] = params['index'] + + elif params['append']: + _edit['action'] = 'append' + + edits.append(_edit) + + elif params['edits'] is not None: + edits = params['edits'] + + if edits: + results = Yedit.process_edits(edits, yamlfile) + + # if there were changes and a src provided to us we need to write + if results['changed'] and params['src']: + yamlfile.write() + + return {'changed': results['changed'], 'result': results['results'], 'state': state} + + # no edits to make + if params['src']: + # pylint: disable=redefined-variable-type + rval = yamlfile.write() + return {'changed': rval[0], + 'result': rval[1], + 'state': state} + + # We were passed content but no src, key or value, or edits. Return contents in memory + return {'changed': False, 'result': yamlfile.yaml_dict, 'state': state} + return {'failed': True, 'msg': 'Unkown state passed'} diff --git a/roles/lib_utils/src/doc/generated b/roles/lib_utils/src/doc/generated new file mode 100644 index 000000000..054780313 --- /dev/null +++ b/roles/lib_utils/src/doc/generated @@ -0,0 +1,9 @@ +#!/usr/bin/env python +# pylint: disable=missing-docstring +# ___ ___ _ _ ___ ___ _ _____ ___ ___ +# / __| __| \| | __| _ \ /_\_ _| __| \ +# | (_ | _|| .` | _|| / / _ \| | | _|| |) | +# \___|___|_|\_|___|_|_\/_/_\_\_|_|___|___/_ _____ +# | \ / _ \ | \| |/ _ \_ _| | __| \_ _|_ _| +# | |) | (_) | | .` | (_) || | | _|| |) | | | | +# |___/ \___/ |_|\_|\___/ |_| |___|___/___| |_| diff --git a/roles/lib_utils/src/doc/license b/roles/lib_utils/src/doc/license new file mode 100644 index 000000000..717bb7f17 --- /dev/null +++ b/roles/lib_utils/src/doc/license @@ -0,0 +1,16 @@ +# +# Copyright 2016 Red Hat, Inc. and/or its affiliates +# and other contributors as indicated by the @author tags. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/roles/lib_utils/src/doc/repoquery b/roles/lib_utils/src/doc/repoquery new file mode 100644 index 000000000..82e273a42 --- /dev/null +++ b/roles/lib_utils/src/doc/repoquery @@ -0,0 +1,275 @@ +# flake8: noqa +# pylint: skip-file + +DOCUMENTATION = ''' +--- +module: repoquery +short_description: Query package information from Yum repositories +description: + - Query package information from Yum repositories. +options: + state: + description: + - The expected state. Currently only supports list. + required: false + default: list + choices: ["list"] + aliases: [] + name: + description: + - The name of the package to query + required: true + default: None + aliases: [] + query_type: + description: + - Narrows the packages queried based off of this value. + - If repos, it narrows the query to repositories defined on the machine. + - If installed, it narrows the query to only packages installed on the machine. + - If available, it narrows the query to packages that are available to be installed. + - If recent, it narrows the query to only recently edited packages. + - If updates, it narrows the query to only packages that are updates to existing installed packages. + - If extras, it narrows the query to packages that are not present in any of the available repositories. + - If all, it queries all of the above. + required: false + default: repos + aliases: [] + verbose: + description: + - Shows more detail for the requested query. + required: false + default: false + aliases: [] + show_duplicates: + description: + - Shows multiple versions of a package. + required: false + default: false + aliases: [] + match_version: + description: + - Match the specific version given to the package. + required: false + default: None + aliases: [] +author: +- "Matt Woodson <mwoodson@redhat.com>" +extends_documentation_fragment: [] +''' + +EXAMPLES = ''' +# Example 1: Get bash versions + - name: Get bash version + repoquery: + name: bash + show_duplicates: True + register: bash_out + +# Results: +# ok: [localhost] => { +# "bash_out": { +# "changed": false, +# "results": { +# "cmd": "/usr/bin/repoquery --quiet --pkgnarrow=repos --queryformat=%{version}|%{release}|%{arch}|%{repo}|%{version}-%{release} --show-duplicates bash", +# "package_found": true, +# "package_name": "bash", +# "returncode": 0, +# "versions": { +# "available_versions": [ +# "4.2.45", +# "4.2.45", +# "4.2.45", +# "4.2.46", +# "4.2.46", +# "4.2.46", +# "4.2.46" +# ], +# "available_versions_full": [ +# "4.2.45-5.el7", +# "4.2.45-5.el7_0.2", +# "4.2.45-5.el7_0.4", +# "4.2.46-12.el7", +# "4.2.46-19.el7", +# "4.2.46-20.el7_2", +# "4.2.46-21.el7_3" +# ], +# "latest": "4.2.46", +# "latest_full": "4.2.46-21.el7_3" +# } +# }, +# "state": "present" +# } +# } + + + +# Example 2: Get bash versions verbosely + - name: Get bash versions verbosely + repoquery: + name: bash + show_duplicates: True + verbose: True + register: bash_out + +# Results: +# ok: [localhost] => { +# "bash_out": { +# "changed": false, +# "results": { +# "cmd": "/usr/bin/repoquery --quiet --pkgnarrow=repos --queryformat=%{version}|%{release}|%{arch}|%{repo}|%{version}-%{release} --show-duplicates bash", +# "package_found": true, +# "package_name": "bash", +# "raw_versions": { +# "4.2.45-5.el7": { +# "arch": "x86_64", +# "release": "5.el7", +# "repo": "rhel-7-server-rpms", +# "version": "4.2.45", +# "version_release": "4.2.45-5.el7" +# }, +# "4.2.45-5.el7_0.2": { +# "arch": "x86_64", +# "release": "5.el7_0.2", +# "repo": "rhel-7-server-rpms", +# "version": "4.2.45", +# "version_release": "4.2.45-5.el7_0.2" +# }, +# "4.2.45-5.el7_0.4": { +# "arch": "x86_64", +# "release": "5.el7_0.4", +# "repo": "rhel-7-server-rpms", +# "version": "4.2.45", +# "version_release": "4.2.45-5.el7_0.4" +# }, +# "4.2.46-12.el7": { +# "arch": "x86_64", +# "release": "12.el7", +# "repo": "rhel-7-server-rpms", +# "version": "4.2.46", +# "version_release": "4.2.46-12.el7" +# }, +# "4.2.46-19.el7": { +# "arch": "x86_64", +# "release": "19.el7", +# "repo": "rhel-7-server-rpms", +# "version": "4.2.46", +# "version_release": "4.2.46-19.el7" +# }, +# "4.2.46-20.el7_2": { +# "arch": "x86_64", +# "release": "20.el7_2", +# "repo": "rhel-7-server-rpms", +# "version": "4.2.46", +# "version_release": "4.2.46-20.el7_2" +# }, +# "4.2.46-21.el7_3": { +# "arch": "x86_64", +# "release": "21.el7_3", +# "repo": "rhel-7-server-rpms", +# "version": "4.2.46", +# "version_release": "4.2.46-21.el7_3" +# } +# }, +# "results": "4.2.45|5.el7|x86_64|rhel-7-server-rpms|4.2.45-5.el7\n4.2.45|5.el7_0.2|x86_64|rhel-7-server-rpms|4.2.45-5.el7_0.2\n4.2.45|5.el7_0.4|x86_64|rhel-7-server-rpms|4.2.45-5.el7_0.4\n4.2.46|12.el7|x86_64|rhel-7-server-rpms|4.2.46-12.el7\n4.2.46|19.el7|x86_64|rhel-7-server-rpms|4.2.46-19.el7\n4.2.46|20.el7_2|x86_64|rhel-7-server-rpms|4.2.46-20.el7_2\n4.2.46|21.el7_3|x86_64|rhel-7-server-rpms|4.2.46-21.el7_3\n", +# "returncode": 0, +# "versions": { +# "available_versions": [ +# "4.2.45", +# "4.2.45", +# "4.2.45", +# "4.2.46", +# "4.2.46", +# "4.2.46", +# "4.2.46" +# ], +# "available_versions_full": [ +# "4.2.45-5.el7", +# "4.2.45-5.el7_0.2", +# "4.2.45-5.el7_0.4", +# "4.2.46-12.el7", +# "4.2.46-19.el7", +# "4.2.46-20.el7_2", +# "4.2.46-21.el7_3" +# ], +# "latest": "4.2.46", +# "latest_full": "4.2.46-21.el7_3" +# } +# }, +# "state": "present" +# } +# } + +# Example 3: Match a specific version + - name: matched versions repoquery test + repoquery: + name: atomic-openshift + show_duplicates: True + match_version: 3.3 + register: openshift_out + +# Result: + +# ok: [localhost] => { +# "openshift_out": { +# "changed": false, +# "results": { +# "cmd": "/usr/bin/repoquery --quiet --pkgnarrow=repos --queryformat=%{version}|%{release}|%{arch}|%{repo}|%{version}-%{release} --show-duplicates atomic-openshift", +# "package_found": true, +# "package_name": "atomic-openshift", +# "returncode": 0, +# "versions": { +# "available_versions": [ +# "3.2.0.43", +# "3.2.1.23", +# "3.3.0.32", +# "3.3.0.34", +# "3.3.0.35", +# "3.3.1.3", +# "3.3.1.4", +# "3.3.1.5", +# "3.3.1.7", +# "3.4.0.39" +# ], +# "available_versions_full": [ +# "3.2.0.43-1.git.0.672599f.el7", +# "3.2.1.23-1.git.0.88a7a1d.el7", +# "3.3.0.32-1.git.0.37bd7ea.el7", +# "3.3.0.34-1.git.0.83f306f.el7", +# "3.3.0.35-1.git.0.d7bd9b6.el7", +# "3.3.1.3-1.git.0.86dc49a.el7", +# "3.3.1.4-1.git.0.7c8657c.el7", +# "3.3.1.5-1.git.0.62700af.el7", +# "3.3.1.7-1.git.0.0988966.el7", +# "3.4.0.39-1.git.0.5f32f06.el7" +# ], +# "latest": "3.4.0.39", +# "latest_full": "3.4.0.39-1.git.0.5f32f06.el7", +# "matched_version_found": true, +# "matched_version_full_latest": "3.3.1.7-1.git.0.0988966.el7", +# "matched_version_latest": "3.3.1.7", +# "matched_versions": [ +# "3.3.0.32", +# "3.3.0.34", +# "3.3.0.35", +# "3.3.1.3", +# "3.3.1.4", +# "3.3.1.5", +# "3.3.1.7" +# ], +# "matched_versions_full": [ +# "3.3.0.32-1.git.0.37bd7ea.el7", +# "3.3.0.34-1.git.0.83f306f.el7", +# "3.3.0.35-1.git.0.d7bd9b6.el7", +# "3.3.1.3-1.git.0.86dc49a.el7", +# "3.3.1.4-1.git.0.7c8657c.el7", +# "3.3.1.5-1.git.0.62700af.el7", +# "3.3.1.7-1.git.0.0988966.el7" +# ], +# "requested_match_version": "3.3" +# } +# }, +# "state": "present" +# } +# } + +''' diff --git a/roles/lib_utils/src/doc/yedit b/roles/lib_utils/src/doc/yedit new file mode 100644 index 000000000..82af1f675 --- /dev/null +++ b/roles/lib_utils/src/doc/yedit @@ -0,0 +1,154 @@ +# flake8: noqa +# pylint: skip-file + +DOCUMENTATION = ''' +--- +module: yedit +short_description: Create, modify, and idempotently manage yaml files. +description: + - Modify yaml files programmatically. +options: + state: + description: + - State represents whether to create, modify, delete, or list yaml + required: true + default: present + choices: ["present", "absent", "list"] + aliases: [] + debug: + description: + - Turn on debug information. + required: false + default: false + aliases: [] + src: + description: + - The file that is the target of the modifications. + required: false + default: None + aliases: [] + content: + description: + - Content represents the yaml content you desire to work with. This + - could be the file contents to write or the inmemory data to modify. + required: false + default: None + aliases: [] + content_type: + description: + - The python type of the content parameter. + required: false + default: 'dict' + aliases: [] + key: + description: + - The path to the value you wish to modify. Emtpy string means the top of + - the document. + required: false + default: '' + aliases: [] + value: + description: + - The incoming value of parameter 'key'. + required: false + default: + aliases: [] + value_type: + description: + - The python type of the incoming value. + required: false + default: '' + aliases: [] + update: + description: + - Whether the update should be performed on a dict/hash or list/array + - object. + required: false + default: false + aliases: [] + append: + description: + - Whether to append to an array/list. When the key does not exist or is + - null, a new array is created. When the key is of a non-list type, + - nothing is done. + required: false + default: false + aliases: [] + index: + description: + - Used in conjunction with the update parameter. This will update a + - specific index in an array/list. + required: false + default: false + aliases: [] + curr_value: + description: + - Used in conjunction with the update parameter. This is the current + - value of 'key' in the yaml file. + required: false + default: false + aliases: [] + curr_value_format: + description: + - Format of the incoming current value. + choices: ["yaml", "json", "str"] + required: false + default: false + aliases: [] + backup: + description: + - Whether to make a backup copy of the current file when performing an + - edit. + required: false + default: true + aliases: [] + separator: + description: + - The separator being used when parsing strings. + required: false + default: '.' + aliases: [] +author: +- "Kenny Woodson <kwoodson@redhat.com>" +extends_documentation_fragment: [] +''' + +EXAMPLES = ''' +# Simple insert of key, value +- name: insert simple key, value + yedit: + src: somefile.yml + key: test + value: somevalue + state: present +# Results: +# test: somevalue + +# Multilevel insert of key, value +- name: insert simple key, value + yedit: + src: somefile.yml + key: a#b#c + value: d + state: present +# Results: +# a: +# b: +# c: d +# +# multiple edits at the same time +- name: perform multiple edits + yedit: + src: somefile.yml + edits: + - key: a#b#c + value: d + - key: a#b#c#d + value: e + state: present +# Results: +# a: +# b: +# c: +# d: e +''' diff --git a/roles/lib_utils/src/generate.py b/roles/lib_utils/src/generate.py new file mode 100755 index 000000000..3f23455b5 --- /dev/null +++ b/roles/lib_utils/src/generate.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +''' + Generate the openshift-ansible/roles/lib_openshift_cli/library/ modules. +''' + +import argparse +import os +import yaml +import six + +OPENSHIFT_ANSIBLE_PATH = os.path.dirname(os.path.realpath(__file__)) +OPENSHIFT_ANSIBLE_SOURCES_PATH = os.path.join(OPENSHIFT_ANSIBLE_PATH, 'sources.yml') # noqa: E501 +LIBRARY = os.path.join(OPENSHIFT_ANSIBLE_PATH, '..', 'library/') + + +class GenerateAnsibleException(Exception): + '''General Exception for generate function''' + pass + + +def parse_args(): + '''parse arguments to generate''' + parser = argparse.ArgumentParser(description="Generate ansible modules.") + parser.add_argument('--verify', action='store_true', default=False, + help='Verify library code matches the generated code.') + + return parser.parse_args() + + +def fragment_banner(fragment_path, side, data): + """Generate a banner to wrap around file fragments + +:param string fragment_path: A path to a module fragment +:param string side: ONE OF: "header", "footer" +:param StringIO data: A StringIO object to write the banner to +""" + side_msg = { + "header": "Begin included fragment: {}", + "footer": "End included fragment: {}" + } + annotation = side_msg[side].format(fragment_path) + + banner = """ +# -*- -*- -*- {} -*- -*- -*- +""".format(annotation) + + # Why skip? + # + # * 'generated' - This is the head of the script, we don't want to + # put comments before the #!shebang + # + # * 'license' - Wrapping this just seemed like gratuitous extra + if ("generated" not in fragment_path) and ("license" not in fragment_path): + data.write(banner) + + # Make it self-contained testable + return banner + + +def generate(parts): + '''generate the source code for the ansible modules + +:param Array parts: An array of paths (strings) to module fragments + ''' + + data = six.StringIO() + for fpart in parts: + # first line is pylint disable so skip it + with open(os.path.join(OPENSHIFT_ANSIBLE_PATH, fpart)) as pfd: + fragment_banner(fpart, "header", data) + for idx, line in enumerate(pfd): + if idx in [0, 1] and 'flake8: noqa' in line or 'pylint: skip-file' in line: # noqa: E501 + continue + + data.write(line) + + fragment_banner(fpart, "footer", data) + return data + + +def get_sources(): + '''return the path to the generate sources''' + return yaml.load(open(OPENSHIFT_ANSIBLE_SOURCES_PATH).read()) + + +def verify(): + '''verify if the generated code matches the library code''' + for fname, parts in get_sources().items(): + data = generate(parts) + fname = os.path.join(LIBRARY, fname) + if not open(fname).read() == data.getvalue(): + raise GenerateAnsibleException('Generated content does not match for %s' % fname) + + +def main(): + ''' combine the necessary files to create the ansible module ''' + args = parse_args() + if args.verify: + verify() + + for fname, parts in get_sources().items(): + data = generate(parts) + fname = os.path.join(LIBRARY, fname) + with open(fname, 'w') as afd: + afd.seek(0) + afd.write(data.getvalue()) + + +if __name__ == '__main__': + main() diff --git a/roles/lib_utils/src/lib/import.py b/roles/lib_utils/src/lib/import.py new file mode 100644 index 000000000..07a04b7ae --- /dev/null +++ b/roles/lib_utils/src/lib/import.py @@ -0,0 +1,20 @@ +# flake8: noqa +# pylint: skip-file + +# pylint: disable=wrong-import-order,wrong-import-position,unused-import + +from __future__ import print_function # noqa: F401 +import copy # noqa: F401 +import json # noqa: F401 +import os # noqa: F401 +import re # noqa: F401 +import shutil # noqa: F401 +import tempfile # noqa: F401 +import time # noqa: F401 + +try: + import ruamel.yaml as yaml # noqa: F401 +except ImportError: + import yaml # noqa: F401 + +from ansible.module_utils.basic import AnsibleModule diff --git a/roles/lib_utils/src/lib/repoquery.py b/roles/lib_utils/src/lib/repoquery.py new file mode 100644 index 000000000..91ccd9815 --- /dev/null +++ b/roles/lib_utils/src/lib/repoquery.py @@ -0,0 +1,92 @@ +# pylint: skip-file +# flake8: noqa + +''' + class that wraps the repoquery commands in a subprocess +''' + +# pylint: disable=too-many-lines,wrong-import-position,wrong-import-order + +from collections import defaultdict # noqa: E402 + + +# pylint: disable=no-name-in-module,import-error +# Reason: pylint errors with "No name 'version' in module 'distutils'". +# This is a bug: https://github.com/PyCQA/pylint/issues/73 +from distutils.version import LooseVersion # noqa: E402 + +import subprocess # noqa: E402 + + +class RepoqueryCLIError(Exception): + '''Exception class for repoquerycli''' + pass + + +def _run(cmds): + ''' Actually executes the command. This makes mocking easier. ''' + proc = subprocess.Popen(cmds, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + stdout, stderr = proc.communicate() + + return proc.returncode, stdout, stderr + + +# pylint: disable=too-few-public-methods +class RepoqueryCLI(object): + ''' Class to wrap the command line tools ''' + def __init__(self, + verbose=False): + ''' Constructor for RepoqueryCLI ''' + self.verbose = verbose + self.verbose = True + + def _repoquery_cmd(self, cmd, output=False, output_type='json'): + '''Base command for repoquery ''' + cmds = ['/usr/bin/repoquery', '--plugins', '--quiet'] + + cmds.extend(cmd) + + rval = {} + results = '' + err = None + + if self.verbose: + print(' '.join(cmds)) + + returncode, stdout, stderr = _run(cmds) + + rval = { + "returncode": returncode, + "results": results, + "cmd": ' '.join(cmds), + } + + if returncode == 0: + if output: + if output_type == 'raw': + rval['results'] = stdout + + if self.verbose: + print(stdout) + print(stderr) + + if err: + rval.update({ + "err": err, + "stderr": stderr, + "stdout": stdout, + "cmd": cmds + }) + + else: + rval.update({ + "stderr": stderr, + "stdout": stdout, + "results": {}, + }) + + return rval diff --git a/roles/lib_utils/src/sources.yml b/roles/lib_utils/src/sources.yml new file mode 100644 index 000000000..053b59f77 --- /dev/null +++ b/roles/lib_utils/src/sources.yml @@ -0,0 +1,17 @@ +--- +yedit.py: +- doc/generated +- doc/license +- lib/import.py +- doc/yedit +- class/yedit.py +- ansible/yedit.py + +repoquery.py: +- doc/generated +- doc/license +- lib/import.py +- doc/repoquery +- lib/repoquery.py +- class/repoquery.py +- ansible/repoquery.py diff --git a/roles/lib_utils/src/test/generate-and-run-tests.sh b/roles/lib_utils/src/test/generate-and-run-tests.sh new file mode 100755 index 000000000..4b534c8f2 --- /dev/null +++ b/roles/lib_utils/src/test/generate-and-run-tests.sh @@ -0,0 +1,42 @@ +#!/bin/bash -e + + +# Put us in the same dir as the script. +cd $(dirname $0) + +echo +echo "Running lib_openshift generate" +echo "------------------------------" +../generate.py + + +echo +echo "Running lib_utils Unit Tests" +echo "----------------------------" +cd unit + +for test in *.py; do + echo + echo "--------------------------------------------------------------------------------" + echo + echo "Running $test..." + ./$test +done + + +echo +echo "Running lib_utils Integration Tests" +echo "-----------------------------------" +cd ../integration + +for test in *.yml; do + echo + echo "--------------------------------------------------------------------------------" + echo + echo "Running $test..." + ./$test -vvv +done + +# Clean up this damn file +# TODO: figure out why this is being written and clean it up. +rm kube-manager-test.yaml diff --git a/roles/lib_utils/src/test/integration/files/kube-manager.yaml b/roles/lib_utils/src/test/integration/files/kube-manager.yaml new file mode 100644 index 000000000..6f4b9e6dc --- /dev/null +++ b/roles/lib_utils/src/test/integration/files/kube-manager.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: kube-controller-manager + namespace: kube-system +spec: + hostNetwork: true + containers: + - name: kube-controller-manager + image: openshift/kube:v1.0.0 + command: + - /hyperkube + - controller-manager + - --master=http://127.0.0.1:8080 + - --leader-elect=true + - --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem + - --root-ca-file=/etc/kubernetes/ssl/ca.pem + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10252 + initialDelaySeconds: 15 + timeoutSeconds: 1 + volumeMounts: + - mountPath: /etc/kubernetes/ssl + name: ssl-certs-kubernetes + readOnly: true + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + volumes: + - hostPath: + path: /etc/kubernetes/ssl + name: ssl-certs-kubernetes + - hostPath: + path: /usr/share/ca-certificates + name: ssl-certs-host diff --git a/roles/lib_utils/src/test/integration/repoquery.yml b/roles/lib_utils/src/test/integration/repoquery.yml new file mode 100755 index 000000000..425324387 --- /dev/null +++ b/roles/lib_utils/src/test/integration/repoquery.yml @@ -0,0 +1,136 @@ +#!/usr/bin/ansible-playbook --module-path=../../../library/ +--- +- hosts: localhost + gather_facts: no + + tasks: + - name: basic query test - Act + repoquery: + name: bash + register: rq_out + + - name: Set a real package version to be used later + set_fact: + latest_available_bash_version: "{{ rq_out.results.versions.latest }}" + latest_available_full_bash_version: "{{ rq_out.results.versions.latest_full }}" + + - name: basic query test - Assert + assert: + that: + - "rq_out.state == 'list'" + - "rq_out.changed == False" + - "rq_out.results.returncode == 0" + - "rq_out.results.package_found == True" + - "rq_out.results.package_name == 'bash'" + - "rq_out.results.versions.available_versions | length == 1" + - "rq_out.results.versions.available_versions_full | length == 1" + - "rq_out.results.versions.latest is defined" + - "rq_out.results.versions.latest in rq_out.results.versions.available_versions" + - "rq_out.results.versions.latest_full is defined" + - "rq_out.results.versions.latest_full in rq_out.results.versions.available_versions_full" + + - name: show_duplicates query test - Act + repoquery: + name: bash + show_duplicates: True + register: rq_out + + - name: show_duplicates query test - Assert + assert: + that: + - "rq_out.state == 'list'" + - "rq_out.changed == False" + - "rq_out.results.returncode == 0" + - "rq_out.results.package_found == True" + - "rq_out.results.package_name == 'bash'" + - "rq_out.results.versions.available_versions | length >= 1" + - "rq_out.results.versions.available_versions_full | length >= 1" + - "rq_out.results.versions.latest is defined" + - "rq_out.results.versions.latest in rq_out.results.versions.available_versions" + - "rq_out.results.versions.latest_full is defined" + - "rq_out.results.versions.latest_full in rq_out.results.versions.available_versions_full" + + - name: show_duplicates verbose query test - Act + repoquery: + name: bash + show_duplicates: True + verbose: True + register: rq_out + + - name: show_duplicates verbose query test - Assert + assert: + that: + - "rq_out.state == 'list'" + - "rq_out.changed == False" + - "rq_out.results.returncode == 0" + - "rq_out.results.package_found == True" + - "rq_out.results.package_name == 'bash'" + - "rq_out.results.raw_versions | length > 0" + - "rq_out.results.versions.available_versions | length > 0" + - "rq_out.results.versions.available_versions_full | length > 0" + - "rq_out.results.versions.latest is defined" + - "rq_out.results.versions.latest in rq_out.results.versions.available_versions" + - "rq_out.results.versions.latest_full is defined" + - "rq_out.results.versions.latest_full in rq_out.results.versions.available_versions_full" + + - name: query package does not exist query test - Act + repoquery: + name: somemadeuppackagenamethatwontmatch + show_duplicates: True + register: rq_out + + - name: query package does not exist query test - Assert + assert: + that: + - "rq_out.state == 'list'" + - "rq_out.changed == False" + - "rq_out.results.returncode == 0" + - "rq_out.results.package_found == False" + - "rq_out.results.results == ''" + + + - name: query match_version does not exist query test - Act + repoquery: + name: bash + show_duplicates: True + match_version: somemadeupversionnotexist + register: rq_out + + - name: query match_version does not exist query test - Assert + assert: + that: + - "rq_out.state == 'list'" + - "rq_out.changed == False" + - "rq_out.results.returncode == 0" + - "rq_out.results.package_found == True" + - "rq_out.results.package_name == 'bash'" + - "rq_out.results.versions.matched_version_found == False" + - "rq_out.results.versions.available_versions | length > 0" + - "rq_out.results.versions.available_versions_full | length > 0" + - "rq_out.results.versions.latest is defined" + - "rq_out.results.versions.latest in rq_out.results.versions.available_versions" + - "rq_out.results.versions.latest_full is defined" + - "rq_out.results.versions.latest_full in rq_out.results.versions.available_versions_full" + + - name: query match_version exists query test - Act + repoquery: + name: bash + show_duplicates: True + match_version: "{{ latest_available_bash_version }}" + register: rq_out + + - name: query match_version exists query test - Assert + assert: + that: + - "rq_out.state == 'list'" + - "rq_out.changed == False" + - "rq_out.results.returncode == 0" + - "rq_out.results.package_found == True" + - "rq_out.results.package_name == 'bash'" + - "rq_out.results.versions.matched_version_found == True" + - "rq_out.results.versions.available_versions | length > 0" + - "rq_out.results.versions.available_versions_full | length > 0" + - "rq_out.results.versions.latest is defined" + - "rq_out.results.versions.latest in rq_out.results.versions.available_versions" + - "rq_out.results.versions.latest_full is defined" + - "rq_out.results.versions.latest_full in rq_out.results.versions.available_versions_full" diff --git a/roles/lib_utils/src/test/integration/yedit.yml b/roles/lib_utils/src/test/integration/yedit.yml new file mode 100755 index 000000000..65209bade --- /dev/null +++ b/roles/lib_utils/src/test/integration/yedit.yml @@ -0,0 +1,251 @@ +#!/usr/bin/ansible-playbook --module-path=../../../library/ +# +# Yedit test so that we can quickly determine if features are working +# Ensure that the kube-manager.yaml file exists +# +# ./yedit_test.yml +# +--- +- hosts: localhost + gather_facts: no + vars: + test_file: kube-manager-test.yaml + test: test + strategy: debug + + post_tasks: + - name: copy the kube-manager.yaml file so that we have a pristine copy each time + copy: + src: kube-manager.yaml + dest: "./{{ test_file }}" + changed_when: False + + ####### add key to top level ##### + - name: add a key at the top level + yedit: + src: "{{ test_file }}" + key: yedittest + value: yedittest + + - name: retrieve the inserted key + yedit: + src: "{{ test_file }}" + state: list + key: yedittest + register: results + + - name: Assert that key is at top level + assert: + that: results.result == 'yedittest' + msg: 'Test: add a key to top level failed. yedittest != [{{ results.result }}]' + ###### end add key to top level ##### + + ###### modify multilevel key, value ##### + - name: modify multilevel key, value + yedit: + src: "{{ test_file }}" + key: metadata-namespace + value: openshift-is-awesome + separator: '-' + + - name: retrieve the inserted key + yedit: + src: "{{ test_file }}" + state: list + key: metadata-namespace + separator: '-' + register: results + + - name: Assert that key is as expected + assert: + that: results.result == 'openshift-is-awesome' + msg: 'Test: multilevel key, value modification: openshift-is-awesome != [{{ results.result }}]' + ###### end modify multilevel key, value ##### + + ###### test a string boolean ##### + - name: test a string boolean + yedit: + src: "{{ test_file }}" + key: spec.containers[0].volumeMounts[1].readOnly + value: 'true' + value_type: str + + - name: retrieve the inserted key + yedit: + src: "{{ test_file }}" + state: list + key: spec.containers[0].volumeMounts[1].readOnly + register: results + + - name: Assert that key is a string + assert: + that: results.result == "true" + msg: "Test: boolean str: 'true' != [{{ results.result }}]" + + - name: Assert that key is not bool + assert: + that: results.result != true + msg: "Test: boolean str: true != [{{ results.result }}]" + ###### end test boolean string ##### + + ###### test array append ##### + - name: test array append + yedit: + src: "{{ test_file }}" + key: spec.containers[0].command + value: --my-new-parameter=openshift + append: True + + - name: retrieve the array + yedit: + src: "{{ test_file }}" + state: list + key: spec.containers[0].command + register: results + + - name: Assert that the last element in array is our value + assert: + that: results.result[-1] == "--my-new-parameter=openshift" + msg: "Test: '--my-new-parameter=openshift' != [{{ results.result[-1] }}]" + ###### end test array append ##### + + ###### test non-existing array append ##### + - name: test array append to non-existing key + yedit: + src: "{{ test_file }}" + key: nonexistingkey + value: --my-new-parameter=openshift + append: True + + - name: retrieve the array + yedit: + src: "{{ test_file }}" + state: list + key: nonexistingkey + register: results + + - name: Assert that the last element in array is our value + assert: + that: results.result[-1] == "--my-new-parameter=openshift" + msg: "Test: '--my-new-parameter=openshift' != [{{ results.result[-1] }}]" + ###### end test non-existing array append ##### + + ###### test array update modify ##### + - name: test array update modify + yedit: + src: "{{ test_file }}" + key: spec.containers[0].command + value: --root-ca-file=/etc/k8s/ssl/my.pem + curr_value: --root-ca-file=/etc/kubernetes/ssl/ca.pem + curr_value_format: str + update: True + + - name: retrieve the array + yedit: + src: "{{ test_file }}" + state: list + key: spec.containers[0].command + register: results + + - name: Assert that the element in array is our value + assert: + that: results.result[5] == "--root-ca-file=/etc/k8s/ssl/my.pem" + msg: "Test: '--root-ca-file=/etc/k8s/ssl/my.pem' != [{{ results.result[5] }}]" + ###### end test array update modify##### + + ###### test dict create ##### + - name: test dict create + yedit: + src: "{{ test_file }}" + key: a.b.c + value: d + + - name: retrieve the key + yedit: + src: "{{ test_file }}" + state: list + key: a.b.c + register: results + + - name: Assert that the key was created + assert: + that: results.result == "d" + msg: "Test: 'd' != [{{ results.result }}]" + ###### end test dict create ##### + + ###### test create dict value ##### + - name: test create dict value + yedit: + src: "{{ test_file }}" + key: e.f.g + value: + h: + i: + j: k + + - name: retrieve the key + yedit: + src: "{{ test_file }}" + state: list + key: e.f.g.h.i.j + register: results + + - name: Assert that the key was created + assert: + that: results.result == "k" + msg: "Test: 'k' != [{{ results.result }}]" + ###### end test dict create ##### + + ###### test create list value ##### + - name: test create list value + yedit: + src: "{{ test_file }}" + key: z.x.y + value: + - 1 + - 2 + - 3 + + - name: retrieve the key + yedit: + src: "{{ test_file }}" + state: list + key: z#x#y + separator: '#' + register: results + - debug: var=results + + - name: Assert that the key was created + assert: + that: results.result == [1, 2, 3] + msg: "Test: '[1, 2, 3]' != [{{ results.result }}]" + ###### end test create list value ##### + + ###### test create multiple list value ##### + - name: test multiple edits + yedit: + src: "{{ test_file }}" + edits: + - key: z.x.y + value: + - 1 + - 2 + - 3 + - key: z.x.y + value: 4 + action: append + + - name: retrieve the key + yedit: + src: "{{ test_file }}" + state: list + key: z#x#y + separator: '#' + register: results + - debug: var=results + + - name: Assert that the key was created + assert: + that: results.result == [1, 2, 3, 4] + msg: "Test: '[1, 2, 3, 4]' != [{{ results.result }}]" + ###### end test create multiple list value ##### diff --git a/roles/lib_utils/src/test/unit/test_repoquery.py b/roles/lib_utils/src/test/unit/test_repoquery.py new file mode 100755 index 000000000..325f41dab --- /dev/null +++ b/roles/lib_utils/src/test/unit/test_repoquery.py @@ -0,0 +1,68 @@ +''' + Unit tests for repoquery +''' + +import os +import sys +import unittest +import mock + +# Removing invalid variable names for tests so that I can +# keep them brief +# pylint: disable=invalid-name,no-name-in-module +# Disable import-error b/c our libraries aren't loaded in jenkins +# pylint: disable=import-error,wrong-import-position +# place class in our python path +module_path = os.path.join('/'.join(os.path.realpath(__file__).split('/')[:-4]), 'library') # noqa: E501 +sys.path.insert(0, module_path) +from repoquery import Repoquery # noqa: E402 + + +class RepoQueryTest(unittest.TestCase): + ''' + Test class for RepoQuery + ''' + + @mock.patch('repoquery._run') + def test_querying_a_package(self, mock_cmd): + ''' Testing querying a package ''' + + # Arrange + + # run_ansible input parameters + params = { + 'state': 'list', + 'name': 'bash', + 'query_type': 'repos', + 'verbose': False, + 'show_duplicates': False, + 'match_version': None, + 'ignore_excluders': False, + } + + valid_stderr = '''Repo rhel-7-server-extras-rpms forced skip_if_unavailable=True due to: /etc/pki/entitlement/3268107132875399464-key.pem + Repo rhel-7-server-rpms forced skip_if_unavailable=True due to: /etc/pki/entitlement/4128505182875899164-key.pem''' # not real + + # Return values of our mocked function call. These get returned once per call. + mock_cmd.side_effect = [ + (0, b'4.2.46|21.el7_3|x86_64|rhel-7-server-rpms|4.2.46-21.el7_3', valid_stderr), # first call to the mock + ] + + # Act + results = Repoquery.run_ansible(params, False) + + # Assert + self.assertEqual(results['state'], 'list') + self.assertFalse(results['changed']) + self.assertTrue(results['results']['package_found']) + self.assertEqual(results['results']['returncode'], 0) + self.assertEqual(results['results']['package_name'], 'bash') + self.assertEqual(results['results']['versions'], {'latest_full': '4.2.46-21.el7_3', + 'available_versions': ['4.2.46'], + 'available_versions_full': ['4.2.46-21.el7_3'], + 'latest': '4.2.46'}) + + # Making sure our mock was called as we expected + mock_cmd.assert_has_calls([ + mock.call(['/usr/bin/repoquery', '--plugins', '--quiet', '--pkgnarrow=repos', '--queryformat=%{version}|%{release}|%{arch}|%{repo}|%{version}-%{release}', 'bash']), + ]) diff --git a/roles/lib_utils/src/test/unit/test_yedit.py b/roles/lib_utils/src/test/unit/test_yedit.py new file mode 100755 index 000000000..f9f42843a --- /dev/null +++ b/roles/lib_utils/src/test/unit/test_yedit.py @@ -0,0 +1,368 @@ +''' + Unit tests for yedit +''' + +import os +import sys +import unittest +import mock + +# Removing invalid variable names for tests so that I can +# keep them brief +# pylint: disable=invalid-name,no-name-in-module +# Disable import-error b/c our libraries aren't loaded in jenkins +# pylint: disable=import-error +# place yedit in our path +yedit_path = os.path.join('/'.join(os.path.realpath(__file__).split('/')[:-4]), 'library') # noqa: E501 +sys.path.insert(0, yedit_path) + +from yedit import Yedit, YeditException # noqa: E402 + +# pylint: disable=too-many-public-methods +# Silly pylint, moar tests! + + +class YeditTest(unittest.TestCase): + ''' + Test class for yedit + ''' + data = {'a': 'a', + 'b': {'c': {'d': [{'e': 'x'}, 'f', 'g']}}, + } # noqa: E124 + + filename = 'yedit_test.yml' + + def setUp(self): + ''' setup method will create a file and set to known configuration ''' + yed = Yedit(YeditTest.filename) + yed.yaml_dict = YeditTest.data + yed.write() + + def test_load(self): + ''' Testing a get ''' + yed = Yedit('yedit_test.yml') + self.assertEqual(yed.yaml_dict, self.data) + + def test_write(self): + ''' Testing a simple write ''' + yed = Yedit('yedit_test.yml') + yed.put('key1', 1) + yed.write() + self.assertTrue('key1' in yed.yaml_dict) + self.assertEqual(yed.yaml_dict['key1'], 1) + + def test_write_x_y_z(self): + '''Testing a write of multilayer key''' + yed = Yedit('yedit_test.yml') + yed.put('x.y.z', 'modified') + yed.write() + yed.load() + self.assertEqual(yed.get('x.y.z'), 'modified') + + def test_delete_a(self): + '''Testing a simple delete ''' + yed = Yedit('yedit_test.yml') + yed.delete('a') + yed.write() + yed.load() + self.assertTrue('a' not in yed.yaml_dict) + + def test_delete_b_c(self): + '''Testing delete of layered key ''' + yed = Yedit('yedit_test.yml', separator=':') + yed.delete('b:c') + yed.write() + yed.load() + self.assertTrue('b' in yed.yaml_dict) + self.assertFalse('c' in yed.yaml_dict['b']) + + def test_create(self): + '''Testing a create ''' + os.unlink(YeditTest.filename) + yed = Yedit('yedit_test.yml') + yed.create('foo', 'bar') + yed.write() + yed.load() + self.assertTrue('foo' in yed.yaml_dict) + self.assertTrue(yed.yaml_dict['foo'] == 'bar') + + def test_create_content(self): + '''Testing a create with content ''' + content = {"foo": "bar"} + yed = Yedit("yedit_test.yml", content) + yed.write() + yed.load() + self.assertTrue('foo' in yed.yaml_dict) + self.assertTrue(yed.yaml_dict['foo'], 'bar') + + def test_array_insert(self): + '''Testing a create with content ''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('b:c:d[0]', 'inject') + self.assertTrue(yed.get('b:c:d[0]') == 'inject') + + def test_array_insert_first_index(self): + '''Testing a create with content ''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('b:c:d[0]', 'inject') + self.assertTrue(yed.get('b:c:d[1]') == 'f') + + def test_array_insert_second_index(self): + '''Testing a create with content ''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('b:c:d[0]', 'inject') + self.assertTrue(yed.get('b:c:d[2]') == 'g') + + def test_dict_array_dict_access(self): + '''Testing a create with content''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('b:c:d[0]', [{'x': {'y': 'inject'}}]) + self.assertTrue(yed.get('b:c:d[0]:[0]:x:y') == 'inject') + + def test_dict_array_dict_replace(self): + '''Testing multilevel delete''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('b:c:d[0]', [{'x': {'y': 'inject'}}]) + yed.put('b:c:d[0]:[0]:x:y', 'testing') + self.assertTrue('b' in yed.yaml_dict) + self.assertTrue('c' in yed.yaml_dict['b']) + self.assertTrue('d' in yed.yaml_dict['b']['c']) + self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'], list)) + self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'][0], list)) + self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'][0][0], dict)) + self.assertTrue('y' in yed.yaml_dict['b']['c']['d'][0][0]['x']) + self.assertTrue(yed.yaml_dict['b']['c']['d'][0][0]['x']['y'] == 'testing') # noqa: E501 + + def test_dict_array_dict_remove(self): + '''Testing multilevel delete''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('b:c:d[0]', [{'x': {'y': 'inject'}}]) + yed.delete('b:c:d[0]:[0]:x:y') + self.assertTrue('b' in yed.yaml_dict) + self.assertTrue('c' in yed.yaml_dict['b']) + self.assertTrue('d' in yed.yaml_dict['b']['c']) + self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'], list)) + self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'][0], list)) + self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'][0][0], dict)) + self.assertFalse('y' in yed.yaml_dict['b']['c']['d'][0][0]['x']) + + def test_key_exists_in_dict(self): + '''Testing exist in dict''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('b:c:d[0]', [{'x': {'y': 'inject'}}]) + self.assertTrue(yed.exists('b:c', 'd')) + + def test_key_exists_in_list(self): + '''Testing exist in list''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('b:c:d[0]', [{'x': {'y': 'inject'}}]) + self.assertTrue(yed.exists('b:c:d', [{'x': {'y': 'inject'}}])) + self.assertFalse(yed.exists('b:c:d', [{'x': {'y': 'test'}}])) + + def test_update_to_list_with_index(self): + '''Testing update to list with index''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('x:y:z', [1, 2, 3]) + yed.update('x:y:z', [5, 6], index=2) + self.assertTrue(yed.get('x:y:z') == [1, 2, [5, 6]]) + self.assertTrue(yed.exists('x:y:z', [5, 6])) + self.assertFalse(yed.exists('x:y:z', 4)) + + def test_update_to_list_with_curr_value(self): + '''Testing update to list with index''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('x:y:z', [1, 2, 3]) + yed.update('x:y:z', [5, 6], curr_value=3) + self.assertTrue(yed.get('x:y:z') == [1, 2, [5, 6]]) + self.assertTrue(yed.exists('x:y:z', [5, 6])) + self.assertFalse(yed.exists('x:y:z', 4)) + + def test_update_to_list(self): + '''Testing update to list''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('x:y:z', [1, 2, 3]) + yed.update('x:y:z', [5, 6]) + self.assertTrue(yed.get('x:y:z') == [1, 2, 3, [5, 6]]) + self.assertTrue(yed.exists('x:y:z', [5, 6])) + self.assertFalse(yed.exists('x:y:z', 4)) + + def test_append_twice_to_list(self): + '''Testing append to list''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('x:y:z', [1, 2, 3]) + yed.append('x:y:z', [5, 6]) + yed.append('x:y:z', [5, 6]) + self.assertTrue(yed.get('x:y:z') == [1, 2, 3, [5, 6], [5, 6]]) + self.assertFalse(yed.exists('x:y:z', 4)) + + def test_add_item_to_dict(self): + '''Testing update to dict''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('x:y:z', {'a': 1, 'b': 2}) + yed.update('x:y:z', {'c': 3, 'd': 4}) + self.assertTrue(yed.get('x:y:z') == {'a': 1, 'b': 2, 'c': 3, 'd': 4}) + self.assertTrue(yed.exists('x:y:z', {'c': 3})) + + def test_first_level_dict_with_none_value(self): + '''test dict value with none value''' + yed = Yedit(content={'a': None}, separator=":") + yed.put('a:b:c', 'test') + self.assertTrue(yed.get('a:b:c') == 'test') + self.assertTrue(yed.get('a:b'), {'c': 'test'}) + + def test_adding_yaml_variable(self): + '''test dict value with none value''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('z:y', '{{test}}') + self.assertTrue(yed.get('z:y') == '{{test}}') + + def test_keys_with_underscore(self): + '''test dict value with none value''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('z_:y_y', {'test': '{{test}}'}) + self.assertTrue(yed.get('z_:y_y') == {'test': '{{test}}'}) + + def test_first_level_array_update(self): + '''test update on top level array''' + yed = Yedit(content=[{'a': 1}, {'b': 2}, {'b': 3}], separator=':') + yed.update('', {'c': 4}) + self.assertTrue({'c': 4} in yed.get('')) + + def test_first_level_array_delete(self): + '''test remove top level key''' + yed = Yedit(content=[{'a': 1}, {'b': 2}, {'b': 3}]) + yed.delete('') + self.assertTrue({'b': 3} not in yed.get('')) + + def test_first_level_array_get(self): + '''test dict value with none value''' + yed = Yedit(content=[{'a': 1}, {'b': 2}, {'b': 3}]) + yed.get('') + self.assertTrue([{'a': 1}, {'b': 2}, {'b': 3}] == yed.yaml_dict) + + def test_pop_list_item(self): + '''test dict value with none value''' + yed = Yedit(content=[{'a': 1}, {'b': 2}, {'b': 3}], separator=':') + yed.pop('', {'b': 2}) + self.assertTrue([{'a': 1}, {'b': 3}] == yed.yaml_dict) + + def test_pop_list_item_2(self): + '''test dict value with none value''' + z = list(range(10)) + yed = Yedit(content=z, separator=':') + yed.pop('', 5) + z.pop(5) + self.assertTrue(z == yed.yaml_dict) + + def test_pop_dict_key(self): + '''test dict value with none value''' + yed = Yedit(content={'a': {'b': {'c': 1, 'd': 2}}}, separator='#') + yed.pop('a#b', 'c') + self.assertTrue({'a': {'b': {'d': 2}}} == yed.yaml_dict) + + def test_accessing_path_with_unexpected_objects(self): + '''test providing source path objects that differ from current object state''' + yed = Yedit(content={'a': {'b': {'c': ['d', 'e']}}}) + with self.assertRaises(YeditException): + yed.put('a.b.c.d', 'x') + + def test_creating_new_objects_with_embedded_list(self): + '''test creating new objects with an embedded list in the creation path''' + yed = Yedit(content={'a': {'b': 12}}) + with self.assertRaises(YeditException): + yed.put('new.stuff[0].here', 'value') + + def test_creating_new_objects_with_trailing_list(self): + '''test creating new object(s) where the final piece is a list''' + yed = Yedit(content={'a': {'b': 12}}) + with self.assertRaises(YeditException): + yed.put('new.stuff.here[0]', 'item') + + def test_empty_key_with_int_value(self): + '''test editing top level with not list or dict''' + yed = Yedit(content={'a': {'b': 12}}) + result = yed.put('', 'b') + self.assertFalse(result[0]) + + def test_setting_separator(self): + '''test editing top level with not list or dict''' + yed = Yedit(content={'a': {'b': 12}}) + yed.separator = ':' + self.assertEqual(yed.separator, ':') + + def test_remove_all(self): + '''test removing all data''' + data = Yedit.remove_entry({'a': {'b': 12}}, '') + self.assertTrue(data) + + def test_remove_list_entry(self): + '''test removing list entry''' + data = {'a': {'b': [{'c': 3}]}} + results = Yedit.remove_entry(data, 'a.b[0]') + self.assertTrue(results) + self.assertTrue(data, {'a': {'b': []}}) + + def test_parse_value_string_true(self): + '''test parse_value''' + results = Yedit.parse_value('true', 'str') + self.assertEqual(results, 'true') + + def test_parse_value_bool_true(self): + '''test parse_value''' + results = Yedit.parse_value('true', 'bool') + self.assertTrue(results) + + def test_parse_value_bool_exception(self): + '''test parse_value''' + with self.assertRaises(YeditException): + Yedit.parse_value('TTT', 'bool') + + @mock.patch('yedit.Yedit.write') + def test_run_ansible_basic(self, mock_write): + '''test parse_value''' + params = { + 'src': None, + 'backup': False, + 'separator': '.', + 'state': 'present', + 'edits': [], + 'value': None, + 'key': None, + 'content': {'a': {'b': {'c': 1}}}, + 'content_type': '', + } + + results = Yedit.run_ansible(params) + + mock_write.side_effect = [ + (True, params['content']), + ] + + self.assertFalse(results['changed']) + + @mock.patch('yedit.Yedit.write') + def test_run_ansible_and_write(self, mock_write): + '''test parse_value''' + params = { + 'src': '/tmp/test', + 'backup': False, + 'separator': '.', + 'state': 'present', + 'edits': [], + 'value': None, + 'key': None, + 'content': {'a': {'b': {'c': 1}}}, + 'content_type': '', + } + + results = Yedit.run_ansible(params) + + mock_write.side_effect = [ + (True, params['content']), + ] + + self.assertTrue(results['changed']) + + def tearDown(self): + '''TearDown method''' + os.unlink(YeditTest.filename) |