From 2d1a7761386574455ff98112fe4341cbe22d775e Mon Sep 17 00:00:00 2001 From: Kenny Woodson Date: Tue, 3 Jan 2017 10:22:47 -0500 Subject: Adding ability to systematically modify yaml from ansible. --- roles/lib_utils/library/yedit.py | 766 +++++++++++++++++++++ roles/lib_utils/src/ansible/yedit.py | 84 +++ roles/lib_utils/src/class/import.py | 11 + roles/lib_utils/src/class/yedit.py | 520 ++++++++++++++ roles/lib_utils/src/doc/license | 16 + roles/lib_utils/src/doc/yedit | 132 ++++ roles/lib_utils/src/generate.py | 45 ++ roles/lib_utils/src/generate_sources.yml | 7 + .../src/test/integration/files/kube-manager.yaml | 39 ++ .../lib_utils/src/test/integration/yedit_test.yml | 221 ++++++ roles/lib_utils/src/test/unit/yedit_test.py | 277 ++++++++ 11 files changed, 2118 insertions(+) create mode 100644 roles/lib_utils/library/yedit.py create mode 100644 roles/lib_utils/src/ansible/yedit.py create mode 100644 roles/lib_utils/src/class/import.py create mode 100644 roles/lib_utils/src/class/yedit.py create mode 100644 roles/lib_utils/src/doc/license create mode 100644 roles/lib_utils/src/doc/yedit create mode 100755 roles/lib_utils/src/generate.py create mode 100644 roles/lib_utils/src/generate_sources.yml create mode 100644 roles/lib_utils/src/test/integration/files/kube-manager.yaml create mode 100755 roles/lib_utils/src/test/integration/yedit_test.yml create mode 100755 roles/lib_utils/src/test/unit/yedit_test.py diff --git a/roles/lib_utils/library/yedit.py b/roles/lib_utils/library/yedit.py new file mode 100644 index 000000000..fb545c7c8 --- /dev/null +++ b/roles/lib_utils/library/yedit.py @@ -0,0 +1,766 @@ +#!/usr/bin/env python +# pylint: disable=missing-docstring +# ___ ___ _ _ ___ ___ _ _____ ___ ___ +# / __| __| \| | __| _ \ /_\_ _| __| \ +# | (_ | _|| .` | _|| / / _ \| | | _|| |) | +# \___|___|_|\_|___|_|_\/_/_\_\_|_|___|___/_ _____ +# | \ / _ \ | \| |/ _ \_ _| | __| \_ _|_ _| +# | |) | (_) | | .` | (_) || | | _|| |) | | | | +# |___/ \___/ |_|\_|\___/ |_| |___|___/___| |_| +# +# Copyright 2016 Red Hat, Inc. and/or its affiliates +# and other contributors as indicated by the @author tags. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +# pylint: disable=wrong-import-order +import json +import os +import re +# pylint: disable=import-error +import ruamel.yaml as yaml +import shutil +from ansible.module_utils.basic import AnsibleModule + +DOCUMENTATION = ''' +--- +module: yedit +short_description: Create, modify, and idempotently manage yaml files. +description: + - Modify yaml files programmatically. +options: + state: + description: + - State represents whether to create, modify, delete, or list yaml + required: true + default: present + choices: ["present", "absent", "list"] + aliases: [] + debug: + description: + - Turn on debug information. + required: false + default: false + aliases: [] + src: + description: + - The file that is the target of the modifications. + required: false + default: None + aliases: [] + content: + description: + - Content represents the yaml content you desire to work with. This + - could be the file contents to write or the inmemory data to modify. + required: false + default: None + aliases: [] + content_type: + description: + - The python type of the content parameter. + required: false + default: 'dict' + aliases: [] + key: + description: + - The path to the value you wish to modify. Emtpy string means the top of + - the document. + required: false + default: '' + aliases: [] + value: + description: + - The incoming value of parameter 'key'. + required: false + default: + aliases: [] + value_type: + description: + - The python type of the incoming value. + required: false + default: '' + aliases: [] + update: + description: + - Whether the update should be performed on a dict/hash or list/array + - object. + required: false + default: false + aliases: [] + append: + description: + - Whether to append to an array/list. When the key does not exist or is + - null, a new array is created. When the key is of a non-list type, + - nothing is done. + required: false + default: false + aliases: [] + index: + description: + - Used in conjunction with the update parameter. This will update a + - specific index in an array/list. + required: false + default: false + aliases: [] + curr_value: + description: + - Used in conjunction with the update parameter. This is the current + - value of 'key' in the yaml file. + required: false + default: false + aliases: [] + curr_value_format: + description: + - Format of the incoming current value. + choices: ["yaml", "json", "str"] + required: false + default: false + aliases: [] + backup: + description: + - Whether to make a backup copy of the current file when performing an + - edit. + required: false + default: true + aliases: [] +author: +- "Kenny Woodson " +extends_documentation_fragment: [] +''' + +EXAMPLES = ''' +# Simple insert of key, value +- name: insert simple key, value + yedit: + src: somefile.yml + key: test + value: somevalue + state: present +# Results: +# test: somevalue + +# Multilevel insert of key, value +- name: insert simple key, value + yedit: + src: somefile.yml + key: a#b#c + value: d + state: present +# Results: +# a: +# b: +# c: d +''' + + +class YeditException(Exception): + ''' Exception class for Yedit ''' + pass + + +class Yedit(object): + ''' Class to modify yaml files ''' + re_valid_key = r"(((\[-?\d+\])|([0-9a-zA-Z%s/_-]+)).?)+$" + re_key = r"(?:\[(-?\d+)\])|([0-9a-zA-Z%s/_-]+)" + com_sep = set(['.', '#', '|', ':']) + + # pylint: disable=too-many-arguments + def __init__(self, + filename=None, + content=None, + content_type='yaml', + separator='.', + backup=False): + self.content = content + self._separator = separator + self.filename = filename + self.__yaml_dict = content + self.content_type = content_type + self.backup = backup + self.load(content_type=self.content_type) + if self.__yaml_dict is None: + self.__yaml_dict = {} + + @property + def separator(self): + ''' getter method for yaml_dict ''' + return self._separator + + @separator.setter + def separator(self): + ''' getter method for yaml_dict ''' + return self._separator + + @property + def yaml_dict(self): + ''' getter method for yaml_dict ''' + return self.__yaml_dict + + @yaml_dict.setter + def yaml_dict(self, value): + ''' setter method for yaml_dict ''' + self.__yaml_dict = value + + @staticmethod + def parse_key(key, sep='.'): + '''parse the key allowing the appropriate separator''' + common_separators = list(Yedit.com_sep - set([sep])) + return re.findall(Yedit.re_key % ''.join(common_separators), key) + + @staticmethod + def valid_key(key, sep='.'): + '''validate the incoming key''' + common_separators = list(Yedit.com_sep - set([sep])) + if not re.match(Yedit.re_valid_key % ''.join(common_separators), key): + return False + + return True + + @staticmethod + def remove_entry(data, key, sep='.'): + ''' remove data at location key ''' + if key == '' and isinstance(data, dict): + data.clear() + return True + elif key == '' and isinstance(data, list): + del data[:] + return True + + if not (key and Yedit.valid_key(key, sep)) and \ + isinstance(data, (list, dict)): + return None + + key_indexes = Yedit.parse_key(key, sep) + for arr_ind, dict_key in key_indexes[:-1]: + if dict_key and isinstance(data, dict): + data = data.get(dict_key, None) + elif (arr_ind and isinstance(data, list) and + int(arr_ind) <= len(data) - 1): + data = data[int(arr_ind)] + else: + return None + + # process last index for remove + # expected list entry + if key_indexes[-1][0]: + if isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: # noqa: E501 + del data[int(key_indexes[-1][0])] + return True + + # expected dict entry + elif key_indexes[-1][1]: + if isinstance(data, dict): + del data[key_indexes[-1][1]] + return True + + @staticmethod + def add_entry(data, key, item=None, sep='.'): + ''' Get an item from a dictionary with key notation a.b.c + d = {'a': {'b': 'c'}}} + key = a#b + return c + ''' + if key == '': + pass + elif (not (key and Yedit.valid_key(key, sep)) and + isinstance(data, (list, dict))): + return None + + key_indexes = Yedit.parse_key(key, sep) + for arr_ind, dict_key in key_indexes[:-1]: + if dict_key: + if isinstance(data, dict) and dict_key in data and data[dict_key]: # noqa: E501 + data = data[dict_key] + continue + + elif data and not isinstance(data, dict): + return None + + data[dict_key] = {} + data = data[dict_key] + + elif (arr_ind and isinstance(data, list) and + int(arr_ind) <= len(data) - 1): + data = data[int(arr_ind)] + else: + return None + + if key == '': + data = item + + # process last index for add + # expected list entry + elif key_indexes[-1][0] and isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: # noqa: E501 + data[int(key_indexes[-1][0])] = item + + # expected dict entry + elif key_indexes[-1][1] and isinstance(data, dict): + data[key_indexes[-1][1]] = item + + return data + + @staticmethod + def get_entry(data, key, sep='.'): + ''' Get an item from a dictionary with key notation a.b.c + d = {'a': {'b': 'c'}}} + key = a.b + return c + ''' + if key == '': + pass + elif (not (key and Yedit.valid_key(key, sep)) and + isinstance(data, (list, dict))): + return None + + key_indexes = Yedit.parse_key(key, sep) + for arr_ind, dict_key in key_indexes: + if dict_key and isinstance(data, dict): + data = data.get(dict_key, None) + elif (arr_ind and isinstance(data, list) and + int(arr_ind) <= len(data) - 1): + data = data[int(arr_ind)] + else: + return None + + return data + + def write(self): + ''' write to file ''' + if not self.filename: + raise YeditException('Please specify a filename.') + + if self.backup and self.file_exists(): + shutil.copy(self.filename, self.filename + '.orig') + + tmp_filename = self.filename + '.yedit' + with open(tmp_filename, 'w') as yfd: + # pylint: disable=no-member + if hasattr(self.yaml_dict, 'fa'): + self.yaml_dict.fa.set_block_style() + + yfd.write(yaml.dump(self.yaml_dict, Dumper=yaml.RoundTripDumper)) + + os.rename(tmp_filename, self.filename) + + return (True, self.yaml_dict) + + def read(self): + ''' read from file ''' + # check if it exists + if self.filename is None or not self.file_exists(): + return None + + contents = None + with open(self.filename) as yfd: + contents = yfd.read() + + return contents + + def file_exists(self): + ''' return whether file exists ''' + if os.path.exists(self.filename): + return True + + return False + + def load(self, content_type='yaml'): + ''' return yaml file ''' + contents = self.read() + + if not contents and not self.content: + return None + + if self.content: + if isinstance(self.content, dict): + self.yaml_dict = self.content + return self.yaml_dict + elif isinstance(self.content, str): + contents = self.content + + # check if it is yaml + try: + if content_type == 'yaml' and contents: + self.yaml_dict = yaml.load(contents, yaml.RoundTripLoader) + # pylint: disable=no-member + if hasattr(self.yaml_dict, 'fa'): + self.yaml_dict.fa.set_block_style() + elif content_type == 'json' and contents: + self.yaml_dict = json.loads(contents) + except yaml.YAMLError as err: + # Error loading yaml or json + raise YeditException('Problem with loading yaml file. %s' % err) + + return self.yaml_dict + + def get(self, key): + ''' get a specified key''' + try: + entry = Yedit.get_entry(self.yaml_dict, key, self.separator) + except KeyError: + entry = None + + return entry + + def pop(self, path, key_or_item): + ''' remove a key, value pair from a dict or an item for a list''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry is None: + return (False, self.yaml_dict) + + if isinstance(entry, dict): + # pylint: disable=no-member,maybe-no-member + if key_or_item in entry: + entry.pop(key_or_item) + return (True, self.yaml_dict) + return (False, self.yaml_dict) + + elif isinstance(entry, list): + # pylint: disable=no-member,maybe-no-member + ind = None + try: + ind = entry.index(key_or_item) + except ValueError: + return (False, self.yaml_dict) + + entry.pop(ind) + return (True, self.yaml_dict) + + return (False, self.yaml_dict) + + def delete(self, path): + ''' remove path from a dict''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry is None: + return (False, self.yaml_dict) + + result = Yedit.remove_entry(self.yaml_dict, path, self.separator) + if not result: + return (False, self.yaml_dict) + + return (True, self.yaml_dict) + + def exists(self, path, value): + ''' check if value exists at path''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if isinstance(entry, list): + if value in entry: + return True + return False + + elif isinstance(entry, dict): + if isinstance(value, dict): + rval = False + for key, val in value.items(): + if entry[key] != val: + rval = False + break + else: + rval = True + return rval + + return value in entry + + return entry == value + + def append(self, path, value): + '''append value to a list''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry is None: + self.put(path, []) + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + if not isinstance(entry, list): + return (False, self.yaml_dict) + + # pylint: disable=no-member,maybe-no-member + entry.append(value) + return (True, self.yaml_dict) + + # pylint: disable=too-many-arguments + def update(self, path, value, index=None, curr_value=None): + ''' put path, value into a dict ''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if isinstance(entry, dict): + # pylint: disable=no-member,maybe-no-member + if not isinstance(value, dict): + raise YeditException('Cannot replace key, value entry in ' + + 'dict with non-dict type. value=[%s] [%s]' % (value, type(value))) # noqa: E501 + + entry.update(value) + return (True, self.yaml_dict) + + elif isinstance(entry, list): + # pylint: disable=no-member,maybe-no-member + ind = None + if curr_value: + try: + ind = entry.index(curr_value) + except ValueError: + return (False, self.yaml_dict) + + elif index is not None: + ind = index + + if ind is not None and entry[ind] != value: + entry[ind] = value + return (True, self.yaml_dict) + + # see if it exists in the list + try: + ind = entry.index(value) + except ValueError: + # doesn't exist, append it + entry.append(value) + return (True, self.yaml_dict) + + # already exists, return + if ind is not None: + return (False, self.yaml_dict) + return (False, self.yaml_dict) + + def put(self, path, value): + ''' put path, value into a dict ''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry == value: + return (False, self.yaml_dict) + + # deepcopy didn't work + tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, + default_flow_style=False), + yaml.RoundTripLoader) + # pylint: disable=no-member + if hasattr(self.yaml_dict, 'fa'): + tmp_copy.fa.set_block_style() + result = Yedit.add_entry(tmp_copy, path, value, self.separator) + if not result: + return (False, self.yaml_dict) + + self.yaml_dict = tmp_copy + + return (True, self.yaml_dict) + + def create(self, path, value): + ''' create a yaml file ''' + if not self.file_exists(): + # deepcopy didn't work + tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, default_flow_style=False), # noqa: E501 + yaml.RoundTripLoader) + # pylint: disable=no-member + if hasattr(self.yaml_dict, 'fa'): + tmp_copy.fa.set_block_style() + result = Yedit.add_entry(tmp_copy, path, value, self.separator) + if result: + self.yaml_dict = tmp_copy + return (True, self.yaml_dict) + + return (False, self.yaml_dict) + + # pylint: disable=too-many-return-statements,too-many-branches + @staticmethod + def run_ansible(module): + '''perform the idempotent crud operations''' + yamlfile = Yedit(filename=module.params['src'], + backup=module.params['backup'], + separator=module.params['separator']) + + if module.params['src']: + rval = yamlfile.load() + + if yamlfile.yaml_dict is None and \ + module.params['state'] != 'present': + return {'failed': True, + 'msg': 'Error opening file [%s]. Verify that the ' + + 'file exists, that it is has correct' + + ' permissions, and is valid yaml.'} + + if module.params['state'] == 'list': + if module.params['content']: + content = parse_value(module.params['content'], + module.params['content_type']) + yamlfile.yaml_dict = content + + if module.params['key']: + rval = yamlfile.get(module.params['key']) or {} + + return {'changed': False, 'result': rval, 'state': "list"} + + elif module.params['state'] == 'absent': + if module.params['content']: + content = parse_value(module.params['content'], + module.params['content_type']) + yamlfile.yaml_dict = content + + if module.params['update']: + rval = yamlfile.pop(module.params['key'], + module.params['value']) + else: + rval = yamlfile.delete(module.params['key']) + + if rval[0] and module.params['src']: + yamlfile.write() + + return {'changed': rval[0], 'result': rval[1], 'state': "absent"} + + elif module.params['state'] == 'present': + # check if content is different than what is in the file + if module.params['content']: + content = parse_value(module.params['content'], + module.params['content_type']) + + # We had no edits to make and the contents are the same + if yamlfile.yaml_dict == content and \ + module.params['value'] is None: + return {'changed': False, + 'result': yamlfile.yaml_dict, + 'state': "present"} + + yamlfile.yaml_dict = content + + # we were passed a value; parse it + if module.params['value']: + value = parse_value(module.params['value'], + module.params['value_type']) + key = module.params['key'] + if module.params['update']: + # pylint: disable=line-too-long + curr_value = get_curr_value(parse_value(module.params['curr_value']), module.params['curr_value_format']) # noqa: #501 + + rval = yamlfile.update(key, value, module.params['index'], curr_value) # noqa: E501 + + elif module.params['append']: + rval = yamlfile.append(key, value) + else: + rval = yamlfile.put(key, value) + + if rval[0] and module.params['src']: + yamlfile.write() + + return {'changed': rval[0], + 'result': rval[1], 'state': "present"} + + # no edits to make + if module.params['src']: + # pylint: disable=redefined-variable-type + rval = yamlfile.write() + return {'changed': rval[0], + 'result': rval[1], + 'state': "present"} + + return {'failed': True, 'msg': 'Unkown state passed'} + + +def get_curr_value(invalue, val_type): + '''return the current value''' + if invalue is None: + return None + + curr_value = invalue + if val_type == 'yaml': + curr_value = yaml.load(invalue) + elif val_type == 'json': + curr_value = json.loads(invalue) + + return curr_value + + +def parse_value(inc_value, vtype=''): + '''determine value type passed''' + true_bools = ['y', 'Y', 'yes', 'Yes', 'YES', 'true', 'True', 'TRUE', + 'on', 'On', 'ON', ] + false_bools = ['n', 'N', 'no', 'No', 'NO', 'false', 'False', 'FALSE', + 'off', 'Off', 'OFF'] + + # It came in as a string but you didn't specify value_type as string + # we will convert to bool if it matches any of the above cases + if isinstance(inc_value, str) and 'bool' in vtype: + if inc_value not in true_bools and inc_value not in false_bools: + raise YeditException('Not a boolean type. str=[%s] vtype=[%s]' + % (inc_value, vtype)) + elif isinstance(inc_value, bool) and 'str' in vtype: + inc_value = str(inc_value) + + # If vtype is not str then go ahead and attempt to yaml load it. + if isinstance(inc_value, str) and 'str' not in vtype: + try: + inc_value = yaml.load(inc_value) + except Exception: + raise YeditException('Could not determine type of incoming ' + + 'value. value=[%s] vtype=[%s]' + % (type(inc_value), vtype)) + + return inc_value + + +# pylint: disable=too-many-branches +def main(): + ''' ansible oc module for secrets ''' + + module = AnsibleModule( + argument_spec=dict( + state=dict(default='present', type='str', + choices=['present', 'absent', 'list']), + debug=dict(default=False, type='bool'), + src=dict(default=None, type='str'), + content=dict(default=None), + content_type=dict(default='dict', choices=['dict']), + key=dict(default='', type='str'), + value=dict(), + value_type=dict(default='', type='str'), + update=dict(default=False, type='bool'), + append=dict(default=False, type='bool'), + index=dict(default=None, type='int'), + curr_value=dict(default=None, type='str'), + curr_value_format=dict(default='yaml', + choices=['yaml', 'json', 'str'], + type='str'), + backup=dict(default=True, type='bool'), + separator=dict(default='.', type='str'), + ), + mutually_exclusive=[["curr_value", "index"], ['update', "append"]], + required_one_of=[["content", "src"]], + ) + + rval = Yedit.run_ansible(module) + if 'failed' in rval and rval['failed']: + module.fail_json(msg=rval['msg']) + + module.exit_json(**rval) + + +if __name__ == '__main__': + main() diff --git a/roles/lib_utils/src/ansible/yedit.py b/roles/lib_utils/src/ansible/yedit.py new file mode 100644 index 000000000..a80cd520c --- /dev/null +++ b/roles/lib_utils/src/ansible/yedit.py @@ -0,0 +1,84 @@ +# flake8: noqa +# pylint: skip-file + + +def get_curr_value(invalue, val_type): + '''return the current value''' + if invalue is None: + return None + + curr_value = invalue + if val_type == 'yaml': + curr_value = yaml.load(invalue) + elif val_type == 'json': + curr_value = json.loads(invalue) + + return curr_value + + +def parse_value(inc_value, vtype=''): + '''determine value type passed''' + true_bools = ['y', 'Y', 'yes', 'Yes', 'YES', 'true', 'True', 'TRUE', + 'on', 'On', 'ON', ] + false_bools = ['n', 'N', 'no', 'No', 'NO', 'false', 'False', 'FALSE', + 'off', 'Off', 'OFF'] + + # It came in as a string but you didn't specify value_type as string + # we will convert to bool if it matches any of the above cases + if isinstance(inc_value, str) and 'bool' in vtype: + if inc_value not in true_bools and inc_value not in false_bools: + raise YeditException('Not a boolean type. str=[%s] vtype=[%s]' + % (inc_value, vtype)) + elif isinstance(inc_value, bool) and 'str' in vtype: + inc_value = str(inc_value) + + # If vtype is not str then go ahead and attempt to yaml load it. + if isinstance(inc_value, str) and 'str' not in vtype: + try: + inc_value = yaml.load(inc_value) + except Exception: + raise YeditException('Could not determine type of incoming ' + + 'value. value=[%s] vtype=[%s]' + % (type(inc_value), vtype)) + + return inc_value + + +# pylint: disable=too-many-branches +def main(): + ''' ansible oc module for secrets ''' + + module = AnsibleModule( + argument_spec=dict( + state=dict(default='present', type='str', + choices=['present', 'absent', 'list']), + debug=dict(default=False, type='bool'), + src=dict(default=None, type='str'), + content=dict(default=None), + content_type=dict(default='dict', choices=['dict']), + key=dict(default='', type='str'), + value=dict(), + value_type=dict(default='', type='str'), + update=dict(default=False, type='bool'), + append=dict(default=False, type='bool'), + index=dict(default=None, type='int'), + curr_value=dict(default=None, type='str'), + curr_value_format=dict(default='yaml', + choices=['yaml', 'json', 'str'], + type='str'), + backup=dict(default=True, type='bool'), + separator=dict(default='.', type='str'), + ), + mutually_exclusive=[["curr_value", "index"], ['update', "append"]], + required_one_of=[["content", "src"]], + ) + + rval = Yedit.run_ansible(module) + if 'failed' in rval and rval['failed']: + module.fail_json(msg=rval['msg']) + + module.exit_json(**rval) + + +if __name__ == '__main__': + main() diff --git a/roles/lib_utils/src/class/import.py b/roles/lib_utils/src/class/import.py new file mode 100644 index 000000000..249e07228 --- /dev/null +++ b/roles/lib_utils/src/class/import.py @@ -0,0 +1,11 @@ +# flake8: noqa +# pylint: skip-file + +# pylint: disable=wrong-import-order +import json +import os +import re +# pylint: disable=import-error +import ruamel.yaml as yaml +import shutil +from ansible.module_utils.basic import AnsibleModule diff --git a/roles/lib_utils/src/class/yedit.py b/roles/lib_utils/src/class/yedit.py new file mode 100644 index 000000000..e110bc11e --- /dev/null +++ b/roles/lib_utils/src/class/yedit.py @@ -0,0 +1,520 @@ +# flake8: noqa +# pylint: skip-file + +class YeditException(Exception): + ''' Exception class for Yedit ''' + pass + + +class Yedit(object): + ''' Class to modify yaml files ''' + re_valid_key = r"(((\[-?\d+\])|([0-9a-zA-Z%s/_-]+)).?)+$" + re_key = r"(?:\[(-?\d+)\])|([0-9a-zA-Z%s/_-]+)" + com_sep = set(['.', '#', '|', ':']) + + # pylint: disable=too-many-arguments + def __init__(self, + filename=None, + content=None, + content_type='yaml', + separator='.', + backup=False): + self.content = content + self._separator = separator + self.filename = filename + self.__yaml_dict = content + self.content_type = content_type + self.backup = backup + self.load(content_type=self.content_type) + if self.__yaml_dict is None: + self.__yaml_dict = {} + + @property + def separator(self): + ''' getter method for yaml_dict ''' + return self._separator + + @separator.setter + def separator(self): + ''' getter method for yaml_dict ''' + return self._separator + + @property + def yaml_dict(self): + ''' getter method for yaml_dict ''' + return self.__yaml_dict + + @yaml_dict.setter + def yaml_dict(self, value): + ''' setter method for yaml_dict ''' + self.__yaml_dict = value + + @staticmethod + def parse_key(key, sep='.'): + '''parse the key allowing the appropriate separator''' + common_separators = list(Yedit.com_sep - set([sep])) + return re.findall(Yedit.re_key % ''.join(common_separators), key) + + @staticmethod + def valid_key(key, sep='.'): + '''validate the incoming key''' + common_separators = list(Yedit.com_sep - set([sep])) + if not re.match(Yedit.re_valid_key % ''.join(common_separators), key): + return False + + return True + + @staticmethod + def remove_entry(data, key, sep='.'): + ''' remove data at location key ''' + if key == '' and isinstance(data, dict): + data.clear() + return True + elif key == '' and isinstance(data, list): + del data[:] + return True + + if not (key and Yedit.valid_key(key, sep)) and \ + isinstance(data, (list, dict)): + return None + + key_indexes = Yedit.parse_key(key, sep) + for arr_ind, dict_key in key_indexes[:-1]: + if dict_key and isinstance(data, dict): + data = data.get(dict_key, None) + elif (arr_ind and isinstance(data, list) and + int(arr_ind) <= len(data) - 1): + data = data[int(arr_ind)] + else: + return None + + # process last index for remove + # expected list entry + if key_indexes[-1][0]: + if isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: # noqa: E501 + del data[int(key_indexes[-1][0])] + return True + + # expected dict entry + elif key_indexes[-1][1]: + if isinstance(data, dict): + del data[key_indexes[-1][1]] + return True + + @staticmethod + def add_entry(data, key, item=None, sep='.'): + ''' Get an item from a dictionary with key notation a.b.c + d = {'a': {'b': 'c'}}} + key = a#b + return c + ''' + if key == '': + pass + elif (not (key and Yedit.valid_key(key, sep)) and + isinstance(data, (list, dict))): + return None + + key_indexes = Yedit.parse_key(key, sep) + for arr_ind, dict_key in key_indexes[:-1]: + if dict_key: + if isinstance(data, dict) and dict_key in data and data[dict_key]: # noqa: E501 + data = data[dict_key] + continue + + elif data and not isinstance(data, dict): + return None + + data[dict_key] = {} + data = data[dict_key] + + elif (arr_ind and isinstance(data, list) and + int(arr_ind) <= len(data) - 1): + data = data[int(arr_ind)] + else: + return None + + if key == '': + data = item + + # process last index for add + # expected list entry + elif key_indexes[-1][0] and isinstance(data, list) and int(key_indexes[-1][0]) <= len(data) - 1: # noqa: E501 + data[int(key_indexes[-1][0])] = item + + # expected dict entry + elif key_indexes[-1][1] and isinstance(data, dict): + data[key_indexes[-1][1]] = item + + return data + + @staticmethod + def get_entry(data, key, sep='.'): + ''' Get an item from a dictionary with key notation a.b.c + d = {'a': {'b': 'c'}}} + key = a.b + return c + ''' + if key == '': + pass + elif (not (key and Yedit.valid_key(key, sep)) and + isinstance(data, (list, dict))): + return None + + key_indexes = Yedit.parse_key(key, sep) + for arr_ind, dict_key in key_indexes: + if dict_key and isinstance(data, dict): + data = data.get(dict_key, None) + elif (arr_ind and isinstance(data, list) and + int(arr_ind) <= len(data) - 1): + data = data[int(arr_ind)] + else: + return None + + return data + + def write(self): + ''' write to file ''' + if not self.filename: + raise YeditException('Please specify a filename.') + + if self.backup and self.file_exists(): + shutil.copy(self.filename, self.filename + '.orig') + + tmp_filename = self.filename + '.yedit' + with open(tmp_filename, 'w') as yfd: + # pylint: disable=no-member + if hasattr(self.yaml_dict, 'fa'): + self.yaml_dict.fa.set_block_style() + + yfd.write(yaml.dump(self.yaml_dict, Dumper=yaml.RoundTripDumper)) + + os.rename(tmp_filename, self.filename) + + return (True, self.yaml_dict) + + def read(self): + ''' read from file ''' + # check if it exists + if self.filename is None or not self.file_exists(): + return None + + contents = None + with open(self.filename) as yfd: + contents = yfd.read() + + return contents + + def file_exists(self): + ''' return whether file exists ''' + if os.path.exists(self.filename): + return True + + return False + + def load(self, content_type='yaml'): + ''' return yaml file ''' + contents = self.read() + + if not contents and not self.content: + return None + + if self.content: + if isinstance(self.content, dict): + self.yaml_dict = self.content + return self.yaml_dict + elif isinstance(self.content, str): + contents = self.content + + # check if it is yaml + try: + if content_type == 'yaml' and contents: + self.yaml_dict = yaml.load(contents, yaml.RoundTripLoader) + # pylint: disable=no-member + if hasattr(self.yaml_dict, 'fa'): + self.yaml_dict.fa.set_block_style() + elif content_type == 'json' and contents: + self.yaml_dict = json.loads(contents) + except yaml.YAMLError as err: + # Error loading yaml or json + raise YeditException('Problem with loading yaml file. %s' % err) + + return self.yaml_dict + + def get(self, key): + ''' get a specified key''' + try: + entry = Yedit.get_entry(self.yaml_dict, key, self.separator) + except KeyError: + entry = None + + return entry + + def pop(self, path, key_or_item): + ''' remove a key, value pair from a dict or an item for a list''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry is None: + return (False, self.yaml_dict) + + if isinstance(entry, dict): + # pylint: disable=no-member,maybe-no-member + if key_or_item in entry: + entry.pop(key_or_item) + return (True, self.yaml_dict) + return (False, self.yaml_dict) + + elif isinstance(entry, list): + # pylint: disable=no-member,maybe-no-member + ind = None + try: + ind = entry.index(key_or_item) + except ValueError: + return (False, self.yaml_dict) + + entry.pop(ind) + return (True, self.yaml_dict) + + return (False, self.yaml_dict) + + def delete(self, path): + ''' remove path from a dict''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry is None: + return (False, self.yaml_dict) + + result = Yedit.remove_entry(self.yaml_dict, path, self.separator) + if not result: + return (False, self.yaml_dict) + + return (True, self.yaml_dict) + + def exists(self, path, value): + ''' check if value exists at path''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if isinstance(entry, list): + if value in entry: + return True + return False + + elif isinstance(entry, dict): + if isinstance(value, dict): + rval = False + for key, val in value.items(): + if entry[key] != val: + rval = False + break + else: + rval = True + return rval + + return value in entry + + return entry == value + + def append(self, path, value): + '''append value to a list''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry is None: + self.put(path, []) + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + if not isinstance(entry, list): + return (False, self.yaml_dict) + + # pylint: disable=no-member,maybe-no-member + entry.append(value) + return (True, self.yaml_dict) + + # pylint: disable=too-many-arguments + def update(self, path, value, index=None, curr_value=None): + ''' put path, value into a dict ''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if isinstance(entry, dict): + # pylint: disable=no-member,maybe-no-member + if not isinstance(value, dict): + raise YeditException('Cannot replace key, value entry in ' + + 'dict with non-dict type. value=[%s] [%s]' % (value, type(value))) # noqa: E501 + + entry.update(value) + return (True, self.yaml_dict) + + elif isinstance(entry, list): + # pylint: disable=no-member,maybe-no-member + ind = None + if curr_value: + try: + ind = entry.index(curr_value) + except ValueError: + return (False, self.yaml_dict) + + elif index is not None: + ind = index + + if ind is not None and entry[ind] != value: + entry[ind] = value + return (True, self.yaml_dict) + + # see if it exists in the list + try: + ind = entry.index(value) + except ValueError: + # doesn't exist, append it + entry.append(value) + return (True, self.yaml_dict) + + # already exists, return + if ind is not None: + return (False, self.yaml_dict) + return (False, self.yaml_dict) + + def put(self, path, value): + ''' put path, value into a dict ''' + try: + entry = Yedit.get_entry(self.yaml_dict, path, self.separator) + except KeyError: + entry = None + + if entry == value: + return (False, self.yaml_dict) + + # deepcopy didn't work + tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, + default_flow_style=False), + yaml.RoundTripLoader) + # pylint: disable=no-member + if hasattr(self.yaml_dict, 'fa'): + tmp_copy.fa.set_block_style() + result = Yedit.add_entry(tmp_copy, path, value, self.separator) + if not result: + return (False, self.yaml_dict) + + self.yaml_dict = tmp_copy + + return (True, self.yaml_dict) + + def create(self, path, value): + ''' create a yaml file ''' + if not self.file_exists(): + # deepcopy didn't work + tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, default_flow_style=False), # noqa: E501 + yaml.RoundTripLoader) + # pylint: disable=no-member + if hasattr(self.yaml_dict, 'fa'): + tmp_copy.fa.set_block_style() + result = Yedit.add_entry(tmp_copy, path, value, self.separator) + if result: + self.yaml_dict = tmp_copy + return (True, self.yaml_dict) + + return (False, self.yaml_dict) + + # pylint: disable=too-many-return-statements,too-many-branches + @staticmethod + def run_ansible(module): + '''perform the idempotent crud operations''' + yamlfile = Yedit(filename=module.params['src'], + backup=module.params['backup'], + separator=module.params['separator']) + + if module.params['src']: + rval = yamlfile.load() + + if yamlfile.yaml_dict is None and \ + module.params['state'] != 'present': + return {'failed': True, + 'msg': 'Error opening file [%s]. Verify that the ' + + 'file exists, that it is has correct' + + ' permissions, and is valid yaml.'} + + if module.params['state'] == 'list': + if module.params['content']: + content = parse_value(module.params['content'], + module.params['content_type']) + yamlfile.yaml_dict = content + + if module.params['key']: + rval = yamlfile.get(module.params['key']) or {} + + return {'changed': False, 'result': rval, 'state': "list"} + + elif module.params['state'] == 'absent': + if module.params['content']: + content = parse_value(module.params['content'], + module.params['content_type']) + yamlfile.yaml_dict = content + + if module.params['update']: + rval = yamlfile.pop(module.params['key'], + module.params['value']) + else: + rval = yamlfile.delete(module.params['key']) + + if rval[0] and module.params['src']: + yamlfile.write() + + return {'changed': rval[0], 'result': rval[1], 'state': "absent"} + + elif module.params['state'] == 'present': + # check if content is different than what is in the file + if module.params['content']: + content = parse_value(module.params['content'], + module.params['content_type']) + + # We had no edits to make and the contents are the same + if yamlfile.yaml_dict == content and \ + module.params['value'] is None: + return {'changed': False, + 'result': yamlfile.yaml_dict, + 'state': "present"} + + yamlfile.yaml_dict = content + + # we were passed a value; parse it + if module.params['value']: + value = parse_value(module.params['value'], + module.params['value_type']) + key = module.params['key'] + if module.params['update']: + # pylint: disable=line-too-long + curr_value = get_curr_value(parse_value(module.params['curr_value']), module.params['curr_value_format']) # noqa: #501 + + rval = yamlfile.update(key, value, module.params['index'], curr_value) # noqa: E501 + + elif module.params['append']: + rval = yamlfile.append(key, value) + else: + rval = yamlfile.put(key, value) + + if rval[0] and module.params['src']: + yamlfile.write() + + return {'changed': rval[0], + 'result': rval[1], 'state': "present"} + + # no edits to make + if module.params['src']: + # pylint: disable=redefined-variable-type + rval = yamlfile.write() + return {'changed': rval[0], + 'result': rval[1], + 'state': "present"} + + return {'failed': True, 'msg': 'Unkown state passed'} diff --git a/roles/lib_utils/src/doc/license b/roles/lib_utils/src/doc/license new file mode 100644 index 000000000..717bb7f17 --- /dev/null +++ b/roles/lib_utils/src/doc/license @@ -0,0 +1,16 @@ +# +# Copyright 2016 Red Hat, Inc. and/or its affiliates +# and other contributors as indicated by the @author tags. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/roles/lib_utils/src/doc/yedit b/roles/lib_utils/src/doc/yedit new file mode 100644 index 000000000..e367a389e --- /dev/null +++ b/roles/lib_utils/src/doc/yedit @@ -0,0 +1,132 @@ +# flake8: noqa +# pylint: skip-file + +DOCUMENTATION = ''' +--- +module: yedit +short_description: Create, modify, and idempotently manage yaml files. +description: + - Modify yaml files programmatically. +options: + state: + description: + - State represents whether to create, modify, delete, or list yaml + required: true + default: present + choices: ["present", "absent", "list"] + aliases: [] + debug: + description: + - Turn on debug information. + required: false + default: false + aliases: [] + src: + description: + - The file that is the target of the modifications. + required: false + default: None + aliases: [] + content: + description: + - Content represents the yaml content you desire to work with. This + - could be the file contents to write or the inmemory data to modify. + required: false + default: None + aliases: [] + content_type: + description: + - The python type of the content parameter. + required: false + default: 'dict' + aliases: [] + key: + description: + - The path to the value you wish to modify. Emtpy string means the top of + - the document. + required: false + default: '' + aliases: [] + value: + description: + - The incoming value of parameter 'key'. + required: false + default: + aliases: [] + value_type: + description: + - The python type of the incoming value. + required: false + default: '' + aliases: [] + update: + description: + - Whether the update should be performed on a dict/hash or list/array + - object. + required: false + default: false + aliases: [] + append: + description: + - Whether to append to an array/list. When the key does not exist or is + - null, a new array is created. When the key is of a non-list type, + - nothing is done. + required: false + default: false + aliases: [] + index: + description: + - Used in conjunction with the update parameter. This will update a + - specific index in an array/list. + required: false + default: false + aliases: [] + curr_value: + description: + - Used in conjunction with the update parameter. This is the current + - value of 'key' in the yaml file. + required: false + default: false + aliases: [] + curr_value_format: + description: + - Format of the incoming current value. + choices: ["yaml", "json", "str"] + required: false + default: false + aliases: [] + backup: + description: + - Whether to make a backup copy of the current file when performing an + - edit. + required: false + default: true + aliases: [] +author: +- "Kenny Woodson " +extends_documentation_fragment: [] +''' + +EXAMPLES = ''' +# Simple insert of key, value +- name: insert simple key, value + yedit: + src: somefile.yml + key: test + value: somevalue + state: present +# Results: +# test: somevalue + +# Multilevel insert of key, value +- name: insert simple key, value + yedit: + src: somefile.yml + key: a#b#c + value: d + state: present +# Results: +# a: +# b: +# c: d +''' diff --git a/roles/lib_utils/src/generate.py b/roles/lib_utils/src/generate.py new file mode 100755 index 000000000..f4b46aa91 --- /dev/null +++ b/roles/lib_utils/src/generate.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +''' + Generate the openshift-ansible/roles/lib_openshift_cli/library/ modules. +''' + +import os +import yaml + +# pylint: disable=anomalous-backslash-in-string +GEN_STR = "#!/usr/bin/env python\n" + \ + "# pylint: disable=missing-docstring\n" + \ + "# ___ ___ _ _ ___ ___ _ _____ ___ ___\n" + \ + "# / __| __| \| | __| _ \ /_\_ _| __| \\\n" + \ + "# | (_ | _|| .` | _|| / / _ \| | | _|| |) |\n" + \ + "# \___|___|_|\_|___|_|_\/_/_\_\_|_|___|___/_ _____\n" + \ + "# | \ / _ \ | \| |/ _ \_ _| | __| \_ _|_ _|\n" + \ + "# | |) | (_) | | .` | (_) || | | _|| |) | | | |\n" + \ + "# |___/ \___/ |_|\_|\___/ |_| |___|___/___| |_|\n" + +OPENSHIFT_ANSIBLE_PATH = os.path.dirname(os.path.realpath(__file__)) +OPENSHIFT_ANSIBLE_SOURCES_PATH = os.path.join(OPENSHIFT_ANSIBLE_PATH, 'generate_sources.yml') # noqa: E501 + + +def main(): + ''' combine the necessary files to create the ansible module ''' + + library = os.path.join(OPENSHIFT_ANSIBLE_PATH, '..', 'library/') + sources = yaml.load(open(OPENSHIFT_ANSIBLE_SOURCES_PATH).read()) + for fname, parts in sources.items(): + with open(os.path.join(library, fname), 'w') as afd: + afd.seek(0) + afd.write(GEN_STR) + for fpart in parts: + with open(os.path.join(OPENSHIFT_ANSIBLE_PATH, fpart)) as pfd: + # first line is pylint disable so skip it + for idx, line in enumerate(pfd): + if idx in [0, 1] and 'flake8: noqa' in line \ + or 'pylint: skip-file' in line: + continue + + afd.write(line) + + +if __name__ == '__main__': + main() diff --git a/roles/lib_utils/src/generate_sources.yml b/roles/lib_utils/src/generate_sources.yml new file mode 100644 index 000000000..83b21de1b --- /dev/null +++ b/roles/lib_utils/src/generate_sources.yml @@ -0,0 +1,7 @@ +--- +yedit.py: +- doc/license +- class/import.py +- doc/yedit +- class/yedit.py +- ansible/yedit.py diff --git a/roles/lib_utils/src/test/integration/files/kube-manager.yaml b/roles/lib_utils/src/test/integration/files/kube-manager.yaml new file mode 100644 index 000000000..6f4b9e6dc --- /dev/null +++ b/roles/lib_utils/src/test/integration/files/kube-manager.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: kube-controller-manager + namespace: kube-system +spec: + hostNetwork: true + containers: + - name: kube-controller-manager + image: openshift/kube:v1.0.0 + command: + - /hyperkube + - controller-manager + - --master=http://127.0.0.1:8080 + - --leader-elect=true + - --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem + - --root-ca-file=/etc/kubernetes/ssl/ca.pem + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10252 + initialDelaySeconds: 15 + timeoutSeconds: 1 + volumeMounts: + - mountPath: /etc/kubernetes/ssl + name: ssl-certs-kubernetes + readOnly: true + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + volumes: + - hostPath: + path: /etc/kubernetes/ssl + name: ssl-certs-kubernetes + - hostPath: + path: /usr/share/ca-certificates + name: ssl-certs-host diff --git a/roles/lib_utils/src/test/integration/yedit_test.yml b/roles/lib_utils/src/test/integration/yedit_test.yml new file mode 100755 index 000000000..1760a7466 --- /dev/null +++ b/roles/lib_utils/src/test/integration/yedit_test.yml @@ -0,0 +1,221 @@ +#!/usr/bin/ansible-playbook +# Yedit test so that we can quickly determine if features are working +# Ensure that the kube-manager.yaml file exists +# +# ./yedit_test.yml -M ../../library +# +--- +- hosts: localhost + gather_facts: no + vars: + test_file: kube-manager-test.yaml + test: test + strategy: debug + + post_tasks: + - name: copy the kube-manager.yaml file so that we have a pristine copy each time + copy: + src: kube-manager.yaml + dest: "./{{ test_file }}" + changed_when: False + + ####### add key to top level ##### + - name: add a key at the top level + yedit: + src: "{{ test_file }}" + key: yedittest + value: yedittest + + - name: retrieve the inserted key + yedit: + src: "{{ test_file }}" + state: list + key: yedittest + register: results + + - name: Assert that key is at top level + assert: + that: results.result == 'yedittest' + msg: 'Test: add a key to top level failed. yedittest != [{{ results.result }}]' + ###### end add key to top level ##### + + ###### modify multilevel key, value ##### + - name: modify multilevel key, value + yedit: + src: "{{ test_file }}" + key: metadata-namespace + value: openshift-is-awesome + separator: '-' + + - name: retrieve the inserted key + yedit: + src: "{{ test_file }}" + state: list + key: metadata-namespace + separator: '-' + register: results + + - name: Assert that key is as expected + assert: + that: results.result == 'openshift-is-awesome' + msg: 'Test: multilevel key, value modification: openshift-is-awesome != [{{ results.result }}]' + ###### end modify multilevel key, value ##### + + ###### test a string boolean ##### + - name: test a string boolean + yedit: + src: "{{ test_file }}" + key: spec.containers[0].volumeMounts[1].readOnly + value: 'true' + value_type: str + + - name: retrieve the inserted key + yedit: + src: "{{ test_file }}" + state: list + key: spec.containers[0].volumeMounts[1].readOnly + register: results + + - name: Assert that key is a string + assert: + that: results.result == "true" + msg: "Test: boolean str: 'true' != [{{ results.result }}]" + + - name: Assert that key is not bool + assert: + that: results.result != true + msg: "Test: boolean str: true != [{{ results.result }}]" + ###### end test boolean string ##### + + ###### test array append ##### + - name: test array append + yedit: + src: "{{ test_file }}" + key: spec.containers[0].command + value: --my-new-parameter=openshift + append: True + + - name: retrieve the array + yedit: + src: "{{ test_file }}" + state: list + key: spec.containers[0].command + register: results + + - name: Assert that the last element in array is our value + assert: + that: results.result[-1] == "--my-new-parameter=openshift" + msg: "Test: '--my-new-parameter=openshift' != [{{ results.result[-1] }}]" + ###### end test array append ##### + + ###### test non-existing array append ##### + - name: test array append to non-existing key + yedit: + src: "{{ test_file }}" + key: nonexistingkey + value: --my-new-parameter=openshift + append: True + + - name: retrieve the array + yedit: + src: "{{ test_file }}" + state: list + key: nonexistingkey + register: results + + - name: Assert that the last element in array is our value + assert: + that: results.result[-1] == "--my-new-parameter=openshift" + msg: "Test: '--my-new-parameter=openshift' != [{{ results.result[-1] }}]" + ###### end test non-existing array append ##### + + ###### test array update modify ##### + - name: test array update modify + yedit: + src: "{{ test_file }}" + key: spec.containers[0].command + value: --root-ca-file=/etc/k8s/ssl/my.pem + curr_value: --root-ca-file=/etc/kubernetes/ssl/ca.pem + curr_value_format: str + update: True + + - name: retrieve the array + yedit: + src: "{{ test_file }}" + state: list + key: spec.containers[0].command + register: results + + - name: Assert that the element in array is our value + assert: + that: results.result[5] == "--root-ca-file=/etc/k8s/ssl/my.pem" + msg: "Test: '--root-ca-file=/etc/k8s/ssl/my.pem' != [{{ results.result[5] }}]" + ###### end test array update modify##### + + ###### test dict create ##### + - name: test dict create + yedit: + src: "{{ test_file }}" + key: a.b.c + value: d + + - name: retrieve the key + yedit: + src: "{{ test_file }}" + state: list + key: a.b.c + register: results + + - name: Assert that the key was created + assert: + that: results.result == "d" + msg: "Test: 'd' != [{{ results.result }}]" + ###### end test dict create ##### + + ###### test create dict value ##### + - name: test create dict value + yedit: + src: "{{ test_file }}" + key: e.f.g + value: + h: + i: + j: k + + - name: retrieve the key + yedit: + src: "{{ test_file }}" + state: list + key: e.f.g.h.i.j + register: results + + - name: Assert that the key was created + assert: + that: results.result == "k" + msg: "Test: 'k' != [{{ results.result }}]" + ###### end test dict create ##### + + ###### test create list value ##### + - name: test create list value + yedit: + src: "{{ test_file }}" + key: z.x.y + value: + - 1 + - 2 + - 3 + + - name: retrieve the key + yedit: + src: "{{ test_file }}" + state: list + key: z#x#y + separator: '#' + register: results + - debug: var=results + + - name: Assert that the key was created + assert: + that: results.result == [1, 2, 3] + msg: "Test: '[1, 2, 3]' != [{{ results.result }}]" +###### end test create list value ##### diff --git a/roles/lib_utils/src/test/unit/yedit_test.py b/roles/lib_utils/src/test/unit/yedit_test.py new file mode 100755 index 000000000..2793c5c1a --- /dev/null +++ b/roles/lib_utils/src/test/unit/yedit_test.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python2 +''' + Unit tests for yedit +''' +# To run +# python -m unittest yedit_test +# +# ............................. +# ---------------------------------------------------------------------- +# Ran 29 tests in 0.133s +# OK + +import os +import sys +import unittest + +# Removing invalid variable names for tests so that I can +# keep them brief +# pylint: disable=invalid-name,no-name-in-module +# Disable import-error b/c our libraries aren't loaded in jenkins +# pylint: disable=import-error +# place yedit in our path +yedit_path = os.path.join('/'.join(os.path.realpath(__file__).split('/')[:-4]), 'library') # noqa: E501 +sys.path.insert(0, yedit_path) + +from yedit import Yedit # noqa: E402 + +# pylint: disable=too-many-public-methods +# Silly pylint, moar tests! + + +class YeditTest(unittest.TestCase): + ''' + Test class for yedit + ''' + data = {'a': 'a', + 'b': {'c': {'d': [{'e': 'x'}, 'f', 'g']}}, + } # noqa: E124 + + filename = 'yedit_test.yml' + + def setUp(self): + ''' setup method will create a file and set to known configuration ''' + yed = Yedit(YeditTest.filename) + yed.yaml_dict = YeditTest.data + yed.write() + + def test_load(self): + ''' Testing a get ''' + yed = Yedit('yedit_test.yml') + self.assertEqual(yed.yaml_dict, self.data) + + def test_write(self): + ''' Testing a simple write ''' + yed = Yedit('yedit_test.yml') + yed.put('key1', 1) + yed.write() + self.assertTrue('key1' in yed.yaml_dict) + self.assertEqual(yed.yaml_dict['key1'], 1) + + def test_write_x_y_z(self): + '''Testing a write of multilayer key''' + yed = Yedit('yedit_test.yml') + yed.put('x.y.z', 'modified') + yed.write() + yed.load() + self.assertEqual(yed.get('x.y.z'), 'modified') + + def test_delete_a(self): + '''Testing a simple delete ''' + yed = Yedit('yedit_test.yml') + yed.delete('a') + yed.write() + yed.load() + self.assertTrue('a' not in yed.yaml_dict) + + def test_delete_b_c(self): + '''Testing delete of layered key ''' + yed = Yedit('yedit_test.yml', separator=':') + yed.delete('b:c') + yed.write() + yed.load() + self.assertTrue('b' in yed.yaml_dict) + self.assertFalse('c' in yed.yaml_dict['b']) + + def test_create(self): + '''Testing a create ''' + os.unlink(YeditTest.filename) + yed = Yedit('yedit_test.yml') + yed.create('foo', 'bar') + yed.write() + yed.load() + self.assertTrue('foo' in yed.yaml_dict) + self.assertTrue(yed.yaml_dict['foo'] == 'bar') + + def test_create_content(self): + '''Testing a create with content ''' + content = {"foo": "bar"} + yed = Yedit("yedit_test.yml", content) + yed.write() + yed.load() + self.assertTrue('foo' in yed.yaml_dict) + self.assertTrue(yed.yaml_dict['foo'], 'bar') + + def test_array_insert(self): + '''Testing a create with content ''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('b:c:d[0]', 'inject') + self.assertTrue(yed.get('b:c:d[0]') == 'inject') + + def test_array_insert_first_index(self): + '''Testing a create with content ''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('b:c:d[0]', 'inject') + self.assertTrue(yed.get('b:c:d[1]') == 'f') + + def test_array_insert_second_index(self): + '''Testing a create with content ''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('b:c:d[0]', 'inject') + self.assertTrue(yed.get('b:c:d[2]') == 'g') + + def test_dict_array_dict_access(self): + '''Testing a create with content''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('b:c:d[0]', [{'x': {'y': 'inject'}}]) + self.assertTrue(yed.get('b:c:d[0]:[0]:x:y') == 'inject') + + def test_dict_array_dict_replace(self): + '''Testing multilevel delete''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('b:c:d[0]', [{'x': {'y': 'inject'}}]) + yed.put('b:c:d[0]:[0]:x:y', 'testing') + self.assertTrue('b' in yed.yaml_dict) + self.assertTrue('c' in yed.yaml_dict['b']) + self.assertTrue('d' in yed.yaml_dict['b']['c']) + self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'], list)) + self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'][0], list)) + self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'][0][0], dict)) + self.assertTrue('y' in yed.yaml_dict['b']['c']['d'][0][0]['x']) + self.assertTrue(yed.yaml_dict['b']['c']['d'][0][0]['x']['y'] == 'testing') # noqa: E501 + + def test_dict_array_dict_remove(self): + '''Testing multilevel delete''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('b:c:d[0]', [{'x': {'y': 'inject'}}]) + yed.delete('b:c:d[0]:[0]:x:y') + self.assertTrue('b' in yed.yaml_dict) + self.assertTrue('c' in yed.yaml_dict['b']) + self.assertTrue('d' in yed.yaml_dict['b']['c']) + self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'], list)) + self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'][0], list)) + self.assertTrue(isinstance(yed.yaml_dict['b']['c']['d'][0][0], dict)) + self.assertFalse('y' in yed.yaml_dict['b']['c']['d'][0][0]['x']) + + def test_key_exists_in_dict(self): + '''Testing exist in dict''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('b:c:d[0]', [{'x': {'y': 'inject'}}]) + self.assertTrue(yed.exists('b:c', 'd')) + + def test_key_exists_in_list(self): + '''Testing exist in list''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('b:c:d[0]', [{'x': {'y': 'inject'}}]) + self.assertTrue(yed.exists('b:c:d', [{'x': {'y': 'inject'}}])) + self.assertFalse(yed.exists('b:c:d', [{'x': {'y': 'test'}}])) + + def test_update_to_list_with_index(self): + '''Testing update to list with index''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('x:y:z', [1, 2, 3]) + yed.update('x:y:z', [5, 6], index=2) + self.assertTrue(yed.get('x:y:z') == [1, 2, [5, 6]]) + self.assertTrue(yed.exists('x:y:z', [5, 6])) + self.assertFalse(yed.exists('x:y:z', 4)) + + def test_update_to_list_with_curr_value(self): + '''Testing update to list with index''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('x:y:z', [1, 2, 3]) + yed.update('x:y:z', [5, 6], curr_value=3) + self.assertTrue(yed.get('x:y:z') == [1, 2, [5, 6]]) + self.assertTrue(yed.exists('x:y:z', [5, 6])) + self.assertFalse(yed.exists('x:y:z', 4)) + + def test_update_to_list(self): + '''Testing update to list''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('x:y:z', [1, 2, 3]) + yed.update('x:y:z', [5, 6]) + self.assertTrue(yed.get('x:y:z') == [1, 2, 3, [5, 6]]) + self.assertTrue(yed.exists('x:y:z', [5, 6])) + self.assertFalse(yed.exists('x:y:z', 4)) + + def test_append_twice_to_list(self): + '''Testing append to list''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('x:y:z', [1, 2, 3]) + yed.append('x:y:z', [5, 6]) + yed.append('x:y:z', [5, 6]) + self.assertTrue(yed.get('x:y:z') == [1, 2, 3, [5, 6], [5, 6]]) + self.assertTrue(2 == yed.get('x:y:z').count([5, 6])) + self.assertFalse(yed.exists('x:y:z', 4)) + + def test_add_item_to_dict(self): + '''Testing update to dict''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('x:y:z', {'a': 1, 'b': 2}) + yed.update('x:y:z', {'c': 3, 'd': 4}) + self.assertTrue(yed.get('x:y:z') == {'a': 1, 'b': 2, 'c': 3, 'd': 4}) + self.assertTrue(yed.exists('x:y:z', {'c': 3})) + + def test_first_level_dict_with_none_value(self): + '''test dict value with none value''' + yed = Yedit(content={'a': None}, separator=":") + yed.put('a:b:c', 'test') + self.assertTrue(yed.get('a:b:c') == 'test') + self.assertTrue(yed.get('a:b'), {'c': 'test'}) + + def test_adding_yaml_variable(self): + '''test dict value with none value''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('z:y', '{{test}}') + self.assertTrue(yed.get('z:y') == '{{test}}') + + def test_keys_with_underscore(self): + '''test dict value with none value''' + yed = Yedit("yedit_test.yml", separator=':') + yed.put('z_:y_y', {'test': '{{test}}'}) + self.assertTrue(yed.get('z_:y_y') == {'test': '{{test}}'}) + + def test_first_level_array_update(self): + '''test update on top level array''' + yed = Yedit(content=[{'a': 1}, {'b': 2}, {'b': 3}], separator=':') + yed.update('', {'c': 4}) + self.assertTrue({'c': 4} in yed.get('')) + + def test_first_level_array_delete(self): + '''test remove top level key''' + yed = Yedit(content=[{'a': 1}, {'b': 2}, {'b': 3}]) + yed.delete('') + self.assertTrue({'b': 3} not in yed.get('')) + + def test_first_level_array_get(self): + '''test dict value with none value''' + yed = Yedit(content=[{'a': 1}, {'b': 2}, {'b': 3}]) + yed.get('') + self.assertTrue([{'a': 1}, {'b': 2}, {'b': 3}] == yed.yaml_dict) + + def test_pop_list_item(self): + '''test dict value with none value''' + yed = Yedit(content=[{'a': 1}, {'b': 2}, {'b': 3}], separator=':') + yed.pop('', {'b': 2}) + self.assertTrue([{'a': 1}, {'b': 3}] == yed.yaml_dict) + + def test_pop_list_item_2(self): + '''test dict value with none value''' + z = range(10) + yed = Yedit(content=z, separator=':') + yed.pop('', 5) + z.pop(5) + self.assertTrue(z == yed.yaml_dict) + + def test_pop_dict_key(self): + '''test dict value with none value''' + yed = Yedit(content={'a': {'b': {'c': 1, 'd': 2}}}, separator='#') + yed.pop('a#b', 'c') + self.assertTrue({'a': {'b': {'d': 2}}} == yed.yaml_dict) + + def tearDown(self): + '''TearDown method''' + os.unlink(YeditTest.filename) + + +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3