From ba823255fcde17124b6dcc447bb00bc241346312 Mon Sep 17 00:00:00 2001 From: Tim Bielawa Date: Thu, 29 Sep 2016 07:32:56 -0700 Subject: Fix conflicts in spec file --- callback_plugins/openshift_quick_installer.py | 115 ++++++++++++++++++++++++++ openshift-ansible.spec | 25 ++++++ utils/Makefile | 4 +- utils/etc/ansible-quiet.cfg | 33 ++++++++ utils/etc/ansible.cfg | 4 +- utils/setup.py | 2 +- utils/src/MANIFEST.in | 1 + utils/src/ooinstall/cli_installer.py | 3 + utils/src/ooinstall/oo_config.py | 6 ++ utils/src/ooinstall/openshift_ansible.py | 16 +++- utils/src/ooinstall/utils.py | 10 +++ 11 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 callback_plugins/openshift_quick_installer.py create mode 100644 utils/etc/ansible-quiet.cfg create mode 100644 utils/src/ooinstall/utils.py diff --git a/callback_plugins/openshift_quick_installer.py b/callback_plugins/openshift_quick_installer.py new file mode 100644 index 000000000..abb22d2fa --- /dev/null +++ b/callback_plugins/openshift_quick_installer.py @@ -0,0 +1,115 @@ +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +This file is a stdout callback plugin for the OpenShift Quick +Installer. The purpose of this callback plugin is to reduce the amount +of produced output for customers and enable simpler progress checking. + +What's different: + +* Playbook progress is expressed as: Play / (Play Name) + Ex: Play 3/30 (Initialize Megafrobber) + +* The Tasks and Handlers in each play (and included roles) are printed + as a series of .'s following the play progress line. + +""" + +from __future__ import (absolute_import, print_function) +import imp +import os +import sys + +ANSIBLE_PATH = imp.find_module('ansible')[1] +DEFAULT_PATH = os.path.join(ANSIBLE_PATH, 'plugins/callback/default.py') +DEFAULT_MODULE = imp.load_source( + 'ansible.plugins.callback.default', + DEFAULT_PATH +) + +try: + from ansible.plugins.callback import CallbackBase + BASECLASS = CallbackBase +except ImportError: # < ansible 2.1 + BASECLASS = DEFAULT_MODULE.CallbackModule + + +reload(sys) +sys.setdefaultencoding('utf-8') + + +class CallbackModule(DEFAULT_MODULE.CallbackModule): + + """ + Ansible callback plugin + """ + CALLBACK_VERSION = 2.2 + CALLBACK_TYPE = 'stdout' + CALLBACK_NAME = 'openshift_quick_installer' + CALLBACK_NEEDS_WHITELIST = False + plays_count = 0 + plays_total_ran = 0 + + def v2_playbook_on_start(self, playbook): + """This is basically the start of it all""" + self.plays_count = len(playbook.get_plays()) + self.plays_total_ran = 0 + + def v2_playbook_on_play_start(self, play): + """Each play calls this once before running any tasks + +We could print the number of tasks here as well by using +`play.get_tasks()` but that is not accurate when a play includes a +role. Only the tasks directly assigned to a play are directly exposed +in the `play` object. + + """ + self.plays_total_ran += 1 + print("") + print("Play %s/%s (%s)" % (self.plays_total_ran, self.plays_count, play.get_name())) + + # pylint: disable=unused-argument,no-self-use + def v2_playbook_on_task_start(self, task, is_conditional): + """This prints out the task header. For example: + +TASK [openshift_facts : Ensure PyYaml is installed] ***... + +Rather than print out all that for every task, we print a dot +character to indicate a task has been started. + """ + sys.stdout.write('.') + + def v2_runner_on_ok(self, result): + """This prints out task results in a fancy format""" + pass + + def v2_runner_item_on_ok(self, result): + """Print out task results for you're iterating""" + pass + + def v2_runner_item_on_skipped(self, result): + """Print out task results when an item is skipped""" + pass + + def v2_runner_on_skipped(self, result): + """Print out task results when a task (or something else?) is skipped""" + pass + + def v2_playbook_on_notify(self, res, handler): + """Printer for handlers + +Rather than print out a header for every handler, we print a dot +character to indicate a handler task has been started. + """ + sys.stdout.write('.') diff --git a/openshift-ansible.spec b/openshift-ansible.spec index 5e297cb2a..d31447d7a 100644 --- a/openshift-ansible.spec +++ b/openshift-ansible.spec @@ -70,6 +70,9 @@ cp -rp filter_plugins %{buildroot}%{_datadir}/ansible_plugins/ # openshift-ansible-lookup-plugins install cp -rp lookup_plugins %{buildroot}%{_datadir}/ansible_plugins/ +# openshift-ansible-callback-plugins install +cp -rp callback_plugins %{buildroot}%{_datadir}/ansible_plugins/ + # create symlinks from /usr/share/ansible/plugins/lookup -> # /usr/share/ansible_plugins/lookup_plugins pushd %{buildroot}%{_datadir} @@ -77,6 +80,7 @@ mkdir -p ansible/plugins pushd ansible/plugins ln -s ../../ansible_plugins/lookup_plugins lookup ln -s ../../ansible_plugins/filter_plugins filter +ln -s ../../ansible_plugins/callback_plugins callback popd popd @@ -89,6 +93,7 @@ mkdir -p %{buildroot}%{_datadir}/atomic-openshift-utils/ cp etc/ansible.cfg %{buildroot}%{_datadir}/atomic-openshift-utils/ansible.cfg mkdir -p %{buildroot}%{_mandir}/man1/ cp -v docs/man/man1/atomic-openshift-installer.1 %{buildroot}%{_mandir}/man1/ +cp etc/ansible-quiet.cfg %{buildroot}%{_datadir}/atomic-openshift-utils/ansible-quiet.cfg popd # Base openshift-ansible files @@ -122,6 +127,7 @@ Requires: %{name} = %{version} Requires: %{name}-roles = %{version} Requires: %{name}-lookup-plugins = %{version} Requires: %{name}-filter-plugins = %{version} +Requires: %{name}-callback-plugins = %{version} BuildArch: noarch %description playbooks @@ -158,6 +164,7 @@ Summary: Openshift and Atomic Enterprise Ansible roles Requires: %{name} = %{version} Requires: %{name}-lookup-plugins = %{version} Requires: %{name}-filter-plugins = %{version} +Requires: %{name}-callback-plugins = %{version} BuildArch: noarch %description roles @@ -199,6 +206,22 @@ BuildArch: noarch %{_datadir}/ansible_plugins/lookup_plugins %{_datadir}/ansible/plugins/lookup + +# ---------------------------------------------------------------------------------- +# openshift-ansible-callback-plugins subpackage +# ---------------------------------------------------------------------------------- +%package callback-plugins +Summary: Openshift and Atomic Enterprise Ansible callback plugins +Requires: %{name} = %{version} +BuildArch: noarch + +%description callback-plugins +%{summary}. + +%files callback-plugins +%{_datadir}/ansible_plugins/callback_plugins +%{_datadir}/ansible/plugins/callback + # ---------------------------------------------------------------------------------- # atomic-openshift-utils subpackage # ---------------------------------------------------------------------------------- @@ -222,6 +245,8 @@ Atomic OpenShift Utilities includes %{_bindir}/atomic-openshift-installer %{_datadir}/atomic-openshift-utils/ansible.cfg %{_mandir}/man1/* +%{_datadir}/atomic-openshift-utils/ansible-quiet.cfg + %changelog * Mon Sep 26 2016 Scott Dodson 3.4.2-1 diff --git a/utils/Makefile b/utils/Makefile index 078d6f184..59aff92fd 100644 --- a/utils/Makefile +++ b/utils/Makefile @@ -80,7 +80,7 @@ ci-pylint: @echo "#############################################" @echo "# Running PyLint Tests in virtualenv" @echo "#############################################" - . $(NAME)env/bin/activate && python -m pylint --rcfile ../git/.pylintrc src/ooinstall/cli_installer.py src/ooinstall/oo_config.py src/ooinstall/openshift_ansible.py src/ooinstall/variants.py + . $(NAME)env/bin/activate && python -m pylint --rcfile ../git/.pylintrc src/ooinstall/cli_installer.py src/ooinstall/oo_config.py src/ooinstall/openshift_ansible.py src/ooinstall/variants.py ../callback_plugins/openshift_quick_installer.py ci-list-deps: @echo "#############################################" @@ -93,12 +93,14 @@ ci-pyflakes: @echo "# Running Pyflakes Compliance Tests in virtualenv" @echo "#################################################" . $(NAME)env/bin/activate && pyflakes src/ooinstall/*.py + . $(NAME)env/bin/activate && pyflakes ../callback_plugins/openshift_quick_installer.py ci-pep8: @echo "#############################################" @echo "# Running PEP8 Compliance Tests in virtualenv" @echo "#############################################" . $(NAME)env/bin/activate && pep8 --ignore=E501,E121,E124 src/$(SHORTNAME)/ + . $(NAME)env/bin/activate && pep8 --ignore=E501,E121,E124 ../callback_plugins/openshift_quick_installer.py ci: clean virtualenv ci-list-deps ci-pep8 ci-pylint ci-pyflakes ci-unittests : diff --git a/utils/etc/ansible-quiet.cfg b/utils/etc/ansible-quiet.cfg new file mode 100644 index 000000000..0eb0efa49 --- /dev/null +++ b/utils/etc/ansible-quiet.cfg @@ -0,0 +1,33 @@ +# config file for ansible -- http://ansible.com/ +# ============================================== + +# This config file provides examples for running +# the OpenShift playbooks with the provided +# inventory scripts. Only global defaults are +# left uncommented + +[defaults] +# Add the roles directory to the roles path +roles_path = roles/ + +# Set the log_path +log_path = /tmp/ansible.log + +forks = 10 +host_key_checking = False +nocows = 1 + +retry_files_enabled = False + +deprecation_warnings=False + +# Need to handle: +# inventory - derive from OO_ANSIBLE_DIRECTORY env var +# callback_plugins - derive from pkg_resource.resource_filename +# private_key_file - prompt if missing +# remote_tmp - set if provided by user (cli) +# ssh_args - set if provided by user (cli) +# control_path + +stdout_callback = openshift_quick_installer +callback_plugins = /usr/share/ansible_plugins/callback_plugins diff --git a/utils/etc/ansible.cfg b/utils/etc/ansible.cfg index a53ab6cb1..3425e7e62 100644 --- a/utils/etc/ansible.cfg +++ b/utils/etc/ansible.cfg @@ -19,10 +19,12 @@ nocows = 1 retry_files_enabled = False +deprecation_warnings = False + # Need to handle: # inventory - derive from OO_ANSIBLE_DIRECTORY env var # callback_plugins - derive from pkg_resource.resource_filename # private_key_file - prompt if missing # remote_tmp - set if provided by user (cli) # ssh_args - set if provided by user (cli) -# control_path \ No newline at end of file +# control_path diff --git a/utils/setup.py b/utils/setup.py index eac1b4b2e..563897bb1 100644 --- a/utils/setup.py +++ b/utils/setup.py @@ -62,7 +62,7 @@ setup( # installed, specify them here. If using Python 2.6 or less, then these # have to be included in MANIFEST.in as well. package_data={ - 'ooinstall': ['ansible.cfg', 'ansible_plugins/*'], + 'ooinstall': ['ansible.cfg', 'ansible-quiet.cfg', 'ansible_plugins/*'], }, # Although 'package_data' is the preferred approach, in some case you may diff --git a/utils/src/MANIFEST.in b/utils/src/MANIFEST.in index d4153e738..216f57e9c 100644 --- a/utils/src/MANIFEST.in +++ b/utils/src/MANIFEST.in @@ -7,3 +7,4 @@ include DESCRIPTION.rst # it's already declared in setup.py include ooinstall/* include ansible.cfg +include ansible-quiet.cfg diff --git a/utils/src/ooinstall/cli_installer.py b/utils/src/ooinstall/cli_installer.py index 85b4d29cb..81dda2ea4 100644 --- a/utils/src/ooinstall/cli_installer.py +++ b/utils/src/ooinstall/cli_installer.py @@ -25,6 +25,7 @@ installer_file_handler.setLevel(logging.DEBUG) installer_log.addHandler(installer_file_handler) DEFAULT_ANSIBLE_CONFIG = '/usr/share/atomic-openshift-utils/ansible.cfg' +QUIET_ANSIBLE_CONFIG = '/usr/share/atomic-openshift-utils/ansible-quiet.cfg' DEFAULT_PLAYBOOK_DIR = '/usr/share/ansible/openshift-ansible/' UPGRADE_MAPPINGS = { @@ -883,6 +884,8 @@ def cli(ctx, unattended, configuration, ansible_playbook_directory, ansible_conf # If we're installed by RPM this file should exist and we can use it as our default: oo_cfg.settings['ansible_config'] = DEFAULT_ANSIBLE_CONFIG + oo_cfg.settings['ansible_quiet_config'] = QUIET_ANSIBLE_CONFIG + oo_cfg.settings['ansible_log_path'] = ctx.obj['ansible_log_path'] ctx.obj['oo_cfg'] = oo_cfg diff --git a/utils/src/ooinstall/oo_config.py b/utils/src/ooinstall/oo_config.py index 393b36f6f..ce0ae8deb 100644 --- a/utils/src/ooinstall/oo_config.py +++ b/utils/src/ooinstall/oo_config.py @@ -308,6 +308,12 @@ class OOConfig(object): if 'ansible_plugins_directory' not in self.settings: self.settings['ansible_plugins_directory'] = \ resource_filename(__name__, 'ansible_plugins') + installer_log.debug("We think the ansible plugins directory should be: %s (it is not already set)", + self.settings['ansible_plugins_directory']) + else: + installer_log.debug("The ansible plugins directory is already set: %s", + self.settings['ansible_plugins_directory']) + if 'version' not in self.settings: self.settings['version'] = 'v2' diff --git a/utils/src/ooinstall/openshift_ansible.py b/utils/src/ooinstall/openshift_ansible.py index 75d26c10a..79c60af82 100644 --- a/utils/src/ooinstall/openshift_ansible.py +++ b/utils/src/ooinstall/openshift_ansible.py @@ -7,6 +7,7 @@ import os import logging import yaml from ooinstall.variants import find_variant +from ooinstall.utils import debug_env installer_log = logging.getLogger('installer') @@ -225,6 +226,9 @@ def load_system_facts(inventory_file, os_facts_path, env_vars, verbose=False): Retrieves system facts from the remote systems. """ installer_log.debug("Inside load_system_facts") + installer_log.debug("load_system_facts will run with Ansible/Openshift environment variables:") + debug_env(env_vars) + FNULL = open(os.devnull, 'w') args = ['ansible-playbook', '-v'] if verbose \ else ['ansible-playbook'] @@ -232,6 +236,8 @@ def load_system_facts(inventory_file, os_facts_path, env_vars, verbose=False): '--inventory-file={}'.format(inventory_file), os_facts_path]) installer_log.debug("Going to subprocess out to ansible now with these args: %s", ' '.join(args)) + installer_log.debug("Subprocess will run with Ansible/Openshift environment variables:") + debug_env(env_vars) status = subprocess.call(args, env=env_vars, stdout=FNULL) if status != 0: installer_log.debug("Exit status from subprocess was not 0") @@ -280,17 +286,23 @@ def run_main_playbook(inventory_file, hosts, hosts_to_run_on, verbose=False): facts_env = os.environ.copy() if 'ansible_log_path' in CFG.settings: facts_env['ANSIBLE_LOG_PATH'] = CFG.settings['ansible_log_path'] - if 'ansible_config' in CFG.settings: - facts_env['ANSIBLE_CONFIG'] = CFG.settings['ansible_config'] + if 'ansible_quiet_config' in CFG.settings: + facts_env['ANSIBLE_CONFIG'] = CFG.settings['ansible_quiet_config'] + # facts_env["ANSIBLE_CALLBACK_PLUGINS"] = CFG.settings['ansible_plugins_directory'] + return run_ansible(main_playbook_path, inventory_file, facts_env, verbose) def run_ansible(playbook, inventory, env_vars, verbose=False): + installer_log.debug("run_ansible will run with Ansible/Openshift environment variables:") + debug_env(env_vars) + args = ['ansible-playbook', '-v'] if verbose \ else ['ansible-playbook'] args.extend([ '--inventory-file={}'.format(inventory), playbook]) + installer_log.debug("Going to subprocess out to ansible now with these args: %s", ' '.join(args)) return subprocess.call(args, env=env_vars) diff --git a/utils/src/ooinstall/utils.py b/utils/src/ooinstall/utils.py new file mode 100644 index 000000000..eb27a57e4 --- /dev/null +++ b/utils/src/ooinstall/utils.py @@ -0,0 +1,10 @@ +import logging + +installer_log = logging.getLogger('installer') + + +def debug_env(env): + for k in sorted(env.keys()): + if k.startswith("OPENSHIFT") or k.startswith("ANSIBLE") or k.startswith("OO"): + installer_log.debug("{key}: {value}".format( + key=k, value=env[k])) -- cgit v1.2.3 From d53389707f4838249715ccc6e67bdc6c06c4cada Mon Sep 17 00:00:00 2001 From: Tim Bielawa Date: Fri, 23 Sep 2016 10:56:56 -0700 Subject: Silence/dot-print more actions in the callback --- callback_plugins/openshift_quick_installer.py | 51 ++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/callback_plugins/openshift_quick_installer.py b/callback_plugins/openshift_quick_installer.py index abb22d2fa..9d490f02c 100644 --- a/callback_plugins/openshift_quick_installer.py +++ b/callback_plugins/openshift_quick_installer.py @@ -44,6 +44,7 @@ try: except ImportError: # < ansible 2.1 BASECLASS = DEFAULT_MODULE.CallbackModule +from ansible import constants as C reload(sys) sys.setdefaultencoding('utf-8') @@ -61,6 +62,18 @@ class CallbackModule(DEFAULT_MODULE.CallbackModule): plays_count = 0 plays_total_ran = 0 + def banner(self, msg, color=None): + ''' + Prints a header-looking line with stars taking up to 80 columns + of width (3 columns, minimum) + ''' + msg = msg.strip() + star_len = (79 - len(msg)) + if star_len < 0: + star_len = 3 + stars = "*" * star_len + self._display.display("\n%s %s" % (msg, stars), color=color, log_only=True) + def v2_playbook_on_start(self, playbook): """This is basically the start of it all""" self.plays_count = len(playbook.get_plays()) @@ -79,6 +92,14 @@ in the `play` object. print("") print("Play %s/%s (%s)" % (self.plays_total_ran, self.plays_count, play.get_name())) + name = play.get_name().strip() + if not name: + msg = "PLAY" + else: + msg = "PLAY [%s]" % name + + self.banner(msg) + # pylint: disable=unused-argument,no-self-use def v2_playbook_on_task_start(self, task, is_conditional): """This prints out the task header. For example: @@ -90,6 +111,28 @@ character to indicate a task has been started. """ sys.stdout.write('.') + # pylint: disable=unused-argument,no-self-use + def v2_playbook_on_handler_task_start(self, task): + """Print out task header for handlers + +Rather than print out a header for every handler, we print a dot +character to indicate a handler task has been started. +""" + sys.stdout.write('.') + + # pylint: disable=unused-argument,no-self-use + def v2_playbook_on_cleanup_task_start(self, task): + """Print out a task header for cleanup tasks + +Rather than print out a header for every handler, we print a dot +character to indicate a handler task has been started. +""" + sys.stdout.write('.') + + def v2_playbook_on_include(self, included_file): + """Print out paths to statically included files""" + pass + def v2_runner_on_ok(self, result): """This prints out task results in a fancy format""" pass @@ -107,9 +150,7 @@ character to indicate a task has been started. pass def v2_playbook_on_notify(self, res, handler): - """Printer for handlers - -Rather than print out a header for every handler, we print a dot -character to indicate a handler task has been started. + """What happens when a task result is 'changed' and the task has a +'notify' list attached. """ - sys.stdout.write('.') + pass -- cgit v1.2.3 From 088e4ba10dfec171618890ef022c7e86476823bc Mon Sep 17 00:00:00 2001 From: Tim Bielawa Date: Fri, 23 Sep 2016 13:01:33 -0700 Subject: Copy and paste more methods --- callback_plugins/openshift_quick_installer.py | 100 ++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 7 deletions(-) diff --git a/callback_plugins/openshift_quick_installer.py b/callback_plugins/openshift_quick_installer.py index 9d490f02c..f6cc43cf9 100644 --- a/callback_plugins/openshift_quick_installer.py +++ b/callback_plugins/openshift_quick_installer.py @@ -79,6 +79,10 @@ class CallbackModule(DEFAULT_MODULE.CallbackModule): self.plays_count = len(playbook.get_plays()) self.plays_total_ran = 0 + if self._display.verbosity > 1: + from os.path import basename + self.banner("PLAYBOOK: %s" % basename(playbook._file_name)) + def v2_playbook_on_play_start(self, play): """Each play calls this once before running any tasks @@ -111,6 +115,24 @@ character to indicate a task has been started. """ sys.stdout.write('.') + args = '' + # args can be specified as no_log in several places: in the task or in + # the argument spec. We can check whether the task is no_log but the + # argument spec can't be because that is only run on the target + # machine and we haven't run it thereyet at this time. + # + # So we give people a config option to affect display of the args so + # that they can secure this if they feel that their stdout is insecure + # (shoulder surfing, logging stdout straight to a file, etc). + if not task.no_log and C.DISPLAY_ARGS_TO_STDOUT: + args = ', '.join(('%s=%s' % a for a in task.args.items())) + args = ' %s' % args + self.banner("TASK [%s%s]" % (task.get_name().strip(), args)) + if self._display.verbosity >= 2: + path = task.get_path() + if path: + self._display.display("task path: %s" % path, color=C.COLOR_DEBUG, log_only=True) + # pylint: disable=unused-argument,no-self-use def v2_playbook_on_handler_task_start(self, task): """Print out task header for handlers @@ -120,6 +142,8 @@ character to indicate a handler task has been started. """ sys.stdout.write('.') + self.banner("RUNNING HANDLER [%s]" % task.get_name().strip()) + # pylint: disable=unused-argument,no-self-use def v2_playbook_on_cleanup_task_start(self, task): """Print out a task header for cleanup tasks @@ -129,28 +153,90 @@ character to indicate a handler task has been started. """ sys.stdout.write('.') + self.banner("CLEANUP TASK [%s]" % task.get_name().strip()) + def v2_playbook_on_include(self, included_file): """Print out paths to statically included files""" - pass + msg = 'included: %s for %s' % (included_file._filename, ", ".join([h.name for h in included_file._hosts])) + self._display.display(msg, color=C.COLOR_SKIP, log_only=True) def v2_runner_on_ok(self, result): - """This prints out task results in a fancy format""" - pass + """This prints out task results in a fancy format + +The only thing we change here is adding `log_only=True` to the +.display() call + """ + delegated_vars = result._result.get('_ansible_delegated_vars', None) + self._clean_results(result._result, result._task.action) + if result._task.action in ('include', 'include_role'): + return + elif result._result.get('changed', False): + if delegated_vars: + msg = "changed: [%s -> %s]" % (result._host.get_name(), delegated_vars['ansible_host']) + else: + msg = "changed: [%s]" % result._host.get_name() + color = C.COLOR_CHANGED + else: + if delegated_vars: + msg = "ok: [%s -> %s]" % (result._host.get_name(), delegated_vars['ansible_host']) + else: + msg = "ok: [%s]" % result._host.get_name() + color = C.COLOR_OK + + if result._task.loop and 'results' in result._result: + self._process_items(result) + else: + + if (self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and not '_ansible_verbose_override' in result._result: + msg += " => %s" % (self._dump_results(result._result),) + self._display.display(msg, color=color, log_only=True) + + self._handle_warnings(result._result) def v2_runner_item_on_ok(self, result): """Print out task results for you're iterating""" - pass + delegated_vars = result._result.get('_ansible_delegated_vars', None) + if result._task.action in ('include', 'include_role'): + return + elif result._result.get('changed', False): + msg = 'changed' + color = C.COLOR_CHANGED + else: + msg = 'ok' + color = C.COLOR_OK + + if delegated_vars: + msg += ": [%s -> %s]" % (result._host.get_name(), delegated_vars['ansible_host']) + else: + msg += ": [%s]" % result._host.get_name() + + msg += " => (item=%s)" % (self._get_item(result._result),) + + if (self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and not '_ansible_verbose_override' in result._result: + msg += " => %s" % self._dump_results(result._result) + self._display.display(msg, color=color, log_only=True) def v2_runner_item_on_skipped(self, result): """Print out task results when an item is skipped""" - pass + if C.DISPLAY_SKIPPED_HOSTS: + msg = "skipping: [%s] => (item=%s) " % (result._host.get_name(), self._get_item(result._result)) + if (self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and not '_ansible_verbose_override' in result._result: + msg += " => %s" % self._dump_results(result._result) + self._display.display(msg, color=C.COLOR_SKIP, log_only=True) def v2_runner_on_skipped(self, result): """Print out task results when a task (or something else?) is skipped""" - pass + if C.DISPLAY_SKIPPED_HOSTS: + if result._task.loop and 'results' in result._result: + self._process_items(result) + else: + msg = "skipping: [%s]" % result._host.get_name() + if (self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and not '_ansible_verbose_override' in result._result: + msg += " => %s" % self._dump_results(result._result) + self._display.display(msg, color=C.COLOR_SKIP, log_only=True) def v2_playbook_on_notify(self, res, handler): """What happens when a task result is 'changed' and the task has a 'notify' list attached. """ - pass + self._display.display("skipping: no hosts matched", color=C.COLOR_SKIP, log_only=True) -- cgit v1.2.3 From 76f1cd09097dc28c9efe7e2476534fb8a9cfede5 Mon Sep 17 00:00:00 2001 From: Tim Bielawa Date: Mon, 26 Sep 2016 12:03:36 -0700 Subject: 'fix' unittests by removing the users ability to specify an ansible config --- callback_plugins/openshift_quick_installer.py | 37 +++--- utils/src/ooinstall/cli_installer.py | 17 +-- utils/src/ooinstall/oo_config.py | 1 - utils/src/ooinstall/openshift_ansible.py | 3 +- utils/test/cli_installer_tests.py | 166 ++++++++++++++------------ 5 files changed, 119 insertions(+), 105 deletions(-) diff --git a/callback_plugins/openshift_quick_installer.py b/callback_plugins/openshift_quick_installer.py index f6cc43cf9..7923a9a80 100644 --- a/callback_plugins/openshift_quick_installer.py +++ b/callback_plugins/openshift_quick_installer.py @@ -1,3 +1,5 @@ +# pylint: disable=invalid-name,protected-access,import-error,line-too-long + # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -11,8 +13,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -""" -This file is a stdout callback plugin for the OpenShift Quick +"""This file is a stdout callback plugin for the OpenShift Quick Installer. The purpose of this callback plugin is to reduce the amount of produced output for customers and enable simpler progress checking. @@ -24,13 +25,21 @@ What's different: * The Tasks and Handlers in each play (and included roles) are printed as a series of .'s following the play progress line. +* Many of these methods include copy and paste code from the upstream + default.py callback. We do that to give us control over the stdout + output while allowing Ansible to handle the file logging + normally. The biggest changes here are that we are manually setting + `log_only` to True in the Display.display method and we redefine the + Display.banner method locally so we can set log_only on that call as + well. + """ from __future__ import (absolute_import, print_function) import imp import os import sys - +from ansible import constants as C ANSIBLE_PATH = imp.find_module('ansible')[1] DEFAULT_PATH = os.path.join(ANSIBLE_PATH, 'plugins/callback/default.py') DEFAULT_MODULE = imp.load_source( @@ -44,7 +53,6 @@ try: except ImportError: # < ansible 2.1 BASECLASS = DEFAULT_MODULE.CallbackModule -from ansible import constants as C reload(sys) sys.setdefaultencoding('utf-8') @@ -63,9 +71,11 @@ class CallbackModule(DEFAULT_MODULE.CallbackModule): plays_total_ran = 0 def banner(self, msg, color=None): - ''' - Prints a header-looking line with stars taking up to 80 columns + '''Prints a header-looking line with stars taking up to 80 columns of width (3 columns, minimum) + + Overrides the upstream banner method so that display is called + with log_only=True ''' msg = msg.strip() star_len = (79 - len(msg)) @@ -88,9 +98,8 @@ class CallbackModule(DEFAULT_MODULE.CallbackModule): We could print the number of tasks here as well by using `play.get_tasks()` but that is not accurate when a play includes a -role. Only the tasks directly assigned to a play are directly exposed -in the `play` object. - +role. Only the tasks directly assigned to a play are exposed in the +`play` object. """ self.plays_total_ran += 1 print("") @@ -187,14 +196,14 @@ The only thing we change here is adding `log_only=True` to the self._process_items(result) else: - if (self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and not '_ansible_verbose_override' in result._result: + if (self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and '_ansible_verbose_override' not in result._result: msg += " => %s" % (self._dump_results(result._result),) self._display.display(msg, color=color, log_only=True) self._handle_warnings(result._result) def v2_runner_item_on_ok(self, result): - """Print out task results for you're iterating""" + """Print out task results for items you're iterating over""" delegated_vars = result._result.get('_ansible_delegated_vars', None) if result._task.action in ('include', 'include_role'): return @@ -212,7 +221,7 @@ The only thing we change here is adding `log_only=True` to the msg += " => (item=%s)" % (self._get_item(result._result),) - if (self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and not '_ansible_verbose_override' in result._result: + if (self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and '_ansible_verbose_override' not in result._result: msg += " => %s" % self._dump_results(result._result) self._display.display(msg, color=color, log_only=True) @@ -220,7 +229,7 @@ The only thing we change here is adding `log_only=True` to the """Print out task results when an item is skipped""" if C.DISPLAY_SKIPPED_HOSTS: msg = "skipping: [%s] => (item=%s) " % (result._host.get_name(), self._get_item(result._result)) - if (self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and not '_ansible_verbose_override' in result._result: + if (self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and '_ansible_verbose_override' not in result._result: msg += " => %s" % self._dump_results(result._result) self._display.display(msg, color=C.COLOR_SKIP, log_only=True) @@ -231,7 +240,7 @@ The only thing we change here is adding `log_only=True` to the self._process_items(result) else: msg = "skipping: [%s]" % result._host.get_name() - if (self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and not '_ansible_verbose_override' in result._result: + if (self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and '_ansible_verbose_override' not in result._result: msg += " => %s" % self._dump_results(result._result) self._display.display(msg, color=C.COLOR_SKIP, log_only=True) diff --git a/utils/src/ooinstall/cli_installer.py b/utils/src/ooinstall/cli_installer.py index 81dda2ea4..674d1cee4 100644 --- a/utils/src/ooinstall/cli_installer.py +++ b/utils/src/ooinstall/cli_installer.py @@ -816,12 +816,6 @@ def set_infra_nodes(hosts): # callback=validate_ansible_dir, default=DEFAULT_PLAYBOOK_DIR, envvar='OO_ANSIBLE_PLAYBOOK_DIRECTORY') -@click.option('--ansible-config', - type=click.Path(file_okay=True, - dir_okay=False, - writable=True, - readable=True), - default=None) @click.option('--ansible-log-path', type=click.Path(file_okay=True, dir_okay=False, @@ -837,7 +831,7 @@ def set_infra_nodes(hosts): # pylint: disable=too-many-arguments # pylint: disable=line-too-long # Main CLI entrypoint, not much we can do about too many arguments. -def cli(ctx, unattended, configuration, ansible_playbook_directory, ansible_config, ansible_log_path, verbose, debug): +def cli(ctx, unattended, configuration, ansible_playbook_directory, ansible_log_path, verbose, debug): """ atomic-openshift-installer makes the process for installing OSE or AEP easier by interactively gathering the data needed to run on each host. @@ -856,7 +850,6 @@ def cli(ctx, unattended, configuration, ansible_playbook_directory, ansible_conf ctx.obj = {} ctx.obj['unattended'] = unattended ctx.obj['configuration'] = configuration - ctx.obj['ansible_config'] = ansible_config ctx.obj['ansible_log_path'] = ansible_log_path ctx.obj['verbose'] = verbose @@ -877,14 +870,12 @@ def cli(ctx, unattended, configuration, ansible_playbook_directory, ansible_conf oo_cfg.ansible_playbook_directory = ansible_playbook_directory ctx.obj['ansible_playbook_directory'] = ansible_playbook_directory - if ctx.obj['ansible_config']: - oo_cfg.settings['ansible_config'] = ctx.obj['ansible_config'] - elif 'ansible_config' not in oo_cfg.settings and \ - os.path.exists(DEFAULT_ANSIBLE_CONFIG): + if os.path.exists(DEFAULT_ANSIBLE_CONFIG): # If we're installed by RPM this file should exist and we can use it as our default: oo_cfg.settings['ansible_config'] = DEFAULT_ANSIBLE_CONFIG - oo_cfg.settings['ansible_quiet_config'] = QUIET_ANSIBLE_CONFIG + if os.path.exists(QUIET_ANSIBLE_CONFIG): + oo_cfg.settings['ansible_quiet_config'] = QUIET_ANSIBLE_CONFIG oo_cfg.settings['ansible_log_path'] = ctx.obj['ansible_log_path'] diff --git a/utils/src/ooinstall/oo_config.py b/utils/src/ooinstall/oo_config.py index ce0ae8deb..8c5f3396b 100644 --- a/utils/src/ooinstall/oo_config.py +++ b/utils/src/ooinstall/oo_config.py @@ -12,7 +12,6 @@ installer_log = logging.getLogger('installer') CONFIG_PERSIST_SETTINGS = [ 'ansible_ssh_user', 'ansible_callback_facts_yaml', - 'ansible_config', 'ansible_inventory_path', 'ansible_log_path', 'deployment', diff --git a/utils/src/ooinstall/openshift_ansible.py b/utils/src/ooinstall/openshift_ansible.py index 79c60af82..eb1e61a60 100644 --- a/utils/src/ooinstall/openshift_ansible.py +++ b/utils/src/ooinstall/openshift_ansible.py @@ -286,9 +286,10 @@ def run_main_playbook(inventory_file, hosts, hosts_to_run_on, verbose=False): facts_env = os.environ.copy() if 'ansible_log_path' in CFG.settings: facts_env['ANSIBLE_LOG_PATH'] = CFG.settings['ansible_log_path'] + + # override the ansible config for our main playbook run if 'ansible_quiet_config' in CFG.settings: facts_env['ANSIBLE_CONFIG'] = CFG.settings['ansible_quiet_config'] - # facts_env["ANSIBLE_CALLBACK_PLUGINS"] = CFG.settings['ansible_plugins_directory'] return run_ansible(main_playbook_path, inventory_file, facts_env, verbose) diff --git a/utils/test/cli_installer_tests.py b/utils/test/cli_installer_tests.py index 6d9d443ff..34392777b 100644 --- a/utils/test/cli_installer_tests.py +++ b/utils/test/cli_installer_tests.py @@ -599,82 +599,96 @@ class UnattendedCliTests(OOCliFixture): self.assertEquals('openshift-enterprise', inventory.get('OSEv3:vars', 'deployment_type')) - @patch('ooinstall.openshift_ansible.run_ansible') - @patch('ooinstall.openshift_ansible.load_system_facts') - def test_no_ansible_config_specified(self, load_facts_mock, run_ansible_mock): - load_facts_mock.return_value = (MOCK_FACTS, 0) - run_ansible_mock.return_value = 0 - - config = SAMPLE_CONFIG % 'openshift-enterprise' - - self._ansible_config_test(load_facts_mock, run_ansible_mock, - config, None, None) - - @patch('ooinstall.openshift_ansible.run_ansible') - @patch('ooinstall.openshift_ansible.load_system_facts') - def test_ansible_config_specified_cli(self, load_facts_mock, run_ansible_mock): - load_facts_mock.return_value = (MOCK_FACTS, 0) - run_ansible_mock.return_value = 0 - - config = SAMPLE_CONFIG % 'openshift-enterprise' - ansible_config = os.path.join(self.work_dir, 'ansible.cfg') - - self._ansible_config_test(load_facts_mock, run_ansible_mock, - config, ansible_config, ansible_config) - - @patch('ooinstall.openshift_ansible.run_ansible') - @patch('ooinstall.openshift_ansible.load_system_facts') - def test_ansible_config_specified_in_installer_config(self, - load_facts_mock, run_ansible_mock): - - load_facts_mock.return_value = (MOCK_FACTS, 0) - run_ansible_mock.return_value = 0 - - ansible_config = os.path.join(self.work_dir, 'ansible.cfg') - config = SAMPLE_CONFIG % 'openshift-enterprise' - config = "%s\nansible_config: %s" % (config, ansible_config) - self._ansible_config_test(load_facts_mock, run_ansible_mock, - config, None, ansible_config) - - #pylint: disable=too-many-arguments - # This method allows for drastically simpler tests to write, and the args - # are all useful. - def _ansible_config_test(self, load_facts_mock, run_ansible_mock, - installer_config, ansible_config_cli=None, expected_result=None): - """ - Utility method for testing the ways you can specify the ansible config. - """ - - load_facts_mock.return_value = (MOCK_FACTS, 0) - run_ansible_mock.return_value = 0 - - config_file = self.write_config(os.path.join(self.work_dir, - 'ooinstall.conf'), installer_config) - - self.cli_args.extend(["-c", config_file]) - if ansible_config_cli: - self.cli_args.extend(["--ansible-config", ansible_config_cli]) - self.cli_args.append("install") - result = self.runner.invoke(cli.cli, self.cli_args) - self.assert_result(result, 0) - - # Test the env vars for facts playbook: - facts_env_vars = load_facts_mock.call_args[0][2] - if expected_result: - self.assertEquals(expected_result, facts_env_vars['ANSIBLE_CONFIG']) - else: - # If user running test has rpm installed, this might be set to default: - self.assertTrue('ANSIBLE_CONFIG' not in facts_env_vars or - facts_env_vars['ANSIBLE_CONFIG'] == cli.DEFAULT_ANSIBLE_CONFIG) - - # Test the env vars for main playbook: - env_vars = run_ansible_mock.call_args[0][2] - if expected_result: - self.assertEquals(expected_result, env_vars['ANSIBLE_CONFIG']) - else: - # If user running test has rpm installed, this might be set to default: - self.assertTrue('ANSIBLE_CONFIG' not in env_vars or - env_vars['ANSIBLE_CONFIG'] == cli.DEFAULT_ANSIBLE_CONFIG) + # 2016-09-26 - tbielawa - COMMENTING OUT these tests FOR NOW while + # we wait to see if anyone notices that we took away their ability + # to set the ansible_config parameter in the command line options + # and in the installer config file. + # + # We have removed the ability to set the ansible config file + # manually so that our new quieter output mode is the default and + # only output mode. + # + # RE: https://trello.com/c/DSwwizwP - atomic-openshift-install + # should only output relevant information. + + # @patch('ooinstall.openshift_ansible.run_ansible') + # @patch('ooinstall.openshift_ansible.load_system_facts') + # def test_no_ansible_config_specified(self, load_facts_mock, run_ansible_mock): + # load_facts_mock.return_value = (MOCK_FACTS, 0) + # run_ansible_mock.return_value = 0 + + # config = SAMPLE_CONFIG % 'openshift-enterprise' + + # self._ansible_config_test(load_facts_mock, run_ansible_mock, + # config, None, None) + + # @patch('ooinstall.openshift_ansible.run_ansible') + # @patch('ooinstall.openshift_ansible.load_system_facts') + # def test_ansible_config_specified_cli(self, load_facts_mock, run_ansible_mock): + # load_facts_mock.return_value = (MOCK_FACTS, 0) + # run_ansible_mock.return_value = 0 + + # config = SAMPLE_CONFIG % 'openshift-enterprise' + # ansible_config = os.path.join(self.work_dir, 'ansible.cfg') + + # self._ansible_config_test(load_facts_mock, run_ansible_mock, + # config, ansible_config, ansible_config) + + # @patch('ooinstall.openshift_ansible.run_ansible') + # @patch('ooinstall.openshift_ansible.load_system_facts') + # def test_ansible_config_specified_in_installer_config(self, + # load_facts_mock, run_ansible_mock): + + # load_facts_mock.return_value = (MOCK_FACTS, 0) + # run_ansible_mock.return_value = 0 + + # ansible_config = os.path.join(self.work_dir, 'ansible.cfg') + # config = SAMPLE_CONFIG % 'openshift-enterprise' + # config = "%s\nansible_config: %s" % (config, ansible_config) + # self._ansible_config_test(load_facts_mock, run_ansible_mock, + # config, None, ansible_config) + + # #pylint: disable=too-many-arguments + # # This method allows for drastically simpler tests to write, and the args + # # are all useful. + # def _ansible_config_test(self, load_facts_mock, run_ansible_mock, + # installer_config, ansible_config_cli=None, expected_result=None): + # """ + # Utility method for testing the ways you can specify the ansible config. + # """ + + # load_facts_mock.return_value = (MOCK_FACTS, 0) + # run_ansible_mock.return_value = 0 + + # config_file = self.write_config(os.path.join(self.work_dir, + # 'ooinstall.conf'), installer_config) + + # self.cli_args.extend(["-c", config_file]) + # if ansible_config_cli: + # self.cli_args.extend(["--ansible-config", ansible_config_cli]) + # self.cli_args.append("install") + # result = self.runner.invoke(cli.cli, self.cli_args) + # self.assert_result(result, 0) + + # # Test the env vars for facts playbook: + # facts_env_vars = load_facts_mock.call_args[0][2] + # if expected_result: + # self.assertEquals(expected_result, facts_env_vars['ANSIBLE_CONFIG']) + # else: + # # If user running test has rpm installed, this might be set to default: + # self.assertTrue('ANSIBLE_CONFIG' not in facts_env_vars or + # facts_env_vars['ANSIBLE_CONFIG'] == cli.DEFAULT_ANSIBLE_CONFIG) + + # # Test the env vars for main playbook: + # env_vars = run_ansible_mock.call_args[0][2] + # if expected_result: + # self.assertEquals(expected_result, env_vars['ANSIBLE_CONFIG']) + # else: + # # If user running test has rpm installed, this might be set to default: + # # + # # By default we will use the quiet config + # self.assertTrue('ANSIBLE_CONFIG' not in env_vars or + # env_vars['ANSIBLE_CONFIG'] == cli.QUIET_ANSIBLE_CONFIG) # unattended with bad config file and no installed hosts (without --force) @patch('ooinstall.openshift_ansible.run_main_playbook') -- cgit v1.2.3 From b03d8cf25077cb824f1afca0ecd7d1b8edcc028d Mon Sep 17 00:00:00 2001 From: Tim Bielawa Date: Thu, 29 Sep 2016 08:07:39 -0700 Subject: Add messages to let the user know if some plays were skipped, but it's ok. Also, remove the final 'press a key to continue' prompt. --- callback_plugins/openshift_quick_installer.py | 40 +++++++++++++++++++++++++++ utils/src/ooinstall/cli_installer.py | 1 - 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/callback_plugins/openshift_quick_installer.py b/callback_plugins/openshift_quick_installer.py index 7923a9a80..e2f125df9 100644 --- a/callback_plugins/openshift_quick_installer.py +++ b/callback_plugins/openshift_quick_installer.py @@ -40,6 +40,7 @@ import imp import os import sys from ansible import constants as C +from ansible.utils.color import colorize, hostcolor ANSIBLE_PATH = imp.find_module('ansible')[1] DEFAULT_PATH = os.path.join(ANSIBLE_PATH, 'plugins/callback/default.py') DEFAULT_MODULE = imp.load_source( @@ -249,3 +250,42 @@ The only thing we change here is adding `log_only=True` to the 'notify' list attached. """ self._display.display("skipping: no hosts matched", color=C.COLOR_SKIP, log_only=True) + + def v2_playbook_on_stats(self, stats): + """Print the final playbook run stats""" + self._display.display("", screen_only=True) + self.banner("PLAY RECAP") + + hosts = sorted(stats.processed.keys()) + for h in hosts: + t = stats.summarize(h) + + self._display.display( + u"%s : %s %s %s %s" % ( + hostcolor(h, t), + colorize(u'ok', t['ok'], C.COLOR_OK), + colorize(u'changed', t['changed'], C.COLOR_CHANGED), + colorize(u'unreachable', t['unreachable'], C.COLOR_UNREACHABLE), + colorize(u'failed', t['failures'], C.COLOR_ERROR)), + screen_only=True + ) + + self._display.display( + u"%s : %s %s %s %s" % ( + hostcolor(h, t, False), + colorize(u'ok', t['ok'], None), + colorize(u'changed', t['changed'], None), + colorize(u'unreachable', t['unreachable'], None), + colorize(u'failed', t['failures'], None)), + log_only=True + ) + + self._display.display("", screen_only=True) + self._display.display("", screen_only=True) + + # Some plays are conditional and won't run (such as load + # balancers) if they aren't required. Let the user know about + # this to avoid potential confusion. + if self.plays_total_ran != self.plays_count: + print("Installation Complete: Note: Play count is an estimate and some were skipped because your install does not require them") + self._display.display("", screen_only=True) diff --git a/utils/src/ooinstall/cli_installer.py b/utils/src/ooinstall/cli_installer.py index 674d1cee4..43b1b3244 100644 --- a/utils/src/ooinstall/cli_installer.py +++ b/utils/src/ooinstall/cli_installer.py @@ -1078,7 +1078,6 @@ more: http://docs.openshift.com/enterprise/latest/admin_guide/overview.html """ click.echo(message) - click.pause() cli.add_command(install) cli.add_command(upgrade) -- cgit v1.2.3