diff options
Diffstat (limited to 'roles/openshift_health_checker/library')
| -rw-r--r--[-rwxr-xr-x] | roles/openshift_health_checker/library/aos_version.py | 198 | ||||
| -rw-r--r--[-rwxr-xr-x] | roles/openshift_health_checker/library/check_yum_update.py | 0 | ||||
| -rw-r--r-- | roles/openshift_health_checker/library/docker_info.py | 2 | ||||
| -rw-r--r-- | roles/openshift_health_checker/library/etcdkeysize.py | 122 | ||||
| -rw-r--r-- | roles/openshift_health_checker/library/ocutil.py | 73 | ||||
| -rw-r--r-- | roles/openshift_health_checker/library/rpm_version.py | 133 | ||||
| -rw-r--r-- | roles/openshift_health_checker/library/search_journalctl.py | 150 |
7 files changed, 606 insertions, 72 deletions
diff --git a/roles/openshift_health_checker/library/aos_version.py b/roles/openshift_health_checker/library/aos_version.py index 4460ec324..db3c0b654 100755..100644 --- a/roles/openshift_health_checker/library/aos_version.py +++ b/roles/openshift_health_checker/library/aos_version.py @@ -1,5 +1,5 @@ #!/usr/bin/python -''' +""" Ansible module for yum-based systems determining if multiple releases of an OpenShift package are available, and if the release requested (if any) is available down to the given precision. @@ -16,71 +16,87 @@ of release availability already. Without duplicating all that, we would like the user to have a helpful error message if we detect things will not work out right. Note that if openshift_release is not specified in the inventory, the version comparison checks just pass. - -TODO: fail gracefully on non-yum systems (dnf in Fedora) -''' +""" from ansible.module_utils.basic import AnsibleModule +# NOTE: because of the dependency on yum (Python 2-only), this module does not +# work under Python 3. But since we run unit tests against both Python 2 and +# Python 3, we use six for cross compatibility in this module alone: +from ansible.module_utils.six import string_types -IMPORT_EXCEPTION = None +YUM_IMPORT_EXCEPTION = None +DNF_IMPORT_EXCEPTION = None try: import yum # pylint: disable=import-error except ImportError as err: - IMPORT_EXCEPTION = err # in tox test env, yum import fails + YUM_IMPORT_EXCEPTION = err + +try: + import dnf # pylint: disable=import-error +except ImportError as err: + DNF_IMPORT_EXCEPTION = err class AosVersionException(Exception): - '''Base exception class for package version problems''' + """Base exception class for package version problems""" def __init__(self, message, problem_pkgs=None): Exception.__init__(self, message) self.problem_pkgs = problem_pkgs def main(): - '''Entrypoint for this Ansible module''' + """Entrypoint for this Ansible module""" module = AnsibleModule( argument_spec=dict( - requested_openshift_release=dict(type="str", default=''), - openshift_deployment_type=dict(required=True), - rpm_prefix=dict(required=True), # atomic-openshift, origin, ...? + package_list=dict(type="list", required=True), + package_mgr=dict(type="str", required=True), ), supports_check_mode=True ) - if IMPORT_EXCEPTION: - module.fail_json(msg="aos_version module could not import yum: %s" % IMPORT_EXCEPTION) + # determine the package manager to use + package_mgr = module.params['package_mgr'] + if package_mgr not in ('yum', 'dnf'): + module.fail_json(msg="package_mgr must be one of: yum, dnf") + pkg_mgr_exception = dict(yum=YUM_IMPORT_EXCEPTION, dnf=DNF_IMPORT_EXCEPTION)[package_mgr] + if pkg_mgr_exception: + module.fail_json( + msg="aos_version module could not import {}: {}".format(package_mgr, pkg_mgr_exception) + ) # determine the packages we will look for - rpm_prefix = module.params['rpm_prefix'] - if not rpm_prefix: - module.fail_json(msg="rpm_prefix must not be empty") - expected_pkgs = set([ - rpm_prefix, - rpm_prefix + '-master', - rpm_prefix + '-node', - ]) - - # determine what level of precision the user specified for the openshift version. - # should look like a version string with possibly many segments e.g. "3.4.1": - requested_openshift_release = module.params['requested_openshift_release'] + package_list = module.params['package_list'] + if not package_list: + module.fail_json(msg="package_list must not be empty") + + # generate set with only the names of expected packages + expected_pkg_names = [p["name"] for p in package_list] + + # gather packages that require a multi_minor_release check + multi_minor_pkgs = [p for p in package_list if p["check_multi"]] + + # generate list of packages with a specified (non-empty) version + # should look like a version string with possibly many segments e.g. "3.4.1" + versioned_pkgs = [p for p in package_list if p["version"]] # get the list of packages available and complain if anything is wrong try: - pkgs = _retrieve_available_packages(expected_pkgs) - if requested_openshift_release: - _check_precise_version_found(pkgs, expected_pkgs, requested_openshift_release) - _check_higher_version_found(pkgs, expected_pkgs, requested_openshift_release) - if module.params['openshift_deployment_type'] in ['openshift-enterprise']: - _check_multi_minor_release(pkgs, expected_pkgs) + pkgs = _retrieve_available_packages(package_mgr, expected_pkg_names) + if versioned_pkgs: + _check_precise_version_found(pkgs, _to_dict(versioned_pkgs)) + _check_higher_version_found(pkgs, _to_dict(versioned_pkgs)) + if multi_minor_pkgs: + _check_multi_minor_release(pkgs, _to_dict(multi_minor_pkgs)) except AosVersionException as excinfo: module.fail_json(msg=str(excinfo)) module.exit_json(changed=False) -def _retrieve_available_packages(expected_pkgs): - # search for package versions available for openshift pkgs - yb = yum.YumBase() # pylint: disable=invalid-name +def _to_dict(pkg_list): + return {pkg["name"]: pkg for pkg in pkg_list} + +def _retrieve_available_packages(pkg_mgr, expected_pkgs): # The openshift excluder prevents unintended updates to openshift # packages by setting yum excludes on those packages. See: # https://wiki.centos.org/SpecialInterestGroup/PaaS/OpenShift-Origin-Control-Updates @@ -89,71 +105,111 @@ def _retrieve_available_packages(expected_pkgs): # attempt to determine what packages are available via yum they may # be excluded. So, for our purposes here, disable excludes to see # what will really be available during an install or upgrade. - yb.conf.disable_excludes = ['all'] - try: - pkgs = yb.pkgSack.returnPackages(patterns=expected_pkgs) - except yum.Errors.PackageSackError as excinfo: - # you only hit this if *none* of the packages are available - raise AosVersionException('\n'.join([ - 'Unable to find any OpenShift packages.', - 'Check your subscription and repo settings.', - str(excinfo), - ])) + if pkg_mgr == "yum": + # search for package versions available for openshift pkgs + yb = yum.YumBase() # pylint: disable=invalid-name + + yb.conf.disable_excludes = ['all'] + + try: + pkgs = yb.rpmdb.returnPackages(patterns=expected_pkgs) + pkgs += yb.pkgSack.returnPackages(patterns=expected_pkgs) + except yum.Errors.PackageSackError as excinfo: + # you only hit this if *none* of the packages are available + raise AosVersionException('\n'.join([ + 'Unable to find any OpenShift packages.', + 'Check your subscription and repo settings.', + str(excinfo), + ])) + elif pkg_mgr == "dnf": + dbase = dnf.Base() # pyling: disable=invalid-name + + dbase.conf.disable_excludes = ['all'] + dbase.read_all_repos() + dbase.fill_sack(load_system_repo=False, load_available_repos=True) + + dquery = dbase.sack.query() + aquery = dquery.available() + iquery = dquery.installed() + + available_pkgs = list(aquery.filter(name=expected_pkgs)) + installed_pkgs = list(iquery.filter(name=expected_pkgs)) + pkgs = available_pkgs + installed_pkgs + + if not pkgs: + # pkgs list is empty, raise because no expected packages found + raise AosVersionException('\n'.join([ + 'Unable to find any OpenShift packages.', + 'Check your subscription and repo settings.', + ])) + return pkgs class PreciseVersionNotFound(AosVersionException): - '''Exception for reporting packages not available at given release''' - def __init__(self, requested_release, not_found): - msg = ['Not all of the required packages are available at requested version %s:' % requested_release] - msg += [' ' + name for name in not_found] + """Exception for reporting packages not available at given version""" + def __init__(self, not_found): + msg = ['Not all of the required packages are available at their requested version'] + msg += ['{}:{} '.format(pkg["name"], pkg["version"]) for pkg in not_found] msg += ['Please check your subscriptions and enabled repositories.'] AosVersionException.__init__(self, '\n'.join(msg), not_found) -def _check_precise_version_found(pkgs, expected_pkgs, requested_openshift_release): +def _check_precise_version_found(pkgs, expected_pkgs_dict): # see if any packages couldn't be found at requested release version # we would like to verify that the latest available pkgs have however specific a version is given. # so e.g. if there is a package version 3.4.1.5 the check passes; if only 3.4.0, it fails. - pkgs_precise_version_found = {} + pkgs_precise_version_found = set() for pkg in pkgs: - if pkg.name not in expected_pkgs: + if pkg.name not in expected_pkgs_dict: continue - # does the version match, to the precision requested? - # and, is it strictly greater, at the precision requested? - match_version = '.'.join(pkg.version.split('.')[:requested_openshift_release.count('.') + 1]) - if match_version == requested_openshift_release: - pkgs_precise_version_found[pkg.name] = True + expected_pkg_versions = expected_pkgs_dict[pkg.name]["version"] + if isinstance(expected_pkg_versions, string_types): + expected_pkg_versions = [expected_pkg_versions] + for expected_pkg_version in expected_pkg_versions: + # does the version match, to the precision requested? + # and, is it strictly greater, at the precision requested? + match_version = '.'.join(pkg.version.split('.')[:expected_pkg_version.count('.') + 1]) + if match_version == expected_pkg_version: + pkgs_precise_version_found.add(pkg.name) not_found = [] - for name in expected_pkgs: + for name, pkg in expected_pkgs_dict.items(): if name not in pkgs_precise_version_found: - not_found.append(name) + not_found.append(pkg) if not_found: - raise PreciseVersionNotFound(requested_openshift_release, not_found) + raise PreciseVersionNotFound(not_found) class FoundHigherVersion(AosVersionException): - '''Exception for reporting that a higher version than requested is available''' - def __init__(self, requested_release, higher_found): + """Exception for reporting that a higher version than requested is available""" + def __init__(self, higher_found): msg = ['Some required package(s) are available at a version', - 'that is higher than requested %s:' % requested_release] + 'that is higher than requested'] msg += [' ' + name for name in higher_found] msg += ['This will prevent installing the version you requested.'] msg += ['Please check your enabled repositories or adjust openshift_release.'] AosVersionException.__init__(self, '\n'.join(msg), higher_found) -def _check_higher_version_found(pkgs, expected_pkgs, requested_openshift_release): - req_release_arr = [int(segment) for segment in requested_openshift_release.split(".")] +def _check_higher_version_found(pkgs, expected_pkgs_dict): + expected_pkg_names = list(expected_pkgs_dict) + # see if any packages are available in a version higher than requested higher_version_for_pkg = {} for pkg in pkgs: - if pkg.name not in expected_pkgs: + if pkg.name not in expected_pkg_names: continue + expected_pkg_versions = expected_pkgs_dict[pkg.name]["version"] + if isinstance(expected_pkg_versions, string_types): + expected_pkg_versions = [expected_pkg_versions] + # NOTE: the list of versions is assumed to be sorted so that the highest + # desirable version is the last. + highest_desirable_version = expected_pkg_versions[-1] + req_release_arr = [int(segment) for segment in highest_desirable_version.split(".")] version = [int(segment) for segment in pkg.version.split(".")] too_high = version[:len(req_release_arr)] > req_release_arr higher_than_seen = version > higher_version_for_pkg.get(pkg.name, []) @@ -164,11 +220,11 @@ def _check_higher_version_found(pkgs, expected_pkgs, requested_openshift_release higher_found = [] for name, version in higher_version_for_pkg.items(): higher_found.append(name + '-' + '.'.join(str(segment) for segment in version)) - raise FoundHigherVersion(requested_openshift_release, higher_found) + raise FoundHigherVersion(higher_found) class FoundMultiRelease(AosVersionException): - '''Exception for reporting multiple minor releases found for same package''' + """Exception for reporting multiple minor releases found for same package""" def __init__(self, multi_found): msg = ['Multiple minor versions of these packages are available'] msg += [' ' + name for name in multi_found] @@ -176,18 +232,18 @@ class FoundMultiRelease(AosVersionException): AosVersionException.__init__(self, '\n'.join(msg), multi_found) -def _check_multi_minor_release(pkgs, expected_pkgs): +def _check_multi_minor_release(pkgs, expected_pkgs_dict): # see if any packages are available in more than one minor version pkgs_by_name_version = {} for pkg in pkgs: # keep track of x.y (minor release) versions seen minor_release = '.'.join(pkg.version.split('.')[:2]) if pkg.name not in pkgs_by_name_version: - pkgs_by_name_version[pkg.name] = {} - pkgs_by_name_version[pkg.name][minor_release] = True + pkgs_by_name_version[pkg.name] = set() + pkgs_by_name_version[pkg.name].add(minor_release) multi_found = [] - for name in expected_pkgs: + for name in expected_pkgs_dict: if name in pkgs_by_name_version and len(pkgs_by_name_version[name]) > 1: multi_found.append(name) diff --git a/roles/openshift_health_checker/library/check_yum_update.py b/roles/openshift_health_checker/library/check_yum_update.py index 433795b67..433795b67 100755..100644 --- a/roles/openshift_health_checker/library/check_yum_update.py +++ b/roles/openshift_health_checker/library/check_yum_update.py diff --git a/roles/openshift_health_checker/library/docker_info.py b/roles/openshift_health_checker/library/docker_info.py index 7f712bcff..0d0ddae8b 100644 --- a/roles/openshift_health_checker/library/docker_info.py +++ b/roles/openshift_health_checker/library/docker_info.py @@ -1,4 +1,3 @@ -# pylint: disable=missing-docstring """ Ansible module for determining information about the docker host. @@ -13,6 +12,7 @@ from ansible.module_utils.docker_common import AnsibleDockerClient def main(): + """Entrypoint for running an Ansible module.""" client = AnsibleDockerClient() client.module.exit_json( diff --git a/roles/openshift_health_checker/library/etcdkeysize.py b/roles/openshift_health_checker/library/etcdkeysize.py new file mode 100644 index 000000000..620e82d87 --- /dev/null +++ b/roles/openshift_health_checker/library/etcdkeysize.py @@ -0,0 +1,122 @@ +#!/usr/bin/python +"""Ansible module that recursively determines if the size of a key in an etcd cluster exceeds a given limit.""" + +from ansible.module_utils.basic import AnsibleModule + + +try: + import etcd + + IMPORT_EXCEPTION_MSG = None +except ImportError as err: + IMPORT_EXCEPTION_MSG = str(err) + + from collections import namedtuple + EtcdMock = namedtuple("etcd", ["EtcdKeyNotFound"]) + etcd = EtcdMock(KeyError) + + +# pylint: disable=too-many-arguments +def check_etcd_key_size(client, key, size_limit, total_size=0, depth=0, depth_limit=1000, visited=None): + """Check size of an etcd path starting at given key. Returns tuple (string, bool)""" + if visited is None: + visited = set() + + if key in visited: + return 0, False + + visited.add(key) + + try: + result = client.read(key, recursive=False) + except etcd.EtcdKeyNotFound: + return 0, False + + size = 0 + limit_exceeded = False + + for node in result.leaves: + if depth >= depth_limit: + raise Exception("Maximum recursive stack depth ({}) exceeded.".format(depth_limit)) + + if size_limit and total_size + size > size_limit: + return size, True + + if not node.dir: + size += len(node.value) + continue + + key_size, limit_exceeded = check_etcd_key_size(client, node.key, + size_limit, + total_size + size, + depth + 1, + depth_limit, visited) + size += key_size + + max_limit_exceeded = limit_exceeded or (total_size + size > size_limit) + return size, max_limit_exceeded + + +def main(): # pylint: disable=missing-docstring,too-many-branches + module = AnsibleModule( + argument_spec=dict( + size_limit_bytes=dict(type="int", default=0), + paths=dict(type="list", default=["/openshift.io/images"]), + host=dict(type="str", default="127.0.0.1"), + port=dict(type="int", default=4001), + protocol=dict(type="str", default="http"), + version_prefix=dict(type="str", default=""), + allow_redirect=dict(type="bool", default=False), + cert=dict(type="dict", default=""), + ca_cert=dict(type="str", default=None), + ), + supports_check_mode=True + ) + + module.params["cert"] = ( + module.params["cert"]["cert"], + module.params["cert"]["key"], + ) + + size_limit = module.params.pop("size_limit_bytes") + paths = module.params.pop("paths") + + limit_exceeded = False + + try: + # pylint: disable=no-member + client = etcd.Client(**module.params) + except AttributeError as attrerr: + msg = str(attrerr) + if IMPORT_EXCEPTION_MSG: + msg = IMPORT_EXCEPTION_MSG + if "No module named etcd" in IMPORT_EXCEPTION_MSG: + # pylint: disable=redefined-variable-type + msg = ('Unable to import the python "etcd" dependency. ' + 'Make sure python-etcd is installed on the host.') + + module.exit_json( + failed=True, + changed=False, + size_limit_exceeded=limit_exceeded, + msg=msg, + ) + + return + + size = 0 + for path in paths: + path_size, limit_exceeded = check_etcd_key_size(client, path, size_limit - size) + size += path_size + + if limit_exceeded: + break + + module.exit_json( + changed=False, + size_limit_exceeded=limit_exceeded, + ) + + +if __name__ == '__main__': + main() diff --git a/roles/openshift_health_checker/library/ocutil.py b/roles/openshift_health_checker/library/ocutil.py new file mode 100644 index 000000000..c72f4c5b3 --- /dev/null +++ b/roles/openshift_health_checker/library/ocutil.py @@ -0,0 +1,73 @@ +#!/usr/bin/python +"""Interface to OpenShift oc command""" + +import os +import shlex +import shutil +import subprocess + +from ansible.module_utils.basic import AnsibleModule + + +ADDITIONAL_PATH_LOOKUPS = ['/usr/local/bin', os.path.expanduser('~/bin')] + + +def locate_oc_binary(): + """Find and return oc binary file""" + # https://github.com/openshift/openshift-ansible/issues/3410 + # oc can be in /usr/local/bin in some cases, but that may not + # be in $PATH due to ansible/sudo + paths = os.environ.get("PATH", os.defpath).split(os.pathsep) + ADDITIONAL_PATH_LOOKUPS + + oc_binary = 'oc' + + # Use shutil.which if it is available, otherwise fallback to a naive path search + try: + which_result = shutil.which(oc_binary, path=os.pathsep.join(paths)) + if which_result is not None: + oc_binary = which_result + except AttributeError: + for path in paths: + if os.path.exists(os.path.join(path, oc_binary)): + oc_binary = os.path.join(path, oc_binary) + break + + return oc_binary + + +def main(): + """Module that executes commands on a remote OpenShift cluster""" + + module = AnsibleModule( + argument_spec=dict( + namespace=dict(type="str", required=False), + config_file=dict(type="str", required=True), + cmd=dict(type="str", required=True), + extra_args=dict(type="list", default=[]), + ), + ) + + cmd = [locate_oc_binary(), '--config', module.params["config_file"]] + if module.params["namespace"]: + cmd += ['-n', module.params["namespace"]] + cmd += shlex.split(module.params["cmd"]) + module.params["extra_args"] + + failed = True + try: + cmd_result = subprocess.check_output(list(cmd), stderr=subprocess.STDOUT) + failed = False + except subprocess.CalledProcessError as exc: + cmd_result = '[rc {}] {}\n{}'.format(exc.returncode, ' '.join(exc.cmd), exc.output) + except OSError as exc: + # we get this when 'oc' is not there + cmd_result = str(exc) + + module.exit_json( + changed=False, + failed=failed, + result=cmd_result, + ) + + +if __name__ == '__main__': + main() diff --git a/roles/openshift_health_checker/library/rpm_version.py b/roles/openshift_health_checker/library/rpm_version.py new file mode 100644 index 000000000..c24fbba3b --- /dev/null +++ b/roles/openshift_health_checker/library/rpm_version.py @@ -0,0 +1,133 @@ +#!/usr/bin/python +""" +Ansible module for rpm-based systems determining existing package version information in a host. +""" + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.six import string_types + +IMPORT_EXCEPTION = None +try: + import rpm # pylint: disable=import-error +except ImportError as err: + IMPORT_EXCEPTION = err # in tox test env, rpm import fails + + +class RpmVersionException(Exception): + """Base exception class for package version problems""" + def __init__(self, message, problem_pkgs=None): + Exception.__init__(self, message) + self.problem_pkgs = problem_pkgs + + +def main(): + """Entrypoint for this Ansible module""" + module = AnsibleModule( + argument_spec=dict( + package_list=dict(type="list", required=True), + ), + supports_check_mode=True + ) + + if IMPORT_EXCEPTION: + module.fail_json(msg="rpm_version module could not import rpm: %s" % IMPORT_EXCEPTION) + + # determine the packages we will look for + pkg_list = module.params['package_list'] + if not pkg_list: + module.fail_json(msg="package_list must not be empty") + + # get list of packages available and complain if any + # of them are missing or if any errors occur + try: + pkg_versions = _retrieve_expected_pkg_versions(_to_dict(pkg_list)) + _check_pkg_versions(pkg_versions, _to_dict(pkg_list)) + except RpmVersionException as excinfo: + module.fail_json(msg=str(excinfo)) + module.exit_json(changed=False) + + +def _to_dict(pkg_list): + return {pkg["name"]: pkg for pkg in pkg_list} + + +def _retrieve_expected_pkg_versions(expected_pkgs_dict): + """Search for installed packages matching given pkg names + and versions. Returns a dictionary: {pkg_name: [versions]}""" + + transaction = rpm.TransactionSet() + pkgs = {} + + for pkg_name in expected_pkgs_dict: + matched_pkgs = transaction.dbMatch("name", pkg_name) + if not matched_pkgs: + continue + + for header in matched_pkgs: + if header['name'] == pkg_name: + if pkg_name not in pkgs: + pkgs[pkg_name] = [] + + pkgs[pkg_name].append(header['version']) + + return pkgs + + +def _check_pkg_versions(found_pkgs_dict, expected_pkgs_dict): + invalid_pkg_versions = {} + not_found_pkgs = [] + + for pkg_name, pkg in expected_pkgs_dict.items(): + if not found_pkgs_dict.get(pkg_name): + not_found_pkgs.append(pkg_name) + continue + + found_versions = [_parse_version(version) for version in found_pkgs_dict[pkg_name]] + + if isinstance(pkg["version"], string_types): + expected_versions = [_parse_version(pkg["version"])] + else: + expected_versions = [_parse_version(version) for version in pkg["version"]] + + if not set(expected_versions) & set(found_versions): + invalid_pkg_versions[pkg_name] = { + "found_versions": found_versions, + "required_versions": expected_versions, + } + + if not_found_pkgs: + raise RpmVersionException( + '\n'.join([ + "The following packages were not found to be installed: {}".format('\n '.join([ + "{}".format(pkg) + for pkg in not_found_pkgs + ])) + ]), + not_found_pkgs, + ) + + if invalid_pkg_versions: + raise RpmVersionException( + '\n '.join([ + "The following packages were found to be installed with an incorrect version: {}".format('\n'.join([ + " \n{}\n Required version: {}\n Found versions: {}".format( + pkg_name, + ', '.join(pkg["required_versions"]), + ', '.join([version for version in pkg["found_versions"]])) + for pkg_name, pkg in invalid_pkg_versions.items() + ])) + ]), + invalid_pkg_versions, + ) + + +def _parse_version(version_str): + segs = version_str.split('.') + if not segs or len(segs) <= 2: + return version_str + + return '.'.join(segs[0:2]) + + +if __name__ == '__main__': + main() diff --git a/roles/openshift_health_checker/library/search_journalctl.py b/roles/openshift_health_checker/library/search_journalctl.py new file mode 100644 index 000000000..3631f71c8 --- /dev/null +++ b/roles/openshift_health_checker/library/search_journalctl.py @@ -0,0 +1,150 @@ +#!/usr/bin/python +"""Interface to journalctl.""" + +from time import time +import json +import re +import subprocess + +from ansible.module_utils.basic import AnsibleModule + + +class InvalidMatcherRegexp(Exception): + """Exception class for invalid matcher regexp.""" + pass + + +class InvalidLogEntry(Exception): + """Exception class for invalid / non-json log entries.""" + pass + + +class LogInputSubprocessError(Exception): + """Exception class for errors that occur while executing a subprocess.""" + pass + + +def main(): + """Scan a given list of "log_matchers" for journalctl messages containing given patterns. + "log_matchers" is a list of dicts consisting of three keys that help fine-tune log searching: + 'start_regexp', 'regexp', and 'unit'. + + Sample "log_matchers" list: + + [ + { + 'start_regexp': r'Beginning of systemd unit', + 'regexp': r'the specific log message to find', + 'unit': 'etcd', + } + ] + """ + module = AnsibleModule( + argument_spec=dict( + log_count_limit=dict(type="int", default=500), + log_matchers=dict(type="list", required=True), + ), + ) + + timestamp_limit_seconds = time() - 60 * 60 # 1 hour + + log_count_limit = module.params["log_count_limit"] + log_matchers = module.params["log_matchers"] + + matched_regexp, errors = get_log_matches(log_matchers, log_count_limit, timestamp_limit_seconds) + + module.exit_json( + changed=False, + failed=bool(errors), + errors=errors, + matched=matched_regexp, + ) + + +def get_log_matches(matchers, log_count_limit, timestamp_limit_seconds): + """Return a list of up to log_count_limit matches for each matcher. + + Log entries are only considered if newer than timestamp_limit_seconds. + """ + matched_regexp = [] + errors = [] + + for matcher in matchers: + try: + log_output = get_log_output(matcher) + except LogInputSubprocessError as err: + errors.append(str(err)) + continue + + try: + matched = find_matches(log_output, matcher, log_count_limit, timestamp_limit_seconds) + if matched: + matched_regexp.append(matcher.get("regexp", "")) + except InvalidMatcherRegexp as err: + errors.append(str(err)) + except InvalidLogEntry as err: + errors.append(str(err)) + + return matched_regexp, errors + + +def get_log_output(matcher): + """Return an iterator on the logs of a given matcher.""" + try: + cmd_output = subprocess.Popen(list([ + '/bin/journalctl', + '-ru', matcher.get("unit", ""), + '--output', 'json', + ]), stdout=subprocess.PIPE) + + return iter(cmd_output.stdout.readline, '') + + except subprocess.CalledProcessError as exc: + msg = "Could not obtain journalctl logs for the specified systemd unit: {}: {}" + raise LogInputSubprocessError(msg.format(matcher.get("unit", "<missing>"), str(exc))) + except OSError as exc: + raise LogInputSubprocessError(str(exc)) + + +def find_matches(log_output, matcher, log_count_limit, timestamp_limit_seconds): + """Return log messages matched in iterable log_output by a given matcher. + + Ignore any log_output items older than timestamp_limit_seconds. + """ + try: + regexp = re.compile(matcher.get("regexp", "")) + start_regexp = re.compile(matcher.get("start_regexp", "")) + except re.error as err: + msg = "A log matcher object was provided with an invalid regular expression: {}" + raise InvalidMatcherRegexp(msg.format(str(err))) + + matched = None + + for log_count, line in enumerate(log_output): + if log_count >= log_count_limit: + break + + try: + obj = json.loads(line) + + # don't need to look past the most recent service restart + if start_regexp.match(obj["MESSAGE"]): + break + + log_timestamp_seconds = float(obj["__REALTIME_TIMESTAMP"]) / 1000000 + if log_timestamp_seconds < timestamp_limit_seconds: + break + + if regexp.match(obj["MESSAGE"]): + matched = line + break + + except ValueError: + msg = "Log entry for systemd unit {} contained invalid json syntax: {}" + raise InvalidLogEntry(msg.format(matcher.get("unit"), line)) + + return matched + + +if __name__ == '__main__': + main() |