4 files changed, 174 insertions, 12 deletions
diff --git a/roles/openshift_health_checker/library/aos_version.py b/roles/openshift_health_checker/library/aos_version.py
index 4c205e48c..f9babebb9 100755..100644
--- a/roles/openshift_health_checker/library/aos_version.py
+++ b/roles/openshift_health_checker/library/aos_version.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-'''
+"""
 Ansible module for yum-based systems determining if multiple releases
 of an OpenShift package are available, and if the release requested
 (if any) is available down to the given precision.
@@ -16,9 +16,13 @@ of release availability already. Without duplicating all that, we would
 like the user to have a helpful error message if we detect things will
 not work out right. Note that if openshift_release is not specified in
 the inventory, the version comparison checks just pass.
-'''
+"""
 
 from ansible.module_utils.basic import AnsibleModule
+# NOTE: because of the dependency on yum (Python 2-only), this module does not
+# work under Python 3. But since we run unit tests against both Python 2 and
+# Python 3, we use six for cross compatibility in this module alone:
+from ansible.module_utils.six import string_types
 
 IMPORT_EXCEPTION = None
 try:
@@ -28,7 +32,7 @@ except ImportError as err:
 
 
 class AosVersionException(Exception):
-    '''Base exception class for package version problems'''
+    """Base exception class for package version problems"""
     def __init__(self, message, problem_pkgs=None):
         Exception.__init__(self, message)
         self.problem_pkgs = problem_pkgs
@@ -122,12 +126,15 @@ def _check_precise_version_found(pkgs, expected_pkgs_dict):
     for pkg in pkgs:
         if pkg.name not in expected_pkgs_dict:
             continue
-        # does the version match, to the precision requested?
-        # and, is it strictly greater, at the precision requested?
-        expected_pkg_version = expected_pkgs_dict[pkg.name]["version"]
-        match_version = '.'.join(pkg.version.split('.')[:expected_pkg_version.count('.') + 1])
-        if match_version == expected_pkg_version:
-            pkgs_precise_version_found.add(pkg.name)
+        expected_pkg_versions = expected_pkgs_dict[pkg.name]["version"]
+        if isinstance(expected_pkg_versions, string_types):
+            expected_pkg_versions = [expected_pkg_versions]
+        for expected_pkg_version in expected_pkg_versions:
+            # does the version match, to the precision requested?
+            # and, is it strictly greater, at the precision requested?
+            match_version = '.'.join(pkg.version.split('.')[:expected_pkg_version.count('.') + 1])
+            if match_version == expected_pkg_version:
+                pkgs_precise_version_found.add(pkg.name)
 
     not_found = []
     for name, pkg in expected_pkgs_dict.items():
@@ -157,8 +164,13 @@ def _check_higher_version_found(pkgs, expected_pkgs_dict):
     for pkg in pkgs:
         if pkg.name not in expected_pkg_names:
             continue
-        expected_pkg_version = expected_pkgs_dict[pkg.name]["version"]
-        req_release_arr = [int(segment) for segment in expected_pkg_version.split(".")]
+        expected_pkg_versions = expected_pkgs_dict[pkg.name]["version"]
+        if isinstance(expected_pkg_versions, string_types):
+            expected_pkg_versions = [expected_pkg_versions]
+        # NOTE: the list of versions is assumed to be sorted so that the highest
+        # desirable version is the last.
+        highest_desirable_version = expected_pkg_versions[-1]
+        req_release_arr = [int(segment) for segment in highest_desirable_version.split(".")]
         version = [int(segment) for segment in pkg.version.split(".")]
         too_high = version[:len(req_release_arr)] > req_release_arr
         higher_than_seen = version > higher_version_for_pkg.get(pkg.name, [])
diff --git a/roles/openshift_health_checker/library/check_yum_update.py b/roles/openshift_health_checker/library/check_yum_update.py
index 433795b67..433795b67 100755..100644
--- a/roles/openshift_health_checker/library/check_yum_update.py
+++ b/roles/openshift_health_checker/library/check_yum_update.py
diff --git a/roles/openshift_health_checker/library/docker_info.py b/roles/openshift_health_checker/library/docker_info.py
index 7f712bcff..0d0ddae8b 100644
--- a/roles/openshift_health_checker/library/docker_info.py
+++ b/roles/openshift_health_checker/library/docker_info.py
@@ -1,4 +1,3 @@
-# pylint: disable=missing-docstring
 """
 Ansible module for determining information about the docker host.
 
@@ -13,6 +12,7 @@ from ansible.module_utils.docker_common import AnsibleDockerClient
 
 
 def main():
+    """Entrypoint for running an Ansible module."""
     client = AnsibleDockerClient()
 
     client.module.exit_json(
diff --git a/roles/openshift_health_checker/library/search_journalctl.py b/roles/openshift_health_checker/library/search_journalctl.py
new file mode 100644
index 000000000..3631f71c8
--- /dev/null
+++ b/roles/openshift_health_checker/library/search_journalctl.py
@@ -0,0 +1,150 @@
+#!/usr/bin/python
+"""Interface to journalctl."""
+
+from time import time
+import json
+import re
+import subprocess
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+class InvalidMatcherRegexp(Exception):
+    """Exception class for invalid matcher regexp."""
+    pass
+
+
+class InvalidLogEntry(Exception):
+    """Exception class for invalid / non-json log entries."""
+    pass
+
+
+class LogInputSubprocessError(Exception):
+    """Exception class for errors that occur while executing a subprocess."""
+    pass
+
+
+def main():
+    """Scan a given list of "log_matchers" for journalctl messages containing given patterns.
+    "log_matchers" is a list of dicts consisting of three keys that help fine-tune log searching:
+    'start_regexp', 'regexp', and 'unit'.
+
+    Sample "log_matchers" list:
+
+    [
+      {
+        'start_regexp': r'Beginning of systemd unit',
+        'regexp': r'the specific log message to find',
+        'unit': 'etcd',
+      }
+    ]
+    """
+    module = AnsibleModule(
+        argument_spec=dict(
+            log_count_limit=dict(type="int", default=500),
+            log_matchers=dict(type="list", required=True),
+        ),
+    )
+
+    timestamp_limit_seconds = time() - 60 * 60  # 1 hour
+
+    log_count_limit = module.params["log_count_limit"]
+    log_matchers = module.params["log_matchers"]
+
+    matched_regexp, errors = get_log_matches(log_matchers, log_count_limit, timestamp_limit_seconds)
+
+    module.exit_json(
+        changed=False,
+        failed=bool(errors),
+        errors=errors,
+        matched=matched_regexp,
+    )
+
+
+def get_log_matches(matchers, log_count_limit, timestamp_limit_seconds):
+    """Return a list of up to log_count_limit matches for each matcher.
+
+    Log entries are only considered if newer than timestamp_limit_seconds.
+    """
+    matched_regexp = []
+    errors = []
+
+    for matcher in matchers:
+        try:
+            log_output = get_log_output(matcher)
+        except LogInputSubprocessError as err:
+            errors.append(str(err))
+            continue
+
+        try:
+            matched = find_matches(log_output, matcher, log_count_limit, timestamp_limit_seconds)
+            if matched:
+                matched_regexp.append(matcher.get("regexp", ""))
+        except InvalidMatcherRegexp as err:
+            errors.append(str(err))
+        except InvalidLogEntry as err:
+            errors.append(str(err))
+
+    return matched_regexp, errors
+
+
+def get_log_output(matcher):
+    """Return an iterator on the logs of a given matcher."""
+    try:
+        cmd_output = subprocess.Popen(list([
+            '/bin/journalctl',
+            '-ru', matcher.get("unit", ""),
+            '--output', 'json',
+        ]), stdout=subprocess.PIPE)
+
+        return iter(cmd_output.stdout.readline, '')
+
+    except subprocess.CalledProcessError as exc:
+        msg = "Could not obtain journalctl logs for the specified systemd unit: {}: {}"
+        raise LogInputSubprocessError(msg.format(matcher.get("unit", "<missing>"), str(exc)))
+    except OSError as exc:
+        raise LogInputSubprocessError(str(exc))
+
+
+def find_matches(log_output, matcher, log_count_limit, timestamp_limit_seconds):
+    """Return log messages matched in iterable log_output by a given matcher.
+
+    Ignore any log_output items older than timestamp_limit_seconds.
+    """
+    try:
+        regexp = re.compile(matcher.get("regexp", ""))
+        start_regexp = re.compile(matcher.get("start_regexp", ""))
+    except re.error as err:
+        msg = "A log matcher object was provided with an invalid regular expression: {}"
+        raise InvalidMatcherRegexp(msg.format(str(err)))
+
+    matched = None
+
+    for log_count, line in enumerate(log_output):
+        if log_count >= log_count_limit:
+            break
+
+        try:
+            obj = json.loads(line)
+
+            # don't need to look past the most recent service restart
+            if start_regexp.match(obj["MESSAGE"]):
+                break
+
+            log_timestamp_seconds = float(obj["__REALTIME_TIMESTAMP"]) / 1000000
+            if log_timestamp_seconds < timestamp_limit_seconds:
+                break
+
+            if regexp.match(obj["MESSAGE"]):
+                matched = line
+                break
+
+        except ValueError:
+            msg = "Log entry for systemd unit {} contained invalid json syntax: {}"
+            raise InvalidLogEntry(msg.format(matcher.get("unit"), line))
+
+    return matched
+
+
+if __name__ == '__main__':
+    main()