Add --skip-known-failures flag for smart rerun filtering

docs/index.rst

@@ -9,6 +9,7 @@ Content Index
    fragment_merging.rst
    siteconfig.rst
    detailed_test_config.rst
+   known_failure_patterns.rst
    openstack_backend.rst
    libcloud_backend.rst
    downburst_vms.rst

docs/known_failure_patterns.rst

@@ -0,0 +1,54 @@
+Known Failure Patterns
+======================
+
+When using the ``--skip-known-failures`` option with ``teuthology-suite --rerun``,
+teuthology can filter out jobs with known failure patterns. The known patterns are
+loaded from a JSON or YAML file specified with ``--known-failure-patterns``, or from
+the default bundled file ``teuthology/suite/patterns/known-failures.json``.
+
+File Format
+-----------
+
+The file format supports both JSON and YAML. The file must contain a ``patterns``
+key with a list of regex patterns (strings) that will be matched against job
+failure reasons.
+
+JSON format example::
+
+    {
+        "patterns": [
+            "Command failed on .* with status 1:",
+            "clocks not synchronized",
+            "cluster \\[WRN\\] Health check failed:.*OBJECT_UNFOUND"
+        ]
+    }
+
+YAML format example::
+
+    patterns:
+      - "Command failed on .* with status 1:"
+      - "clocks not synchronized"
+      - "cluster \\[WRN\\] Health check failed:.*OBJECT_UNFOUND"
+
+Pattern Matching
+----------------
+
+Patterns are matched using Python's ``re.search()`` function, so they support
+full regular expression syntax. If a job's ``failure_reason`` matches any of
+the patterns, it is considered a known failure and will be skipped during
+rerun when ``--skip-known-failures`` is enabled.
+
+Only jobs with failure reasons that don't match any known pattern will be
+scheduled for rerun.
+
+Usage
+-----
+
+To use known failure patterns during rerun::
+
+    teuthology-suite --rerun <run_name> --skip-known-failures
+
+To specify a custom patterns file::
+
+    teuthology-suite --rerun <run_name> --skip-known-failures --known-failure-patterns /path/to/patterns.json
+

scripts/suite.py

@@ -2,7 +2,7 @@
 import sys
 
 import teuthology.suite
-from teuthology.suite import override_arg_defaults as defaults
+from teuthology.suite import override_arg_defaults as defaults, _get_default_known_failure_patterns_file
 from teuthology.config import config
 
 doc = """
@@ -173,6 +173,14 @@
                               but can be overide if passed again.
                               This is important for tests involving random facet
                               (path ends with '$' operator).
+ --skip-known-failures <bool>
+                              Skip jobs with known failure patterns during rerun.
+                              To be used with --rerun only. Only rerun jobs with
+                              unknown failures by checking against known patterns
+                              file [default: false].
+ --known-failure-patterns <path> Path to known failure patterns file (JSON or YAML).
+                              To be used with --skip-known-failures.
+                              [default: {default_known_failure_patterns}]
  -R, --rerun-statuses <statuses>
                               A comma-separated list of statuses to be used
                               with --rerun. Supported statuses are: 'dead',
@@ -224,6 +232,7 @@
                             config.get_ceph_qa_suite_git_url()),
     default_ceph_branch=defaults('--ceph-branch', 'main'),
     default_job_threshold=config.job_threshold,
+    default_known_failure_patterns=_get_default_known_failure_patterns_file(),
 )
 
 

setup.cfg

@@ -130,6 +130,8 @@ teuthology.openstack =
     setup-openstack.sh
 teuthology.suite =
     fragment-merge.lua
+teuthology.suite.patterns =
+    known-failures.json
 teuthology.task.install =
     bin/adjust-ulimits
     bin/daemon-helper

teuthology/suite/__init__.py

@@ -4,11 +4,15 @@
 
 import logging
 import os
+import json
+import re
 import random
 import sys
 import time
 from distutils.util import strtobool
 
+import yaml
+
 import teuthology
 from teuthology.config import config, YamlConfig
 from teuthology.report import ResultsReporter
@@ -71,6 +75,8 @@ def process_args(args):
                 value = []
             else:
                 value = [x.strip() for x in value.split(',')]
+        elif key == 'skip_known_failures':
+            value = strtobool(value)
         elif key == 'ceph_repo':
             value = expand_short_repo_name(
                 value,
@@ -130,7 +136,9 @@ def main(args):
         log.info('Will upload archives to ' + conf.archive_upload)
 
     if conf.rerun:
-        get_rerun_conf_overrides(conf)
+        if get_rerun_conf_overrides(conf) is False:
+            log.info("Rerun cancelled due to no unknown failures")
+            return
     if conf.seed < 0:
         conf.seed = random.randint(0, 9999)
         log.info('Using random seed=%s', conf.seed)
@@ -154,7 +162,7 @@ def get_rerun_conf_overrides(conf):
     except IndexError:
         job0 = None
 
-    seed = None if job0 is None else job0.get('seed')
+    seed = None if job0 is None else int(job0.get('seed'))
     if conf.seed >= 0 and conf.seed != seed:
         log.error('--seed %s does not match with rerun seed: %s',
                   conf.seed, seed)
@@ -200,7 +208,103 @@ def get_rerun_conf_overrides(conf):
         )
         return
 
-    conf.filter_in.extend(rerun_filters['descriptions'])
+    if conf.skip_known_failures:
+        # Filter jobs by rerun statuses first
+        rerun_statuses_jobs = (job for job in run['jobs'] if job['status'] in conf.rerun_statuses)
+        # Filter out known failures
+        rerun_jobs = filter_out_known_failures(rerun_statuses_jobs, conf.known_failure_patterns)
+        if rerun_jobs:
+            conf.filter_in.extend(job['description'] for job in rerun_jobs if job.get('description'))
+            log.info("Using unknown failure filtering for rerun with %d unknown failures", len(rerun_jobs))
+        else:
+            log.info("No unknown failures found, skipping rerun")
+            return False
+    else:
+        original_descriptions = [job['description'] for job in run['jobs'] if job['status'] in conf.rerun_statuses and job['description']]
+        conf.filter_in.extend(original_descriptions)
+        log.info("Using backward-compatible job filtering for rerun with %d job descriptions", len(original_descriptions))
+
+
+def _get_default_known_failure_patterns_file():
+    """Return the path to the default known patterns file bundled with teuthology.
+
+    The file format is documented in docs/known_failure_patterns.rst.
+
+    :returns: Path to the default known-failures.json file
+    """
+    return os.path.join(os.path.dirname(__file__), 'patterns', 'known-failures.json')
+
+
+def _load_known_patterns(path):
+    """Load known failure patterns from a JSON or YAML file.
+
+    :param path: Path to the patterns file. Default is set in docopt.
+                The file format should be:
+                JSON: {"patterns": ["pattern1", "pattern2", ...]}
+                YAML: patterns: ["pattern1", "pattern2", ...]
+    :returns: List of regex patterns (strings) to match against failure reasons.
+    """
+    if path is None:
+        path = _get_default_known_failure_patterns_file()
+
+    try:
+        with open(path, 'r') as f:
+            try:
+                patterns_data = json.load(f)
+            except json.JSONDecodeError:
+                f.seek(0)
+                patterns_data = yaml.safe_load(f)
+
+        known_patterns = patterns_data.get('patterns', [])
+        if isinstance(known_patterns, list):
+            return known_patterns
+        else:
+            log.warning(f"Patterns in {path} should be a list, treating as empty")
+            return []
+    except FileNotFoundError:
+        log.warning(f"Could not load known patterns from {path}, treating all as unknown")
+        return []
+    except (yaml.YAMLError, json.JSONDecodeError) as e:
+        log.warning(f"Could not parse known patterns from {path}: {e}, treating all as unknown")
+        return []
+
+
+def filter_out_known_failures(jobs, path=None):
+    """Filter out jobs with known failure patterns.
+
+    The known patterns file format is documented in docs/known_failure_patterns.rst.
+
+    :param jobs: Iterable of job dictionaries with 'failure_reason' and 'description' keys.
+                 Can be a list, generator, or any iterable.
+    :param path: Optional path to known patterns file. If None, uses
+                 the default bundled file.
+    :returns: List of job dictionaries for jobs with unknown failures (i.e., failures
+              that don't match any known pattern). Only includes jobs that were passed
+              in the input and have descriptions.
+    """
+    known_patterns = _load_known_patterns(path)
+    job_descriptions = []
+    for job in jobs:
+        if not job.get('description'):
+            continue
+
+        failure_reason = job.get('failure_reason')
+        if not failure_reason:
+            job_descriptions.append(job)
+            log.debug(f"Job {job['description']} has no failure reason, treating as unknown")
+            continue
+
+        is_known = False
+        for pattern in known_patterns:
+            if re.search(pattern, failure_reason):
+                is_known = True
+                log.debug(f"Job {job['description']} matches known pattern: {pattern}")
+                break
+        if not is_known:
+            job_descriptions.append(job)
+            log.debug(f"Job {job['description']} has unknown failure: {failure_reason}")
+
+    return job_descriptions
 
 
 def get_rerun_filters(run, statuses):

teuthology/suite/patterns/known-failures.json

@@ -0,0 +1,7 @@
+{
+    "patterns": [
+        "Command failed on .* with status 1:",
+        "clocks not synchronized",
+        "cluster \\[WRN\\] Health check failed: 575/44578 objects unfound \\(1\\.290%\\) \\(OBJECT_UNFOUND\\)"
+    ]
+}

teuthology/suite/test/test_init.py

@@ -265,3 +265,94 @@ def test_schedule_suite(self):
                 '--machine-type', machine_type
             ])
             m_sleep.assert_called_with(int(throttle))
+
+
+class TestFilterOutKnownFailures(object):
+    """Tests for filter_out_known_failures function."""
+
+    def test_filter_out_known_failures_json(self, tmpdir):
+        """Test filtering with JSON patterns file."""
+        patterns_file = tmpdir.join('patterns.json')
+        patterns_file.write('{"patterns": ["Command failed", "timeout"]}')
+
+        jobs = [
+            {'description': 'job1', 'failure_reason': 'Command failed on host'},
+            {'description': 'job2', 'failure_reason': 'Unknown error'},
+            {'description': 'job3', 'failure_reason': 'timeout occurred'},
+        ]
+
+        result = suite.filter_out_known_failures(jobs, str(patterns_file))
+        assert len(result) == 1
+        assert result[0]['description'] == 'job2'
+
+    def test_filter_out_known_failures_yaml(self, tmpdir):
+        """Test filtering with YAML patterns file."""
+        patterns_file = tmpdir.join('patterns.yaml')
+        patterns_file.write('patterns:\n  - "Command failed"\n  - "timeout"')
+
+        jobs = [
+            {'description': 'job1', 'failure_reason': 'Command failed on host'},
+            {'description': 'job2', 'failure_reason': 'Unknown error'},
+        ]
+
+        result = suite.filter_out_known_failures(jobs, str(patterns_file))
+        assert len(result) == 1
+        assert result[0]['description'] == 'job2'
+
+    def test_filter_out_known_failures_missing_file(self, tmpdir):
+        """Test filtering when patterns file is missing."""
+        patterns_file = tmpdir.join('nonexistent.json')
+
+        jobs = [
+            {'description': 'job1', 'failure_reason': 'Some error'},
+            {'description': 'job2', 'failure_reason': 'Another error'},
+        ]
+
+        # Should treat all as unknown when file is missing
+        result = suite.filter_out_known_failures(jobs, str(patterns_file))
+        assert len(result) == 2
+
+    def test_filter_out_known_failures_no_patterns(self, tmpdir):
+        """Test filtering with empty patterns."""
+        patterns_file = tmpdir.join('patterns.json')
+        patterns_file.write('{"patterns": []}')
+
+        jobs = [
+            {'description': 'job1', 'failure_reason': 'Some error'},
+            {'description': 'job2', 'failure_reason': 'Another error'},
+        ]
+
+        # All should be unknown when patterns are empty
+        result = suite.filter_out_known_failures(jobs, str(patterns_file))
+        assert len(result) == 2
+
+    def test_filter_out_known_failures_no_description(self, tmpdir):
+        """Test filtering with jobs that have no description."""
+        patterns_file = tmpdir.join('patterns.json')
+        patterns_file.write('{"patterns": ["error"]}')
+
+        jobs = [
+            {'description': 'job1', 'failure_reason': 'error occurred'},
+            {'failure_reason': 'error occurred'},  # No description
+            {'description': 'job2', 'failure_reason': 'unknown'},
+        ]
+
+        result = suite.filter_out_known_failures(jobs, str(patterns_file))
+        # Job without description should be skipped
+        assert len(result) == 1
+        assert result[0]['description'] == 'job2'
+
+    def test_filter_out_known_failures_regex_pattern(self, tmpdir):
+        """Test filtering with regex patterns."""
+        patterns_file = tmpdir.join('patterns.json')
+        patterns_file.write('{"patterns": ["Command failed on .* with status"]}')
+
+        jobs = [
+            {'description': 'job1', 'failure_reason': 'Command failed on host1 with status 1'},
+            {'description': 'job2', 'failure_reason': 'Command failed on host2 with status 2'},
+            {'description': 'job3', 'failure_reason': 'Different error'},
+        ]
+
+        result = suite.filter_out_known_failures(jobs, str(patterns_file))
+        assert len(result) == 1
+        assert result[0]['description'] == 'job3'