Add post-test KubeVirtDeprecatedAPIRequested check

conftest.py

@@ -57,6 +57,7 @@
     deploy_run_in_progress_namespace,
     filter_hpp_tests,
     filter_multiarch_tests,
+    filter_post_test_alerts_tests,
     get_artifactory_server_url,
     get_base_matrix_name,
     get_cnv_version_explorer_url,
@@ -103,13 +104,13 @@
 
 TEAM_MARKERS = {
     "chaos": ["chaos", "deprecated_api"],
-    "virt": ["virt", "deprecated_api"],
-    "network": ["network", "deprecated_api"],
-    "storage": ["storage", "deprecated_api"],
-    "iuo": ["install_upgrade_operators", "deprecated_api"],
-    "observability": ["observability", "deprecated_api"],
-    "infrastructure": ["infrastructure", "deprecated_api"],
-    "data_protection": ["data_protection", "deprecated_api"],
+    "virt": ["virt", "deprecated_api", "post_test_alerts"],
+    "network": ["network", "deprecated_api", "post_test_alerts"],
+    "storage": ["storage", "deprecated_api", "post_test_alerts"],
+    "iuo": ["install_upgrade_operators", "deprecated_api", "post_test_alerts"],
+    "observability": ["observability", "deprecated_api", "post_test_alerts"],
+    "infrastructure": ["infrastructure", "deprecated_api", "post_test_alerts"],
+    "data_protection": ["data_protection", "deprecated_api", "post_test_alerts"],
 }
 NAMESPACE_COLLECTION = {
     "storage": [NamespacesNames.OPENSHIFT_STORAGE],
@@ -139,6 +140,7 @@ def pytest_addoption(parser):
     ci_group = parser.getgroup(name="CI")
     component_sanity_group = parser.getgroup(name="ComponentSanity")
     ai_insights_group = parser.getgroup(name="ai-job-insight")
+    post_test_alerts_group = parser.getgroup(name="PostTestAlerts")
 
     # Upgrade addoption
     install_upgrade_group.addoption(
@@ -307,6 +309,13 @@ def pytest_addoption(parser):
         action="store_true",
     )
 
+    # Post test alerts group
+    post_test_alerts_group.addoption(
+        "--skip-post-test-alerts",
+        help="By default test_no_critical_alerts_after_tests will always run, pass this flag to skip it",
+        action="store_true",
+    )
+
     # LeftoversCollector group
     leftovers_collector.addoption(
         "--leftovers-collector",
@@ -571,12 +580,19 @@ def filter_sno_only_tests(items: list[Item], config: Config) -> list[Item]:
 
 
 def pytest_configure(config):
+    config._test_execution_start_time = datetime.datetime.now(tz=datetime.UTC)
+
     # test_deprecation_audit_logs should always run regardless the path that passed to pytest.
     deprecation_tests_dir_path = "tests/deprecated_api"
     file_or_dir = config.option.file_or_dir
     if file_or_dir and deprecation_tests_dir_path not in file_or_dir and file_or_dir != ["tests"]:
         config.option.file_or_dir.append(deprecation_tests_dir_path)
 
+    # post_test_alerts tests should always run regardless the path that passed to pytest.
+    post_test_alerts_dir_path = "tests/post_test_alerts"
+    if file_or_dir and post_test_alerts_dir_path not in file_or_dir and file_or_dir != ["tests"]:
+        config.option.file_or_dir.append(post_test_alerts_dir_path)
+
     if conformance_storage_class := config.getoption("conformance_storage_class"):
         py_config["storage_class_matrix"] = StorageClassConfig(
             name=conformance_storage_class
@@ -644,6 +660,7 @@ def pytest_collection_modifyitems(session, config, items):
     if discard:
         config.hook.pytest_deselected(items=discard)
     items[:] = filter_deprecated_api_tests(items=items, config=config)
+    items[:] = filter_post_test_alerts_tests(items=items, config=config)
     items[:] = filter_sno_only_tests(items=items, config=config)
     items[:] = filter_multiarch_tests(items=items, config=config)
     items[:] = filter_hpp_tests(items=items, config=config)

tests/post_test_alerts/test_post_test_alerts.py

@@ -0,0 +1,44 @@
+"""
+Post-test alerts verification.
+
+Verifies that critical CNV alerts were not triggered during test execution.
+
+Jira: https://redhat.atlassian.net/browse/CNV-80353
+"""
+
+import datetime
+import logging
+
+import pytest
+
+LOGGER = logging.getLogger(__name__)
+
+DEPRECATED_API_ALERT = "KubeVirtDeprecatedAPIRequested"
+
+
+@pytest.mark.s390x
+@pytest.mark.polarion("CNV-16276")
+@pytest.mark.order("last")
+def test_no_critical_alerts_after_tests(prometheus, request):
+    """
+    Test that critical CNV alerts were not triggered during test execution.
+
+    Preconditions:
+        - Prometheus is accessible on the cluster
+        - Test execution completed
+
+    Steps:
+        1. Query Prometheus for alerts that fired at any point during test execution
+        2. Check that none of the critical alerts were triggered
+
+    Expected:
+        - None of the critical alerts fired during test execution
+    """
+    start_time = request.config._test_execution_start_time
+    duration_seconds = max(int((datetime.datetime.now(tz=datetime.UTC) - start_time).total_seconds()), 1)
+    LOGGER.info(
+        f"Checking {DEPRECATED_API_ALERT} alert was not triggered during test execution (last {duration_seconds}s)"
+    )
+    query = f'ALERTS{{alertname="{DEPRECATED_API_ALERT}", alertstate="firing"}}[{duration_seconds}s]'
+    results = prometheus.query_sampler(query=query)
+    assert not results, f"{DEPRECATED_API_ALERT} alert fired during test execution.\n{results}"

utilities/pytest_utils.py

@@ -12,6 +12,8 @@
 from typing import Any
 
 import pytest
+from _pytest.config import Config
+from _pytest.nodes import Item
 from kubernetes.dynamic import DynamicClient
 from ocp_resources.config_map import ConfigMap
 from ocp_resources.namespace import Namespace
@@ -715,3 +717,17 @@ def assert_incremental_classes_fully_collected(items: list[pytest.Item]) -> None
 
 def _is_xfail_no_run(method: object) -> bool:
     return any(mark.name == "xfail" and mark.kwargs.get("run") is False for mark in getattr(method, "pytestmark", []))
+
+
+def filter_post_test_alerts_tests(items: list[Item], config: Config) -> list[Item]:
+    # filter out post test alerts tests, if explicitly asked or if running upgrade/install tests
+    if (
+        config.getoption("--skip-post-test-alerts")
+        or config.getoption("--install")
+        or config.getoption("--upgrade")
+        or config.getoption("--upgrade_custom")
+    ):
+        discard_tests, items_to_return = remove_tests_from_list(items=items, filter_str="post_test_alerts")
+        config.hook.pytest_deselected(items=discard_tests)
+        return items_to_return
+    return items

utilities/unittests/test_pytest_utils.py

@@ -29,6 +29,7 @@
     exit_pytest_execution,
     filter_hpp_tests,
     filter_multiarch_tests,
+    filter_post_test_alerts_tests,
     generate_common_template_matrix_dicts,
     generate_instance_type_matrix_dicts,
     get_artifactory_server_url,
@@ -2493,6 +2494,81 @@ def test_empty_marker_expression_filters_hpp(self):
         config.hook.pytest_deselected.assert_called_once_with(items=[item_hpp])
 
 
+class TestFilterPostTestAlertsTests:
+    """Test cases for filter_post_test_alerts_tests function."""
+
+    def test_filters_when_skip_post_test_alerts_flag_set(self):
+        """Post-test alert tests are filtered out when --skip-post-test-alerts flag is set."""
+        item_post_test_alerts = MagicMock()
+        item_post_test_alerts.keywords = {"post_test_alerts": True}
+        item_other = MagicMock()
+        item_other.keywords = {"other_test": True}
+        config = MagicMock()
+        config.getoption.side_effect = lambda flag: flag == "--skip-post-test-alerts"
+
+        result = filter_post_test_alerts_tests(items=[item_post_test_alerts, item_other], config=config)
+
+        assert result == [item_other]
+        config.hook.pytest_deselected.assert_called_once_with(items=[item_post_test_alerts])
+
+    def test_filters_when_install_flag_set(self):
+        """Post-test alert tests are filtered out when --install flag is set."""
+        item_post_test_alerts = MagicMock()
+        item_post_test_alerts.keywords = {"post_test_alerts": True}
+        item_other = MagicMock()
+        item_other.keywords = {"other_test": True}
+        config = MagicMock()
+        config.getoption.side_effect = lambda flag: flag == "--install"
+
+        result = filter_post_test_alerts_tests(items=[item_post_test_alerts, item_other], config=config)
+
+        assert result == [item_other]
+        config.hook.pytest_deselected.assert_called_once_with(items=[item_post_test_alerts])
+
+    def test_filters_when_upgrade_flag_set(self):
+        """Post-test alert tests are filtered out when --upgrade flag is set."""
+        item_post_test_alerts = MagicMock()
+        item_post_test_alerts.keywords = {"post_test_alerts": True}
+        item_other = MagicMock()
+        item_other.keywords = {"other_test": True}
+        config = MagicMock()
+        config.getoption.side_effect = lambda flag: flag == "--upgrade"
+
+        result = filter_post_test_alerts_tests(items=[item_post_test_alerts, item_other], config=config)
+
+        assert result == [item_other]
+        config.hook.pytest_deselected.assert_called_once_with(items=[item_post_test_alerts])
+
+    def test_filters_when_upgrade_custom_flag_set(self):
+        """Post-test alert tests are filtered out when --upgrade_custom flag is set."""
+        item_post_test_alerts = MagicMock()
+        item_post_test_alerts.keywords = {"post_test_alerts": True}
+        item_other = MagicMock()
+        item_other.keywords = {"other_test": True}
+        config = MagicMock()
+        config.getoption.side_effect = lambda flag: flag == "--upgrade_custom"
+
+        result = filter_post_test_alerts_tests(items=[item_post_test_alerts, item_other], config=config)
+
+        assert result == [item_other]
+        config.hook.pytest_deselected.assert_called_once_with(items=[item_post_test_alerts])
+
+    def test_no_filtering_when_no_flags_set(self):
+        """All items are returned unchanged when no filtering flags are set."""
+        item_post_test_alerts = MagicMock()
+        item_post_test_alerts.keywords = {"post_test_alerts": True}
+        item_other = MagicMock()
+        item_other.keywords = {"other_test": True}
+        items = [item_post_test_alerts, item_other]
+        config = MagicMock()
+        config.getoption.side_effect = lambda flag: False
+
+        result = filter_post_test_alerts_tests(items=items, config=config)
+
+        assert result == items
+        config.hook.pytest_deselected.assert_not_called()
+
+
 class TestFilterMultiarchTests:
     """Test cases for filter_multiarch_tests function."""