diff --git a/avocado/core/enabled_extension_manager.py b/avocado/core/enabled_extension_manager.py index 668d7fb20f..2b82241477 100644 --- a/avocado/core/enabled_extension_manager.py +++ b/avocado/core/enabled_extension_manager.py @@ -38,10 +38,28 @@ def __init__(self, namespace, invoke_kwds=None): def enabled(self, extension): """ - Checks configuration for explicit mention of plugin in a disable list + Check configuration for explicit mention of plugin in a disable list. + + Defaults to checking for enabled extensions. + """ + return self.enabled_extension(extension) + + def enabled_extension(self, extension): + """ + Check configuration for explicit mention of an extension in a disable list. If configuration section or key doesn't exist, it means no plugin is disabled. """ - disabled = settings.as_dict().get("plugins.disable") + disabled = settings.as_dict().get("plugins.disable") or [] return self.fully_qualified_name(extension) not in disabled + + def enabled_entry_point(self, entry_point): + """ + Check configuration for explicit mention of an entry point in a disable list. + + If configuration section or key doesn't exist, it means no plugin + is disabled. + """ + disabled = settings.as_dict().get("plugins.disable") or [] + return self.fully_qualified_entry_point_name(entry_point) not in disabled diff --git a/avocado/core/extension_manager.py b/avocado/core/extension_manager.py index 77aa7753c1..926e00856d 100644 --- a/avocado/core/extension_manager.py +++ b/avocado/core/extension_manager.py @@ -90,6 +90,8 @@ def __init__(self, namespace, invoke_kwds=None): # load plugins for ep in get_entry_points_for(self.namespace): + if not self.enabled_entry_point(ep): + continue try: plugin = ep.load() obj = plugin(**invoke_kwds) @@ -110,6 +112,16 @@ def enabled(self, extension): # pylint: disable=W0613,R0201 """ return True + def enabled_entry_point(self, entry_point): # pylint: disable=W0613,R0201 + """ + Default configuration on whether an entry point should be imported. + + Subclasses can reject an entry point before loading its module if needed. + This matters for explicitly disabled plugins whose dependencies may not be + available or whose import has side effects. + """ + return True + def plugin_type(self): """ Subset of entry points namespace for this dispatcher @@ -125,13 +137,31 @@ def plugin_type(self): def fully_qualified_name(self, extension): """ - Returns the Avocado fully qualified plugin name + Return the Avocado fully qualified plugin name. + + :param extension: an Extension instance + :type extension: :class:`Extension` + """ + return self.fully_qualified_extension_name(extension) + + def fully_qualified_extension_name(self, extension): + """ + Return the Avocado fully qualified plugin name via extension. :param extension: an Extension instance :type extension: :class:`Extension` """ return f"{self.plugin_type()}.{extension.entry_point.name}" + def fully_qualified_entry_point_name(self, entry_point): + """ + Return the Avocado fully qualified plugin name via entry point. + + :param entry_point: an EntryPoint instance + :type entry_point: :class:`importlib.metadata.EntryPoint` + """ + return f"{self.plugin_type()}.{entry_point.name}" + def settings_section(self): """ Returns the config section name for the plugin type handled by itself diff --git a/selftests/check.py b/selftests/check.py index d3751a4cb8..9005765580 100755 --- a/selftests/check.py +++ b/selftests/check.py @@ -1,13 +1,113 @@ #!/usr/bin/env python3 import argparse +import atexit import copy import glob import multiprocessing import os import platform import re +import shutil import sys +import tempfile +from importlib.metadata import distributions + +SELFTESTS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +OPTIONAL_PLUGINS = { + "ansible": "avocado-framework-plugin-ansible", + "golang": "avocado-framework-plugin-golang", + "html": "avocado-framework-plugin-result-html", + "robot": "avocado-framework-plugin-robot", + "varianter_cit": "avocado-framework-plugin-varianter-cit", + "varianter_yaml_to_mux": "avocado-framework-plugin-varianter-yaml-to-mux", +} +SELFTEST_PLUGIN_DISTRIBUTIONS = {"magic", "avocado-rogue"} + + +def _distribution_name(distribution): + """Return a normalized distribution name.""" + name = distribution.metadata.get("Name", "") + return re.sub(r"[-_.]+", "-", name).lower() + + +def _external_plugins(allowed_distributions): + """Return installed Avocado plugins that do not belong to this tree.""" + disabled = set() + for distribution in distributions(): + if _distribution_name(distribution) in allowed_distributions: + continue + for entry_point in distribution.entry_points: + if entry_point.group.startswith("avocado.plugins."): + plugin_type = entry_point.group.removeprefix("avocado.plugins.") + disabled.add(f"{plugin_type}.{entry_point.name}") + return sorted(disabled) + + +def _disabled_plugin_checks_from_argv(): + """Read plugin exclusions before importing Avocado and its plugins.""" + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--disable-plugin-checks", action="append", default=[]) + parsed, _ = parser.parse_known_args() + return { + plugin for value in parsed.disable_plugin_checks for plugin in value.split(",") + } + + +def _setup_isolated_environment(): + """Keep source-tree selftests independent of the host installation if any.""" + disabled_plugin_checks = _disabled_plugin_checks_from_argv() + enabled_optional_plugins = { + name: distribution + for name, distribution in OPTIONAL_PLUGINS.items() + if name not in disabled_plugin_checks + } + python_paths = [ + SELFTESTS_ROOT, + *( + os.path.join(SELFTESTS_ROOT, "optional_plugins", plugin) + for plugin in enabled_optional_plugins + ), + ] + inherited_python_path = os.environ.get("PYTHONPATH") + if inherited_python_path: + inherited_paths = { + os.path.abspath(path) + for path in inherited_python_path.split(os.pathsep) + if path + } + sys.path[:] = [ + path + for path in sys.path + if os.path.abspath(path) not in inherited_paths + or os.path.abspath(path) == SELFTESTS_ROOT + ] + python_paths = list(dict.fromkeys(os.path.abspath(path) for path in python_paths)) + os.environ["PYTHONPATH"] = os.pathsep.join(python_paths) + for path in reversed(python_paths): + if path not in sys.path: + sys.path.insert(0, path) + + # Settings treats VIRTUAL_ENV as both its system and user configuration + # root. Pointing it at an empty temporary directory prevents /etc and + # ~/.config from changing selftest behavior, and is inherited by every + # runner and avocado subprocess started below. + isolation_dir = tempfile.mkdtemp(prefix="selftests-config-") + os.environ["VIRTUAL_ENV"] = isolation_dir + config_dir = os.path.join(isolation_dir, "etc", "avocado") + os.makedirs(config_dir) + config_path = os.path.join(config_dir, "avocado.conf") + with open(config_path, "w", encoding="utf-8") as config_file: + allowed_distributions = {"avocado-framework"} + allowed_distributions.update(enabled_optional_plugins.values()) + allowed_distributions.update(SELFTEST_PLUGIN_DISTRIBUTIONS) + config_file.write( + "[plugins]\n" f"disable = {_external_plugins(allowed_distributions)!r}\n" + ) + atexit.register(shutil.rmtree, isolation_dir, ignore_errors=True) + + +_setup_isolated_environment() from avocado import Test from avocado.core import exit_codes @@ -27,7 +127,7 @@ "job-api-check-tmp-directory-exists": 1, "nrunner-interface": 90, "nrunner-requirement": 28, - "unit": 1027, + "unit": 1028, "jobs": 11, "functional-parallel": 368, "functional-serial": 7, @@ -43,6 +143,31 @@ } +def _optional_plugin_checks_enabled(plugin_name, disabled_plugins): + distribution_name = OPTIONAL_PLUGINS.get(plugin_name) + return ( + distribution_name is not None + and plugin_name not in disabled_plugins + and python_module_available(distribution_name) + ) + + +SERIAL_SUITES = { + "functional-serial", + "nrunner-requirement", + "pre-release", + "vmimage-tests", + "vmimage-variants", +} + + +def _set_max_parallel_tasks(suites, max_parallel_tasks): + for suite in suites: + suite.config["run.max_parallel_tasks"] = ( + 1 if suite.name in SERIAL_SUITES else max_parallel_tasks + ) + + class JobAPIFeaturesTest(Test): def check_directory_exists(self, path=None): """Check if a directory exists""" @@ -269,8 +394,19 @@ def parse_args(): action="append", default=[], ) + parser.add_argument( + "-j", + "--max-parallel-tasks", + type=int, + help=( + "Maximum tasks for each parallel suite. Suites that require serial " + "execution remain limited to one task." + ), + ) arg = parser.parse_args() + if arg.max_parallel_tasks is not None and arg.max_parallel_tasks < 1: + parser.error("--max-parallel-tasks must be greater than zero") return arg @@ -424,7 +560,7 @@ def get_ref(method_short_name): "value": 1, "reference": ["examples/tests/sleeptenmin.py"], "file": "job.log", - "content": "RuntimeError: Test interrupted by SIGTERM", + "content": "Test interrupted: Timeout reached", "assert": True, "exit_code": 8, }, @@ -637,6 +773,7 @@ def create_suites(args): # pylint: disable=W0621 if ( python_module_available("avocado-framework-plugin-golang") + and shutil.which("avocado-runner-golang") is not None and "golang" not in args.disable_plugin_checks ): config_nrunner_interface["run.dict_variants"].append( @@ -648,6 +785,7 @@ def create_suites(args): # pylint: disable=W0621 if ( python_module_available("avocado-framework-plugin-robot") + and shutil.which("avocado-runner-robot") is not None and "robot" not in args.disable_plugin_checks ): config_nrunner_interface["run.dict_variants"].append( @@ -659,6 +797,7 @@ def create_suites(args): # pylint: disable=W0621 if ( python_module_available("avocado-framework-plugin-ansible") + and shutil.which("avocado-runner-ansible-module") is not None and "ansible" not in args.disable_plugin_checks ): config_nrunner_interface["run.dict_variants"].append( @@ -737,7 +876,7 @@ def create_suites(args): # pylint: disable=W0621 config_check_optional["resolver.references"] = [] for optional_plugin in glob.glob("optional_plugins/*"): plugin_name = os.path.basename(optional_plugin) - if plugin_name not in args.disable_plugin_checks: + if _optional_plugin_checks_enabled(plugin_name, args.disable_plugin_checks): pattern = f"{optional_plugin}/tests/*" config_check_optional["resolver.references"] += glob.glob(pattern) @@ -787,6 +926,12 @@ def create_suites(args): # pylint: disable=W0621 def main(args): # pylint: disable=W0621 + args.disable_plugin_checks = [ + plugin for value in args.disable_plugin_checks for plugin in value.split(",") + ] + args.select = [item for value in args.select for item in value.split(",")] + args.skip = [item for value in args.skip for item in value.split(",")] + args.dict_tests = { "static-checks": False, "job-api": False, @@ -802,26 +947,14 @@ def main(args): # pylint: disable=W0621 "pre-release": False, } - if python_module_available("avocado-framework-plugin-golang"): - TEST_SIZE["optional-plugins"] += TEST_SIZE["optional-plugins-golang"] - if python_module_available("avocado-framework-plugin-result-html"): - TEST_SIZE["optional-plugins"] += TEST_SIZE["optional-plugins-html"] - if python_module_available("avocado-framework-plugin-robot"): - TEST_SIZE["optional-plugins"] += TEST_SIZE["optional-plugins-robot"] - if python_module_available("avocado-framework-plugin-varianter-cit"): - TEST_SIZE["optional-plugins"] += TEST_SIZE["optional-plugins-varianter_cit"] - if python_module_available("avocado-framework-plugin-varianter-yaml-to-mux"): - TEST_SIZE["optional-plugins"] += TEST_SIZE[ - "optional-plugins-varianter_yaml_to_mux" - ] - - # Make a list of strings instead of a list with a single string - if len(args.disable_plugin_checks) > 0: - args.disable_plugin_checks = args.disable_plugin_checks[0].split(",") - if len(args.select) > 0: - args.select = args.select[0].split(",") - if len(args.skip) > 0: - args.skip = args.skip[0].split(",") + optional_plugins_size = 0 + for plugin_name in OPTIONAL_PLUGINS: + size_key = f"optional-plugins-{plugin_name}" + if size_key in TEST_SIZE and _optional_plugin_checks_enabled( + plugin_name, args.disable_plugin_checks + ): + optional_plugins_size += TEST_SIZE[size_key] + TEST_SIZE["optional-plugins"] = optional_plugins_size # Print features covered in this test if args.list_features: @@ -887,6 +1020,10 @@ def main(args): # pylint: disable=W0621 if suite.name == "functional-parallel": suite.config["run.max_parallel_tasks"] = max_parallel + max_parallel_tasks = getattr(args, "max_parallel_tasks", None) + if max_parallel_tasks is not None: + _set_max_parallel_tasks(suites, max_parallel_tasks) + with Job(config, suites) as j: pre_job_test_result_dirs = set(os.listdir(os.path.dirname(j.logdir))) exit_code = j.run() diff --git a/selftests/functional/basic.py b/selftests/functional/basic.py index aa50f0e850..8261719cc0 100644 --- a/selftests/functional/basic.py +++ b/selftests/functional/basic.py @@ -151,8 +151,13 @@ def probe_binary(binary): if GNU_ECHO_BINARY is not None: if probe_binary("man") is not None: echo_cmd = f"man {os.path.basename(GNU_ECHO_BINARY)}" - echo_manpage = process.run(echo_cmd, env={"LANG": "C"}, encoding="ascii").stdout - if b"-e" not in echo_manpage: + echo_manpage = process.run( + echo_cmd, + env={"LANG": "C"}, + encoding="ascii", + ignore_status=True, + ) + if echo_manpage.exit_status == 0 and b"-e" not in echo_manpage.stdout: GNU_ECHO_BINARY = probe_binary("gecho") READ_BINARY = probe_binary("read") SLEEP_BINARY = probe_binary("sleep") diff --git a/selftests/functional/plugin/jsonresult.py b/selftests/functional/plugin/jsonresult.py index dc35a6f863..8fa5f2b926 100644 --- a/selftests/functional/plugin/jsonresult.py +++ b/selftests/functional/plugin/jsonresult.py @@ -47,11 +47,11 @@ def test_tags_in_result(self): def test_variant(self): cmd_line = ( f"{AVOCADO} run examples/tests/passtest.py " - "--mux-yaml examples/yaml_to_mux/simple_vars.yaml " + "--json-variants-load selftests/functional/plugin/result_variants.json " f"--job-results-dir {self.tmpdir.name} --disable-sysinfo " "--max-parallel-tasks=1" ) - process.run(cmd_line, ignore_status=True) + process.run(cmd_line) json_path = path.join(self.tmpdir.name, "latest", "results.json") with open(json_path, "r", encoding="utf-8") as json_file: data = json.load(json_file) diff --git a/selftests/functional/plugin/result_variants.json b/selftests/functional/plugin/result_variants.json new file mode 100644 index 0000000000..99a112c5a4 --- /dev/null +++ b/selftests/functional/plugin/result_variants.json @@ -0,0 +1,16 @@ +[ + { + "paths": ["/run/*"], + "variant_id": "run-first-febe", + "variant": [ + ["/run/first", [["/run/first", "variable_one", 1]]] + ] + }, + { + "paths": ["/run/*"], + "variant_id": "run-second-bafe", + "variant": [ + ["/run/second", [["/run/second", "variable_two", 2]]] + ] + } +] diff --git a/selftests/functional/plugin/tmtresult.py b/selftests/functional/plugin/tmtresult.py index c1f5415b26..fb4ca17d10 100644 --- a/selftests/functional/plugin/tmtresult.py +++ b/selftests/functional/plugin/tmtresult.py @@ -1,13 +1,18 @@ import re +import unittest from os import path -import yaml +try: + import yaml +except ImportError: + yaml = None from avocado.utils import process from selftests.utils import AVOCADO, TestCaseTmpDir class TMTResultTest(TestCaseTmpDir): + @unittest.skipUnless(yaml, "PyYAML module not available") def test_logfile(self): cmd_line = ( f"{AVOCADO} run examples/tests/failtest.py examples/tests/passtest.py" diff --git a/selftests/functional/plugin/xunit.py b/selftests/functional/plugin/xunit.py index 5fe88988f9..cb174b94b6 100644 --- a/selftests/functional/plugin/xunit.py +++ b/selftests/functional/plugin/xunit.py @@ -83,11 +83,11 @@ def test_error_reason(self): def test_variant(self): cmd_line = ( f"{AVOCADO} run examples/tests/passtest.py " - "--mux-yaml examples/yaml_to_mux/simple_vars.yaml " + "--json-variants-load selftests/functional/plugin/result_variants.json " f"--job-results-dir {self.tmpdir.name} --disable-sysinfo " "--max-parallel-tasks=1" ) - process.run(cmd_line, ignore_status=True) + process.run(cmd_line) xunit_path = path.join(self.tmpdir.name, "latest", "results.xml") with open(xunit_path, "rb") as fp: diff --git a/selftests/unit/dispatcher.py b/selftests/unit/dispatcher.py index d298c621a3..4ddc08d20f 100644 --- a/selftests/unit/dispatcher.py +++ b/selftests/unit/dispatcher.py @@ -1,7 +1,9 @@ import unittest +from unittest.mock import MagicMock, patch from avocado.core.dispatcher import EnabledExtensionManager from avocado.core.extension_manager import PluginPriority +from avocado.core.settings import settings class DispatcherTest(unittest.TestCase): @@ -28,6 +30,25 @@ def test_order(self): ) self.assertEqual(ext_objects, sort) + def test_disabled_plugin_is_not_imported(self): + entry_point = MagicMock() + entry_point.name = "disabled" + config = { + "plugins.cli.order": [], + "plugins.disable": ["cli.disabled"], + } + with ( + patch( + "avocado.core.extension_manager.get_entry_points_for", + return_value=[entry_point], + ), + patch.object(settings, "as_dict", return_value=config), + ): + manager = EnabledExtensionManager("avocado.plugins.cli") + + entry_point.load.assert_not_called() + self.assertEqual(manager.extensions, []) + if __name__ == "__main__": unittest.main()