diff --git a/avocado/core/status/server.py b/avocado/core/status/server.py index d2ccf9d725..8589255ffa 100644 --- a/avocado/core/status/server.py +++ b/avocado/core/status/server.py @@ -1,7 +1,41 @@ import asyncio import os +from avocado.core.output import LOG_JOB from avocado.core.settings import settings +from avocado.utils.network import ports as network_ports + + +def resolve_listen_uri(uri): + """ + Normalize a status server URI that may contain a port range into + a concrete "host:port" endpoint. + """ + if ":" not in uri: + return uri + host, port_spec = uri.rsplit(":", 1) + if "-" not in port_spec: + return uri + + start_s, end_s = port_spec.split("-", 1) + start = int(start_s) + end = int(end_s) + if start > end: + raise ValueError( + f"Invalid port range (start > end) in status server URI: {uri}" + ) + + port = network_ports.find_free_port( + start_port=start, + end_port=end, + address=host, + sequent=True, + ) + if port is None: + raise OSError( + f"Could not bind status server to any port in range {start}-{end} on {host}" + ) + return f"{host}:{port}" class StatusServer: @@ -16,7 +50,7 @@ def __init__(self, uri, repo): messages :type repo: :class:`avocado.core.status.repo.StatusRepo` """ - self._uri = uri + self._uri = resolve_listen_uri(uri) self._repo = repo self._server_task = None @@ -27,7 +61,7 @@ def uri(self): async def create_server(self): limit = settings.as_dict().get("run.status_server_buffer_size") if ":" in self._uri: - host, port = self._uri.split(":") + host, port = self._uri.rsplit(":", 1) port = int(port) self._server_task = await asyncio.start_server( self.cb, host=host, port=port, limit=limit @@ -36,6 +70,7 @@ async def create_server(self): self._server_task = await asyncio.start_unix_server( self.cb, path=self._uri, limit=limit ) + LOG_JOB.info("Status server listening on %s", self._uri) async def serve_forever(self): if self._server_task is None: diff --git a/avocado/plugins/runner_nrunner.py b/avocado/plugins/runner_nrunner.py index 762741f754..a2aed86491 100644 --- a/avocado/plugins/runner_nrunner.py +++ b/avocado/plugins/runner_nrunner.py @@ -18,10 +18,7 @@ import asyncio import multiprocessing -import os -import platform import random -import tempfile from avocado.core.dispatcher import SpawnerDispatcher from avocado.core.exceptions import JobError, JobFailFast @@ -31,11 +28,12 @@ from avocado.core.plugin_interfaces import CLI, Init, SuiteRunner from avocado.core.settings import settings from avocado.core.status.repo import StatusRepo -from avocado.core.status.server import StatusServer +from avocado.core.status.server import StatusServer, resolve_listen_uri from avocado.core.task.runtime import RuntimeTaskGraph from avocado.core.task.statemachine import TaskStateMachine, Worker -DEFAULT_SERVER_URI = "127.0.0.1:8888" +# Default port range so multiple avocado runs can bind without conflict +DEFAULT_SERVER_URI = "127.0.0.1:8888-9000" class RunnerInit(Init): @@ -55,10 +53,9 @@ def initialize(self): ) help_msg = ( - "If the status server should automatically choose " - 'a "status_server_listen" and "status_server_uri" ' - "configuration. Default is to auto configure a " - "status server." + "If the status server should automatically choose a listen address " + "from the default port range so multiple runs do not conflict. " + "When disabled, use status_server_listen/status_server_uri." ) settings.register_option( section=section, @@ -69,10 +66,10 @@ def initialize(self): ) help_msg = ( - 'URI where status server will listen on. Usually a "HOST:PORT" ' - 'string. This is only effective if "status_server_auto" is disabled. ' - 'If "status_server_uri" is not set, the value from "status_server_listen " ' - "will be used." + 'URI where status server will listen. "HOST:PORT" or "HOST:START-END" ' + "port range (default: 127.0.0.1:8888-9000). Only used when " + '"status_server_auto" is disabled. If "status_server_uri" is not set, ' + '"status_server_listen" is used.' ) settings.register_option( section=section, @@ -83,12 +80,10 @@ def initialize(self): ) help_msg = ( - "URI for connecting to the status server, usually " - 'a "HOST:PORT" string. Use this if your status server ' - "is in another host, or different port. This is only " - 'effective if "status_server_auto" is disabled. ' - 'If "status_server_listen" is not set, the value from "status_server_uri" ' - "will be used." + 'URI for connecting to the status server: "HOST:PORT" or "HOST:START-END" ' + 'port range (default: 127.0.0.1:8888-9000). Only used when "status_server_auto" ' + 'is disabled. If "status_server_listen" is not set, ' + '"status_server_uri" is used.' ) settings.register_option( section=section, @@ -207,19 +202,8 @@ class Runner(SuiteRunner): name = "nrunner" description = "nrunner based implementation of job compliant runner" - def __init__(self): - super().__init__() - self.status_server_dir = None - def _determine_status_server(self, test_suite, config_key): - if test_suite.config.get("run.status_server_auto"): - # no UNIX domain sockets on Windows - if platform.system() != "Windows": - if self.status_server_dir is None: - self.status_server_dir = tempfile.TemporaryDirectory( - prefix="avocado_" - ) - return os.path.join(self.status_server_dir.name, ".status_server.sock") + """Return listen/uri config; default is a port range so multiple runs work.""" return test_suite.config.get(config_key) def _sync_status_server_urls(self, config): @@ -240,10 +224,19 @@ def _sync_status_server_urls(self, config): def _create_status_server(self, test_suite, job): self._sync_status_server_urls(test_suite.config) listen = self._determine_status_server(test_suite, "run.status_server_listen") + try: + resolved_listen = resolve_listen_uri(listen) + except (ValueError, OSError) as exc: + raise JobError(str(exc)) from exc + if resolved_listen != listen: + test_suite.config["run.status_server_listen"] = resolved_listen + server_uri = test_suite.config.get("run.status_server_uri") + if server_uri in (listen, DEFAULT_SERVER_URI): + test_suite.config["run.status_server_uri"] = resolved_listen # pylint: disable=W0201 self.status_repo = StatusRepo(job.unique_id) # pylint: disable=W0201 - self.status_server = StatusServer(listen, self.status_repo) + self.status_server = StatusServer(resolved_listen, self.status_repo) async def _update_status(self, job): message_handler = MessageHandler() @@ -384,8 +377,6 @@ def run_suite(self, job, test_suite): job.result.end_tests() self.status_server.close() - if self.status_server_dir is not None: - self.status_server_dir.cleanup() # Update the overall summary with found test statuses, which will # determine the Avocado command line exit status diff --git a/man/avocado.rst b/man/avocado.rst index e32b180aff..94987020a0 100644 --- a/man/avocado.rst +++ b/man/avocado.rst @@ -179,23 +179,24 @@ Options for subcommand `run` (`avocado run --help`):: nrunner specific options: --shuffle Shuffle the tasks to be executed --status-server-disable-auto - If the status server should automatically choose a - "status_server_listen" and "status_server_uri" - configuration. Default is to auto configure a status - server. + Disable automatic status server port selection. By + default, a port range (127.0.0.1:8888-9000) is used + so multiple avocado runs can run without port + conflicts. When disabled, use + --status-server-listen/--status-server-uri. --status-server-listen HOST_PORT - URI where status server will listen on. Usually a - "HOST:PORT" string. This is only effective if - "status_server_auto" is disabled. If - "status_server_uri" is not set, the value from - "status_server_listen " will be used. + URI where status server will listen: "HOST:PORT" or + "HOST:START-END" (default 127.0.0.1:8888-9000). An + available port in the range is chosen. Only used + when --status-server-disable-auto is set. If + "status_server_uri" is not set, + "status_server_listen" is used. --status-server-uri HOST_PORT - URI for connecting to the status server, usually a - "HOST:PORT" string. Use this if your status server is - in another host, or different port. This is only - effective if "status_server_auto" is disabled. If - "status_server_listen" is not set, the value from - "status_server_uri" will be used. + URI for connecting to the status server: "HOST:PORT" + or "HOST:START-END". Only used when + --status-server-disable-auto is set. If + "status_server_listen" is not set, + "status_server_uri" is used. --max-parallel-tasks NUMBER_OF_TASKS Number of maximum number tasks running in parallel. You can disable parallel execution by setting this to