diff --git a/lisa/microsoft/testsuites/device_passthrough/functional_tests.py b/lisa/microsoft/testsuites/device_passthrough/functional_tests.py index 6a3c536a5f..e7c4584024 100644 --- a/lisa/microsoft/testsuites/device_passthrough/functional_tests.py +++ b/lisa/microsoft/testsuites/device_passthrough/functional_tests.py @@ -1,13 +1,13 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. import re -from typing import TYPE_CHECKING, Any, Dict, Tuple, Union, cast +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union, cast from lisa import Environment, Node, TestCaseMetadata, TestSuite, TestSuiteMetadata from lisa.base_tools import Cat from lisa.operating_system import Windows from lisa.platform_ import Platform -from lisa.sut_orchestrator import CLOUD_HYPERVISOR, HYPERV +from lisa.sut_orchestrator import CLOUD_HYPERVISOR, HYPERV, OPENVMM from lisa.testsuite import TestResult, simple_requirement from lisa.tools import Lspci from lisa.util import LisaException, SkippedException @@ -20,13 +20,17 @@ from lisa.sut_orchestrator.libvirt.schema import ( DeviceAddressSchema as LibvirtDeviceAddressSchema, ) + from lisa.sut_orchestrator.openvmm.context import ( + DeviceAddressSchema as OpenVmmDeviceAddressSchema, + ) HostDeviceAddressSchema = Union[ HypervDeviceAddressSchema, LibvirtDeviceAddressSchema, + OpenVmmDeviceAddressSchema, ] -SUPPORTED_PASSTHROUGH_PLATFORMS = [CLOUD_HYPERVISOR, HYPERV] +SUPPORTED_PASSTHROUGH_PLATFORMS = [CLOUD_HYPERVISOR, HYPERV, OPENVMM] @TestSuiteMetadata( @@ -44,8 +48,8 @@ class DevicePassthroughFunctionalTests(TestSuite): @TestCaseMetadata( description=""" Check if passthrough device is visible to guest. - This testcase supports the CLOUD_HYPERVISOR and HYPERV platforms - of LISA. Please refer below runbook snippet. + This testcase supports the CLOUD_HYPERVISOR, HYPERV, and OPENVMM + platforms of LISA. Please refer below runbook snippet. platform: - type: cloud-hypervisor @@ -89,32 +93,22 @@ def verify_device_passthrough_on_guest( if platform is None: raise SkippedException( "Device passthrough validation requires a LISA platform context. " - "Verify the runbook uses cloud-hypervisor or hyperv." - ) - platform_name = platform.type_name() - node_context: Any - - if platform_name == CLOUD_HYPERVISOR: - # Import at runtime to avoid libvirt dependency on other platforms. - from lisa.sut_orchestrator.libvirt.context import ( - get_node_context as get_libvirt_node_context, + "Verify the runbook uses cloud-hypervisor, hyperv, or openvmm." ) + platform_name = self._get_platform_name(platform, node) + node_context = self._get_passthrough_context(node, platform_name) - node_context = get_libvirt_node_context(node) - elif platform_name == HYPERV: - from lisa.sut_orchestrator.hyperv.context import ( - get_node_context as get_hyperv_node_context, - ) + if not node_context.passthrough_devices: + raise SkippedException("No passthrough devices are assigned to node") - node_context = get_hyperv_node_context(node) - else: + host_node = getattr(node_context, "host", None) + if host_node is None and environment.platform is not None: + host_node = getattr(environment.platform, "host_node", None) + if host_node is None and platform_name != HYPERV: raise SkippedException( - f"Device passthrough validation is not supported on '{platform_name}'" + "No host node is available for passthrough device validation" ) - if not node_context.passthrough_devices: - raise SkippedException("No passthrough devices are assigned to node") - expected_devices: Dict[Tuple[str, str, str], int] = {} for passthrough_context in node_context.passthrough_devices: pool_type = str(passthrough_context.pool_type.value) @@ -124,7 +118,7 @@ def verify_device_passthrough_on_guest( ) for host_device in passthrough_context.device_list: vendor_device_id = self._vendor_device_from_host_device( - platform, host_device + platform_name, platform, host_node, host_device ) key = ( pool_type, @@ -147,12 +141,48 @@ def verify_device_passthrough_on_guest( f"Vendor/Device ID: {ven_id}:{dev_id}" ) + @staticmethod + def _get_platform_name(platform: Platform, node: Node) -> str: + node_type = node.type_name() + if node_type == OPENVMM: + return node_type + + return platform.type_name() + + @staticmethod + def _get_passthrough_context(node: Node, platform_name: str) -> Any: + if platform_name == OPENVMM: + from lisa.sut_orchestrator.openvmm.context import ( + get_node_context as get_openvmm_node_context, + ) + + return get_openvmm_node_context(node) + + if platform_name == CLOUD_HYPERVISOR: + from lisa.sut_orchestrator.libvirt.context import ( + get_node_context as get_libvirt_node_context, + ) + + return get_libvirt_node_context(node) + + if platform_name == HYPERV: + from lisa.sut_orchestrator.hyperv.context import ( + get_node_context as get_hyperv_node_context, + ) + + return get_hyperv_node_context(node) + + raise SkippedException( + f"Device passthrough validation is not supported on '{platform_name}'" + ) + @staticmethod def _vendor_device_from_host_device( + platform_name: str, platform: Platform, + host_node: Optional[Node], device: "HostDeviceAddressSchema", ) -> Dict[str, str]: - platform_name = platform.type_name() if platform_name == HYPERV: hyperv_device = cast("HypervDeviceAddressSchema", device) instance_id = hyperv_device.instance_id @@ -171,19 +201,26 @@ def _vendor_device_from_host_device( "device_id": match.group("device_id").lower(), } - if platform_name != CLOUD_HYPERVISOR: + if platform_name not in [CLOUD_HYPERVISOR, OPENVMM]: raise LisaException( f"Device passthrough host device lookup is not supported on " - f"'{platform_name}'. Use a cloud-hypervisor or hyperv platform." + f"'{platform_name}'. Use a cloud-hypervisor, hyperv, or openvmm " + "platform." ) - cloud_hypervisor = cast("CloudHypervisorPlatform", platform) - libvirt_device = cast("LibvirtDeviceAddressSchema", device) + if host_node is None: + raise LisaException( + "No host node is available for passthrough device vendor lookup" + ) + if platform_name == CLOUD_HYPERVISOR: + cloud_hypervisor = cast("CloudHypervisorPlatform", platform) + host_node = cloud_hypervisor.host_node + pci_device = cast(Any, device) bdf = ( - f"{libvirt_device.domain}:{libvirt_device.bus}:" - f"{libvirt_device.slot}.{libvirt_device.function}" + f"{pci_device.domain}:{pci_device.bus}:" + f"{pci_device.slot}.{pci_device.function}" ).lower() - cat = cloud_hypervisor.host_node.tools[Cat] + cat = host_node.tools[Cat] vendor_raw = cat.read(f"/sys/bus/pci/devices/{bdf}/vendor", sudo=True).strip() device_raw = cat.read(f"/sys/bus/pci/devices/{bdf}/device", sudo=True).strip() # Normalize to 4-digit lowercase hex used by lspci identifiers. diff --git a/lisa/microsoft/testsuites/performance/networkperf_passthrough.py b/lisa/microsoft/testsuites/performance/networkperf_passthrough.py index aea36a7c7e..a5f2d54fa5 100644 --- a/lisa/microsoft/testsuites/performance/networkperf_passthrough.py +++ b/lisa/microsoft/testsuites/performance/networkperf_passthrough.py @@ -26,7 +26,7 @@ ) from lisa.environment import Environment, Node from lisa.operating_system import Windows -from lisa.sut_orchestrator import CLOUD_HYPERVISOR, HYPERV +from lisa.sut_orchestrator import CLOUD_HYPERVISOR, HYPERV, OPENVMM from lisa.testsuite import TestResult from lisa.tools import Dhclient, Kill, PowerShell, Sysctl from lisa.tools.ip import Ip @@ -47,7 +47,7 @@ from lisa.util.logger import get_logger from lisa.util.parallel import run_in_parallel -SUPPORTED_PASSTHROUGH_PLATFORMS = [CLOUD_HYPERVISOR, HYPERV] +SUPPORTED_PASSTHROUGH_PLATFORMS = [CLOUD_HYPERVISOR, HYPERV, OPENVMM] WINDOWS_NTTTCP_MAX_SERVER_THREADS = 64 WINDOWS_NTTTCP_MAX_MIXED_TCP_CONNECTIONS = 512 WINDOWS_NTTTCP_RECEIVER_WAIT_TIMEOUT = 90 @@ -790,6 +790,13 @@ def _refresh_passthrough_nic_address( return passthrough_nic_ip def _get_passthrough_node_context(self, node: Node) -> Any: + if node.type_name() == OPENVMM: + from lisa.sut_orchestrator.openvmm.context import ( + get_node_context as get_openvmm_node_context, + ) + + return get_openvmm_node_context(node) + try: from lisa.sut_orchestrator.libvirt.context import ( get_node_context as get_libvirt_node_context, diff --git a/lisa/node.py b/lisa/node.py index d0a5d6f71c..92085c8208 100644 --- a/lisa/node.py +++ b/lisa/node.py @@ -355,7 +355,9 @@ def execute_async( ) def cleanup(self) -> None: - for guest in self.guests: + guests = list(self.guests) + self._guests = [] + for guest in guests: try: guest.cleanup() except Exception: @@ -747,6 +749,7 @@ def set_connection_info( username: str = "root", password: str = "", private_key_file: str = "", + proxy_jump_boxes: Optional[List[schema.ConnectionInfo]] = None, ) -> None: if not address and not public_address: raise LisaException( @@ -774,7 +777,7 @@ def set_connection_info( password, private_key_file, ) - self._shell = SshShell(self._connection_info) + self._shell = SshShell(self._connection_info, proxy_jump_boxes) self.public_address = public_address self.public_port = public_port diff --git a/lisa/runners/lisa_runner.py b/lisa/runners/lisa_runner.py index 41c0334524..94e6943974 100644 --- a/lisa/runners/lisa_runner.py +++ b/lisa/runners/lisa_runner.py @@ -712,7 +712,9 @@ def _get_runnable_test_results( try: if result.check_environment( environment=tested_environment, - environment_platform_type=self.platform.type_name(), + environment_platform_type=self._get_test_platform_type( + result, self.platform.type_name() + ), save_reason=True, ) and ( not result.runtime_data.use_new_environment @@ -872,7 +874,9 @@ def _merge_test_requirements( test_req: TestCaseRequirement = test_result.runtime_data.requirement environment_requirement: Optional[EnvironmentSpace] = None - check_result = test_result.check_platform(platform_type) + check_result = test_result.check_platform( + self._get_test_platform_type(test_result, platform_type) + ) if not check_result.result: test_result.set_status(TestStatus.SKIPPED, check_result.reasons) continue @@ -926,6 +930,31 @@ def _create_guest_parent_requirement( return EnvironmentSpace(nodes=[schema.NodeSpace()]) + def _get_test_platform_type( + self, test_result: TestResult, platform_type: str + ) -> str: + if not getattr(self, "_guest_enabled", False): + return platform_type + + requirement = test_result.runtime_data.requirement + if ( + not requirement + or not requirement.platform_type + or len(requirement.platform_type.items) == 0 + ): + return platform_type + + platform_runbook = cast(schema.Platform, self.platform.runbook) + for guest_runbook in platform_runbook.guests: + guest_platform_type = getattr(guest_runbook, "type", "") + if ( + guest_platform_type + and test_result.check_platform(guest_platform_type).result + ): + return guest_platform_type + + return platform_type + def _merge_platform_requirement( self, environment_requirement: EnvironmentSpace, diff --git a/lisa/sut_orchestrator/baremetal/cluster/idrac.py b/lisa/sut_orchestrator/baremetal/cluster/idrac.py index a5c6212d0d..4252278ce3 100644 --- a/lisa/sut_orchestrator/baremetal/cluster/idrac.py +++ b/lisa/sut_orchestrator/baremetal/cluster/idrac.py @@ -6,11 +6,11 @@ import xml.etree.ElementTree as ETree from dataclasses import dataclass from pathlib import Path -from typing import Any, Optional, Type +from typing import Any, Callable, Optional, Type import redfish from assertpy import assert_that -from redfish.rest.v1 import JsonDecodingError +from redfish.rest.v1 import JsonDecodingError, RetriesExhaustedError from retry import retry from lisa import features, schema @@ -27,6 +27,14 @@ IDRAC_RESET_TIMEOUT = 120 IDRAC_REMOTE_SERVICES_TIMEOUT = 300 VIRTUAL_MEDIA_INSERTION_POLL_TIMEOUT = 30 +IDRAC_JOB_QUEUE_TIMEOUT = 300 +IDRAC_JOB_QUEUE_POLL_INTERVAL = 5 +IDRAC_LOGIN_RETRY_COUNT = 3 +IDRAC_LOGIN_RETRY_DELAY = 10 + +IDRAC_JOB_ALREADY_RUNNING_MESSAGE_ID = "IDRAC.2.8.RAC0679" +IDRAC_SERVER_ALREADY_POWERED_MESSAGE_ID = "IDRAC.2.8.PSU501" +IDRAC_ACCESS_DENIED_MESSAGE_ID = "Base.1.8.AccessDenied" @dataclass @@ -272,35 +280,46 @@ def get_server_screen_shot(self, file_type: str = "ServerScreenShot") -> str: return str(response.dict["ServerScreenShotFile"]) def reset(self, operation: str, force_run: bool = False) -> None: - if operation in self.state_dict.keys(): - expected_state = self.state_dict[operation] + expected_state = self.state_dict.get(operation) + if expected_state: if not force_run and self.get_power_state() == expected_state: self._log.debug(f"System is already in {expected_state} state.") return body = {"ResetType": operation} + url = "/redfish/v1/Systems/System.Embedded.1/Actions/ComputerSystem.Reset" + + def _post_reset() -> None: + response = self.redfish_instance.post(url, body=body) + self._wait_for_completion(response) # Try reset operation with iDRAC recovery on HTTP 500 errors try: - response = self.redfish_instance.post( - "/redfish/v1/Systems/System.Embedded.1/Actions/ComputerSystem.Reset", - body=body, + self._run_with_idrac_job_retry( + f"ComputerSystem.Reset '{operation}'", + _post_reset, ) - self._wait_for_completion(response) except LisaException as e: - if self._reset_if_idrac_error(str(e)): + error_msg = str(e) + if expected_state and self._is_already_in_expected_state_error( + error_msg, + expected_state, + ): + self._log.debug( + f"ComputerSystem.Reset '{operation}' reported that the server " + f"is already in '{expected_state}' state. Continuing." + ) + elif self._reset_if_idrac_error(error_msg): # iDRAC was reset, retry the operation once - url = ( - "/redfish/v1/Systems/System.Embedded.1/Actions/" - "ComputerSystem.Reset" + self._run_with_idrac_job_retry( + f"ComputerSystem.Reset '{operation}' after iDRAC reset", + _post_reset, ) - response = self.redfish_instance.post(url, body=body) - self._wait_for_completion(response) else: # Not a retriable iDRAC error - re-raise original exception raise - if operation in self.state_dict.keys(): + if expected_state: check_till_timeout( lambda: self.get_power_state() == expected_state, timeout_message=(f"wait for client into '{expected_state}' state"), @@ -319,13 +338,17 @@ def get_power_state(self) -> str: status = getattr(response, "status", "unknown") response_keys = list(response_dict.keys()) - raise LisaException( + error_msg = ( "Failed to get iDRAC power state because the System endpoint response " f"did not include a non-null PowerState. " f"status={status}, keys={response_keys}. " "Verify the iDRAC System endpoint and Lifecycle Controller health." ) + if self._reset_if_idrac_error(error_msg): + raise LisaException(f"{error_msg} iDRAC reset completed; retrying.") + raise LisaException(error_msg) + @retry(tries=IDRAC_LOGIN_RETRY_COUNT, delay=IDRAC_LOGIN_RETRY_DELAY) # type: ignore def login(self) -> None: self.redfish_instance = redfish.redfish_client( base_url="https://" + self.idrac_runbook.address, @@ -362,6 +385,82 @@ def _wait_for_completion(self, response: Any, timeout: int = 600) -> None: error_msg += f", details: {response.dict}" raise LisaException(error_msg) + def _run_with_idrac_job_retry( + self, + operation_description: str, + operation: Callable[[], None], + ) -> None: + access_denied_recovered = False + + def _try_operation() -> bool: + nonlocal access_denied_recovered + try: + operation() + return True + except LisaException as e: + if self._is_access_denied_error(str(e)): + if access_denied_recovered: + raise + access_denied_recovered = True + self._refresh_session_after_access_denied(operation_description) + return False + if not self._is_idrac_job_already_running_error(str(e)): + raise + self._log.debug( + f"{operation_description} is waiting on an existing iDRAC " + f"job to complete: {e}" + ) + return False + + check_till_timeout( + _try_operation, + timeout_message=( + f"wait for existing iDRAC job before {operation_description}" + ), + timeout=IDRAC_JOB_QUEUE_TIMEOUT, + interval=IDRAC_JOB_QUEUE_POLL_INTERVAL, + ) + + @staticmethod + def _is_idrac_job_already_running_error(error_str: str) -> bool: + normalized_error = error_str.lower() + return ( + IDRAC_JOB_ALREADY_RUNNING_MESSAGE_ID.lower() in normalized_error + or "a job operation is already running" in normalized_error + ) + + @staticmethod + def _is_already_in_expected_state_error( + error_str: str, + expected_state: str, + ) -> bool: + normalized_error = error_str.lower() + expected_state = expected_state.lower() + if expected_state == "on" and ( + IDRAC_SERVER_ALREADY_POWERED_MESSAGE_ID.lower() in normalized_error + ): + return True + return f"server is already powered {expected_state}" in normalized_error + + @staticmethod + def _is_access_denied_error(error_str: str) -> bool: + normalized_error = error_str.lower() + return ( + IDRAC_ACCESS_DENIED_MESSAGE_ID.lower() in normalized_error + or "accessdenied" in normalized_error + or "access denied" in normalized_error + or "credentials included with this request are missing or invalid" + in normalized_error + ) + + def _refresh_session_after_access_denied(self, operation_description: str) -> None: + self._log.debug( + f"{operation_description} received iDRAC access denied. " + "Refreshing the Redfish session before retrying." + ) + self.logout() + self.login() + def _eject_vm(self, device_name: str) -> None: """Eject virtual media from specified device (CD or RemovableDisk).""" response = self.redfish_instance.post( @@ -440,6 +539,8 @@ def _reset_if_idrac_error(self, error_str: str) -> bool: or "idrac.2.8.rac0508" in normalized_error or "base.1.12.internalerror" in normalized_error or "provider is not ready" in normalized_error + or "status=500" in normalized_error + or "status 500" in normalized_error ) if is_idrac_internal_error: @@ -626,6 +727,10 @@ def _check_media_inserted() -> bool: except LisaException as e: error_msg = str(e) + if self._is_access_denied_error(error_msg): + self._refresh_session_after_access_denied("VirtualMedia.InsertMedia") + raise + # Check for HTTP 500 internal server errors and reset if needed if self._reset_if_idrac_error(error_msg): # Re-raise to trigger retry @@ -699,24 +804,29 @@ def _change_boot_order_once(self, boot_from: str) -> None: ) try: - response = self.redfish_instance.post(url, body=body) - self._log.debug("Waiting for boot order override task to complete...") - self._wait_for_completion(response) + self._run_with_idrac_job_retry( + "boot order override", + lambda: self._post_boot_order_override(url, body), + ) except LisaException as e: if self._reset_if_idrac_error(str(e)): self._log.debug( "Retrying boot order override after iDRAC reset recovery..." ) - response = self.redfish_instance.post(url, body=body) - self._log.debug( - "Waiting for boot order override task to complete after retry..." + self._run_with_idrac_job_retry( + "boot order override after iDRAC reset", + lambda: self._post_boot_order_override(url, body), ) - self._wait_for_completion(response) else: raise self._log.debug(f"Updating boot source to {boot_from} completed") + def _post_boot_order_override(self, url: str, body: Any) -> None: + response = self.redfish_instance.post(url, body=body) + self._log.debug("Waiting for boot order override task to complete...") + self._wait_for_completion(response) + def _enable_serial_console(self) -> None: # iDRAC may return 503 Service Unavailable transiently (e.g. just after # a reset or during initialisation). Retry for up to IDRAC_RESET_TIMEOUT @@ -762,6 +872,12 @@ def _try_enable() -> bool: f"{e}. Retrying..." ) return False + except RetriesExhaustedError as e: + self._log.info( + "iDRAC serial console setup could not reach Redfish: " + f"{e}. Retrying..." + ) + return False finally: self.logout() @@ -769,7 +885,7 @@ def _try_enable() -> bool: _try_enable, timeout_message=( "iDRAC Attributes endpoint unavailable after retries " - "(repeated 503 / invalid JSON responses)" + "(repeated Redfish connection failures or invalid JSON responses)" ), timeout=IDRAC_RESET_TIMEOUT, interval=10, diff --git a/lisa/sut_orchestrator/openvmm/context.py b/lisa/sut_orchestrator/openvmm/context.py index e494ef6a93..2a1d7b2b07 100644 --- a/lisa/sut_orchestrator/openvmm/context.py +++ b/lisa/sut_orchestrator/openvmm/context.py @@ -2,10 +2,11 @@ # Licensed under the MIT license. from dataclasses import dataclass, field -from threading import Lock +from threading import Lock, RLock from typing import Any, Dict, List, Optional from lisa.node import Node +from lisa.sut_orchestrator.util.schema import HostDevicePoolType from lisa.util import LisaException from .schema import OpenVmmNetworkSchema @@ -23,6 +24,22 @@ def _new_str_dict() -> Dict[str, str]: return {} +@dataclass +class DeviceAddressSchema: + domain: str = "0000" + bus: str = "" + slot: str = "" + function: str = "0" + original_driver: str = "" + + +@dataclass +class DevicePassthroughContext: + pool_type: HostDevicePoolType = HostDevicePoolType.PCI_NIC + device_list: List[DeviceAddressSchema] = field(default_factory=list) + managed: str = "" + + @dataclass class NodeContext: vm_name: str = "" @@ -42,6 +59,7 @@ class NodeContext: forwarded_port: int = 0 forwarding_enabled: bool = False forwarding_interface: str = "" + forwarding_rules_added: List[str] = field(default_factory=_new_str_list) tap_created: bool = False tap_bridge_created: bool = False tap_bridge_netfilter_disabled: bool = False @@ -51,6 +69,7 @@ class NodeContext: effective_network: Optional[OpenVmmNetworkSchema] = None process_id: str = "" command_line: str = "" + passthrough_devices: List[DevicePassthroughContext] = field(default_factory=list) @dataclass @@ -61,8 +80,12 @@ class OpenVmmHostContext: default_factory=_new_str_dict ) active_bridge_netfilter_count: int = 0 + ssh_forwarding_lock: RLock = field(default_factory=RLock) artifact_copy_lock: Lock = field(default_factory=Lock) artifact_cache: Dict[str, str] = field(default_factory=_new_str_dict) + device_pool_lock: Lock = field(default_factory=Lock) + device_pool: Optional[Any] = None + device_pool_config_key: str = "" def get_node_context(node: Node) -> NodeContext: diff --git a/lisa/sut_orchestrator/openvmm/installer.py b/lisa/sut_orchestrator/openvmm/installer.py index b56b86f1a6..b98f6d6f4d 100644 --- a/lisa/sut_orchestrator/openvmm/installer.py +++ b/lisa/sut_orchestrator/openvmm/installer.py @@ -7,7 +7,7 @@ from lisa import schema from lisa.node import Node from lisa.operating_system import CBLMariner, Linux, Ubuntu -from lisa.tools import Cargo, Git, Ln +from lisa.tools import Cargo, Git, Ln, Rm from lisa.tools.openvmm import is_missing_command_output from lisa.util import LisaException, UnsupportedDistroException, subclasses from lisa.util.logger import Logger @@ -132,8 +132,10 @@ def install(self) -> str: ), ) self._node.execute( - f"{shlex.quote(rustup_bin)} component add rust-src --toolchain " - f"{shlex.quote(toolchain)}", + cargo.wrap_with_rustup_lock( + f"{shlex.quote(rustup_bin)} component add rust-src --toolchain " + f"{shlex.quote(toolchain)}" + ), shell=True, expected_exit_code=0, expected_exit_code_failure_message=( @@ -142,12 +144,18 @@ def install(self) -> str: ) git = self._node.tools[Git] + source_dir_name = "openvmm" + code_path = self._node.get_pure_path(str(self._node.working_path)).joinpath( + source_dir_name + ) + self._log.debug(f"Refreshing OpenVMM source checkout at '{code_path}'") + self._node.tools[Rm].remove_directory(self._node.get_str_path(code_path)) code_path = git.clone( url=runbook.repo, cwd=self._node.working_path, + dir_name=source_dir_name, ref=runbook.ref, auth_token=runbook.auth_token, - fail_on_exists=False, ) openvmm_commit = git.get_current_commit_hash(code_path).strip() self._log.info( @@ -155,9 +163,9 @@ def install(self) -> str: f"requested ref: {runbook.ref or ''}, HEAD: {openvmm_commit}" ) - cargo_command = shlex.quote(cargo.command) + cargo_command = self._resolve_cargo_command(cargo.command) cargo_bin_dir = self._node.get_str_path( - self._node.get_pure_path(cargo.command).parent + self._node.get_pure_path(cargo_command).parent ) cargo_env = { "OPENSSL_NO_VENDOR": "1", @@ -166,7 +174,7 @@ def install(self) -> str: "RUSTDOC": f"{cargo_bin_dir}/rustdoc", } restore_packages_cmd = ( - f"{cargo_command} xflowey restore-packages --no-compat-igvm" + f"{shlex.quote(cargo_command)} xflowey restore-packages --no-compat-igvm" ) restore_result = self._node.execute( restore_packages_cmd, @@ -190,8 +198,10 @@ def install(self) -> str: ), ) self._node.execute( - f"{shlex.quote(rustup_bin)} component add rust-src --toolchain " - f"{shlex.quote(toolchain)}", + cargo.wrap_with_rustup_lock( + f"{shlex.quote(rustup_bin)} component add rust-src --toolchain " + f"{shlex.quote(toolchain)}" + ), shell=True, expected_exit_code=0, expected_exit_code_failure_message=( @@ -210,7 +220,7 @@ def install(self) -> str: ), ) - build_cmd = f"{cargo_command} build --release" + build_cmd = f"{shlex.quote(cargo_command)} build --release" if runbook.features: feature_args = ",".join(runbook.features) build_cmd = f"{build_cmd} --features {shlex.quote(feature_args)}" @@ -262,3 +272,17 @@ def install(self) -> str: ) self._create_symlink_to_usr_bin(runbook.install_path) return self.get_version(runbook.install_path) + + def _resolve_cargo_command(self, cargo_command: str) -> str: + if cargo_command and self._node.get_pure_path(cargo_command).is_absolute(): + return cargo_command + + result = self._node.execute( + f"command -v {shlex.quote(cargo_command or 'cargo')}", + shell=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + "failed to resolve cargo command path for OpenVMM build" + ), + ) + return result.stdout.strip() diff --git a/lisa/sut_orchestrator/openvmm/node.py b/lisa/sut_orchestrator/openvmm/node.py index aca4ea8b04..57967752aa 100644 --- a/lisa/sut_orchestrator/openvmm/node.py +++ b/lisa/sut_orchestrator/openvmm/node.py @@ -13,18 +13,19 @@ import uuid from abc import ABC, abstractmethod from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath -from typing import Any, Dict, List, Optional, Type, cast +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Protocol, Type, cast import yaml from lisa import constants, schema, search_space from lisa.feature import Features from lisa.node import Node, RemoteNode -from lisa.tools import Dnsmasq, Ip, Kill, Mkdir, Modprobe, OpenVmm, Rm +from lisa.tools import Dnsmasq, Echo, Ip, Kill, Ls, Lspci, Mkdir, Modprobe, OpenVmm, Rm from lisa.tools.openvmm import OpenVmmLaunchConfig from lisa.util import ( LisaException, LisaTimeoutException, + ResourceAwaitableException, check_till_timeout, create_timer, get_public_key_data, @@ -32,10 +33,20 @@ from lisa.util.logger import Logger from lisa.util.shell import wait_tcp_port_ready +if TYPE_CHECKING: + from lisa.sut_orchestrator.libvirt.libvirt_device_pool import LibvirtDevicePool + from .. import OPENVMM -from .context import NodeContext, get_host_context, get_node_context +from .context import ( + DeviceAddressSchema, + DevicePassthroughContext, + NodeContext, + get_host_context, + get_node_context, +) from .schema import ( OPENVMM_ADDRESS_MODE_STATIC, + OPENVMM_CONNECTION_MODE_HOST_PROXY, OPENVMM_NETWORK_MODE_TAP, OPENVMM_NETWORK_MODE_USER, OpenVmmGuestNodeSchema, @@ -60,6 +71,13 @@ ] +class PciAddressLike(Protocol): + domain: str + bus: str + slot: str + function: str + + def _get_tap_host_interface_name(network: OpenVmmNetworkSchema) -> str: return network.bridge_name or network.tap_name @@ -68,6 +86,16 @@ def _is_raw_disk_image(disk_img_path: str) -> bool: return Path(disk_img_path).suffix.lower() == ".raw" +def _get_pci_address_str(device: PciAddressLike) -> str: + if not device.domain or not device.bus or not device.slot or not device.function: + raise LisaException( + "OpenVMM passthrough device has an incomplete PCI address. " + f"domain='{device.domain}', bus='{device.bus}', " + f"slot='{device.slot}', function='{device.function}'." + ) + return f"{device.domain}:{device.bus}:{device.slot}.{device.function}" + + def _countspace_to_int(value: search_space.CountSpace) -> int: chosen = search_space.choose_value_countspace(value, value) if not isinstance(chosen, int): @@ -390,6 +418,277 @@ def _get_node_network( return cast(OpenVmmGuestNodeSchema, node.runbook).network + def _get_device_passthrough_args(self, node_context: NodeContext) -> List[str]: + args: List[str] = [] + devices = [ + device + for passthrough_context in node_context.passthrough_devices + for device in passthrough_context.device_list + ] + if not devices: + return args + + root_complex_name = "lisa_vfio_rc0" + args.extend(["--pcie-root-complex", root_complex_name]) + for port_index, device in enumerate(devices): + port_name = f"lisa_vfio_rp{port_index}" + args.extend(["--pcie-root-port", f"{root_complex_name}:{port_name}"]) + args.extend( + [ + "--vfio", + f"host={_get_pci_address_str(device)},port={port_name}", + ] + ) + return args + + def _get_device_pool_config_key(self, runbook: OpenVmmGuestNodeSchema) -> str: + return repr(runbook.device_pools or []) + + def _get_or_create_device_pool( + self, + runbook: OpenVmmGuestNodeSchema, + ) -> LibvirtDevicePool: + host_context = get_host_context(self.host_node) + config_key = self._get_device_pool_config_key(runbook) + if host_context.device_pool is None: + from lisa.sut_orchestrator.libvirt.libvirt_device_pool import ( + LibvirtDevicePool, + ) + + if not runbook.device_pools: + raise LisaException( + "OpenVMM device_passthrough requires device_pools on at " + "least one OpenVMM guest runbook for the baremetal host." + ) + device_pool = LibvirtDevicePool(self.host_node, cast(Any, None)) + device_pool.configure_device_passthrough_pool(runbook.device_pools) + host_context.device_pool = device_pool + host_context.device_pool_config_key = config_key + elif runbook.device_pools and host_context.device_pool_config_key != config_key: + raise LisaException( + "OpenVMM guests on the same baremetal host must use the same " + "device_pools configuration. Define the shared host device pool " + "consistently for each guest that requests passthrough devices." + ) + + return cast("LibvirtDevicePool", host_context.device_pool) + + def set_device_passthrough_node_context( + self, + node_context: NodeContext, + runbook: OpenVmmGuestNodeSchema, + ) -> None: + if not runbook.device_passthrough: + return + + host_context = get_host_context(self.host_node) + with host_context.device_pool_lock: + device_pool = self._get_or_create_device_pool(runbook) + try: + for config in runbook.device_passthrough: + if config.count <= 0: + raise LisaException( + "OpenVMM device_passthrough count must be greater " + f"than 0 for pool type '{config.pool_type.value}'." + ) + devices = device_pool.request_devices( + config.pool_type, + config.count, + ) + device_context = DevicePassthroughContext( + managed=config.managed, + pool_type=config.pool_type, + device_list=[ + DeviceAddressSchema( + domain=device.domain, + bus=device.bus, + slot=device.slot, + function=device.function, + original_driver=self._get_pci_device_driver(device), + ) + for device in devices + ], + ) + node_context.passthrough_devices.append(device_context) + + self._bind_device_passthrough_to_vfio(node_context) + except (LisaException, ResourceAwaitableException): + if node_context.passthrough_devices: + try: + self._restore_device_passthrough_drivers(node_context) + except Exception as restore_error: + self._log.debug( + "failed to restore OpenVMM passthrough drivers after " + f"allocation failure: {restore_error}" + ) + device_pool.release_devices(cast(Any, node_context)) + node_context.passthrough_devices.clear() + raise + + def _bind_device_passthrough_to_vfio(self, node_context: NodeContext) -> None: + managed_contexts = [ + context + for context in node_context.passthrough_devices + if context.managed.lower() != "no" + ] + if not managed_contexts: + return + + self.host_node.tools[Modprobe].load("vfio-pci") + for passthrough_context in managed_contexts: + for device in passthrough_context.device_list: + self._bind_pci_device_to_vfio(device) + + def _bind_pci_device_to_vfio(self, device: DeviceAddressSchema) -> None: + bdf = _get_pci_address_str(device) + current_driver = self._get_pci_device_driver(device) + if not device.original_driver: + device.original_driver = current_driver + if current_driver == "vfio-pci": + self._verify_vfio_group_device_exists(device) + return + + device_path = f"/sys/bus/pci/devices/{bdf}" + driver_override_path = f"{device_path}/driver_override" + if not self.host_node.tools[Ls].path_exists(driver_override_path, sudo=True): + raise LisaException( + "OpenVMM passthrough requires PCI driver_override support, but " + f"'{driver_override_path}' does not exist for device '{bdf}'." + ) + + self._log.debug( + f"Binding OpenVMM passthrough device '{bdf}' from driver " + f"'{current_driver or ''}' to vfio-pci" + ) + self._write_sysfs_value("vfio-pci", driver_override_path) + try: + if current_driver: + self._write_sysfs_value(bdf, f"{device_path}/driver/unbind") + self._write_sysfs_value(bdf, "/sys/bus/pci/drivers/vfio-pci/bind") + finally: + self._write_sysfs_value("", driver_override_path, ignore_error=True) + + rebound_driver = self._get_pci_device_driver(device) + if rebound_driver != "vfio-pci": + raise LisaException( + f"failed to bind OpenVMM passthrough device '{bdf}' to vfio-pci. " + f"Current driver: '{rebound_driver or ''}'." + ) + self._verify_vfio_group_device_exists(device) + + def _restore_device_passthrough_drivers(self, node_context: NodeContext) -> None: + for passthrough_context in node_context.passthrough_devices: + if passthrough_context.managed.lower() == "no": + continue + for device in passthrough_context.device_list: + self._restore_pci_device_driver(device) + + def _restore_pci_device_driver(self, device: DeviceAddressSchema) -> None: + original_driver = device.original_driver + if not original_driver or original_driver == "vfio-pci": + return + + bdf = _get_pci_address_str(device) + current_driver = self._get_pci_device_driver(device) + if current_driver == original_driver: + device.original_driver = "" + return + + driver_override_path = f"/sys/bus/pci/devices/{bdf}/driver_override" + if self.host_node.tools[Ls].path_exists(driver_override_path, sudo=True): + self._write_sysfs_value(original_driver, driver_override_path) + + original_bind_path = f"/sys/bus/pci/drivers/{original_driver}/bind" + if not self.host_node.tools[Ls].path_exists(original_bind_path, sudo=True): + self.host_node.tools[Modprobe].load(original_driver) + + self._log.debug( + f"Restoring OpenVMM passthrough device '{bdf}' to driver " + f"'{original_driver}'" + ) + try: + if current_driver: + self._write_sysfs_value( + bdf, f"/sys/bus/pci/devices/{bdf}/driver/unbind" + ) + self._write_sysfs_value(bdf, original_bind_path) + finally: + if self.host_node.tools[Ls].path_exists(driver_override_path, sudo=True): + self._write_sysfs_value("", driver_override_path, ignore_error=True) + + device.original_driver = "" + + def _get_pci_device_driver(self, device: PciAddressLike) -> str: + bdf = _get_pci_address_str(device) + return self.host_node.tools[Lspci].get_used_module(bdf) + + def _verify_vfio_group_device_exists(self, device: DeviceAddressSchema) -> None: + bdf = _get_pci_address_str(device) + group_path = f"/sys/bus/pci/devices/{bdf}/iommu_group" + result = self.host_node.execute( + f"basename $(readlink -f {shlex.quote(group_path)})", + shell=True, + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + f"failed to resolve IOMMU group for OpenVMM passthrough device " + f"'{bdf}'" + ), + ) + group_id = result.stdout.strip() + vfio_group_path = f"/dev/vfio/{group_id}" + if not self.host_node.tools[Ls].path_exists(vfio_group_path, sudo=True): + raise LisaException( + f"OpenVMM passthrough device '{bdf}' is bound to vfio-pci, but " + f"'{vfio_group_path}' was not created. Verify host IOMMU/VFIO " + "configuration for this device." + ) + + def _write_sysfs_value( + self, + value: str, + path: str, + ignore_error: bool = False, + ) -> None: + try: + self.host_node.tools[Echo].write_to_file( + value, + self.host_node.get_pure_path(path), + sudo=True, + ignore_error=ignore_error, + ) + except AssertionError as identifier: + raise LisaException( + f"failed to write '{value}' to '{path}' for OpenVMM " + f"passthrough: {identifier}" + ) from identifier + + def release_device_passthrough(self, node: "OpenVmmGuestNode") -> None: + node_context = get_node_context(node) + if not node_context.passthrough_devices: + return + + host_context = get_host_context(self.host_node) + if host_context.device_pool is None: + node_context.passthrough_devices.clear() + return + + with host_context.device_pool_lock: + restore_error: Optional[Exception] = None + try: + self._restore_device_passthrough_drivers(node_context) + except Exception as identifier: + restore_error = identifier + finally: + host_context.device_pool.release_devices(node_context) + node_context.passthrough_devices.clear() + + if restore_error: + raise LisaException( + "failed to restore OpenVMM passthrough device drivers: " + f"{restore_error}" + ) + def launch(self, node: "OpenVmmGuestNode", log: Logger) -> None: runbook = cast(OpenVmmGuestNodeSchema, node.runbook) node_context = get_node_context(node) @@ -410,7 +709,8 @@ def launch(self, node: "OpenVmmGuestNode", log: Logger) -> None: network_cidr=network.consomme_cidr, serial_mode=runbook.serial.mode, serial_path=node_context.console_log_file_path, - extra_args=runbook.extra_args, + extra_args=self._get_device_passthrough_args(node_context) + + runbook.extra_args, stdout_path=node_context.launcher_log_file_path, stderr_path=node_context.launcher_stderr_log_file_path, ) @@ -580,27 +880,40 @@ def _prepare_tap_network( ) ip_tool.up(bridge_name) - if not ip_tool.nic_exists(tap_name): - whoami_result = host.execute( - "whoami", + whoami_result = host.execute( + "whoami", + shell=True, + no_info_log=True, + no_error_log=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + "failed to determine the host username with 'whoami' before " + f"creating OpenVMM tap interface {tap_name}. Verify that " + "'whoami' is available and working on the host." + ), + ) + username = whoami_result.stdout.strip() + if not username: + raise LisaException( + "failed to determine the host username before creating " + f"OpenVMM tap interface {tap_name}: 'whoami' returned an " + "empty username. Verify that the host shell environment is " + "configured correctly and that 'whoami' returns a valid user." + ) + + if ip_tool.nic_exists(tap_name): + host.execute( + f"ip link delete {shlex.quote(tap_name)}", shell=True, - no_info_log=True, - no_error_log=True, + sudo=True, expected_exit_code=0, expected_exit_code_failure_message=( - "failed to determine the host username with 'whoami' before " - f"creating OpenVMM tap interface {tap_name}. Verify that " - "'whoami' is available and working on the host." + "failed to delete stale OpenVMM tap interface " + f"{tap_name} before recreating it" ), ) - username = whoami_result.stdout.strip() - if not username: - raise LisaException( - "failed to determine the host username before creating " - f"OpenVMM tap interface {tap_name}: 'whoami' returned an " - "empty username. Verify that the host shell environment is " - "configured correctly and that 'whoami' returns a valid user." - ) + + if not ip_tool.nic_exists(tap_name): host.execute( ( f"ip tuntap add {shlex.quote(tap_name)} mode tap " @@ -832,7 +1145,16 @@ def configure_connection(self, node: RemoteNode, log: Logger) -> None: port = network.ssh_port public_port = port - if network.forward_ssh_port: + proxy_jump_boxes = None + if network.connection_mode == OPENVMM_CONNECTION_MODE_HOST_PROXY: + self._wait_for_guest_ssh_from_host( + node_context, guest_address, network.ssh_port, log + ) + if self.host_node.is_remote: + proxy_jump_boxes = [cast(RemoteNode, self.host_node)._connection_info] + public_address = guest_address + public_port = port + elif network.forward_ssh_port: self._enable_ssh_forwarding(node_context, guest_address, network) public_address = ( network.connection_address or self._get_host_public_address() @@ -847,34 +1169,74 @@ def configure_connection(self, node: RemoteNode, log: Logger) -> None: private_key_file=runbook.private_key_file, port=port, public_port=public_port, + use_public_address=not proxy_jump_boxes, + proxy_jump_boxes=proxy_jump_boxes, ) - try: - is_ready, error_code = wait_tcp_port_ready( - public_address, - public_port, - log=log, - timeout=OPENVMM_CONNECTION_TIMEOUT, - ) - except LisaException as identifier: - raise LisaException( - "OpenVMM guest SSH port readiness check failed for " - f"{public_address}:{public_port}. " - "Verify the guest is running, port forwarding or network " - "configuration is correct, the SSH service is listening on the " - "expected port, and review the OpenVMM guest and host logs for " - "startup or networking errors. " - f"{self._get_openvmm_failure_context(node_context, network)}" - ) from identifier - if not is_ready: + if not proxy_jump_boxes: + try: + is_ready, error_code = wait_tcp_port_ready( + public_address, + public_port, + log=log, + timeout=OPENVMM_CONNECTION_TIMEOUT, + ) + except LisaException as identifier: + raise LisaException( + "OpenVMM guest SSH port readiness check failed for " + f"{public_address}:{public_port}. " + "Verify the guest is running, port forwarding or network " + "configuration is correct, the SSH service is listening on the " + "expected port, and review the OpenVMM guest and host logs for " + "startup or networking errors. " + f"{self._get_openvmm_failure_context(node_context, network)}" + ) from identifier + if not is_ready: + raise LisaException( + "OpenVMM guest SSH port did not become reachable at " + f"{public_address}:{public_port} " + f"(error code: {error_code}). Verify the guest is running, " + "port forwarding or network configuration is correct, the SSH " + "service is listening on the expected port, and review the " + "OpenVMM guest and host logs for startup or networking errors. " + f"{self._get_openvmm_failure_context(node_context, network)}" + ) + + def _wait_for_guest_ssh_from_host( + self, + node_context: Any, + guest_address: str, + guest_port: int, + log: Logger, + timeout: int = OPENVMM_CONNECTION_TIMEOUT, + ) -> None: + probe_command = ( + "while true; do " + "timeout 1 bash -c " + f"{shlex.quote(f'/dev/null 2>&1 && exit 0; " + "sleep 1; " + "done" + ) + command = f"timeout {timeout} bash -c " f"{shlex.quote(probe_command)}" + result = self.host_node.execute( + command, + shell=True, + sudo=False, + no_info_log=True, + no_error_log=True, + expected_exit_code=None, + timeout=timeout + 10, + ) + if result.exit_code != 0: raise LisaException( - "OpenVMM guest SSH port did not become reachable at " - f"{public_address}:{public_port} " - f"(error code: {error_code}). Verify the guest is running, " - "port forwarding or network configuration is correct, the SSH " - "service is listening on the expected port, and review the " - "OpenVMM guest and host logs for startup or networking errors. " - f"{self._get_openvmm_failure_context(node_context, network)}" + "OpenVMM guest SSH port did not become reachable from the host at " + f"{guest_address}:{guest_port}. Verify guest networking and sshd. " + f"{self._get_openvmm_failure_context(node_context, None)}" ) + log.debug( + "confirmed OpenVMM guest SSH is reachable from host at " + f"{guest_address}:{guest_port}" + ) def _resolve_guest_address( self, @@ -1270,128 +1632,102 @@ def _get_host_public_address(self) -> str: def _enable_ssh_forwarding( self, - node_context: Any, + node_context: NodeContext, guest_address: str, network: OpenVmmNetworkSchema, ) -> None: host_context = get_host_context(self.host_node) - forwarding_interface, _ = self.host_node.tools[Ip].get_default_route_info() - host_interface = _get_tap_host_interface_name(network) - host_network = ipaddress.ip_interface(network.tap_host_cidr).network - guest_address = shlex.quote(guest_address) - guest_port = network.ssh_port - forwarded_port = network.forwarded_port - - if host_context.active_forwarding_count == 0: - ip_forward_result = self.host_node.execute( - "sysctl -n net.ipv4.ip_forward", - shell=True, - sudo=True, - no_info_log=True, - no_error_log=True, - expected_exit_code=0, - expected_exit_code_failure_message=( - "failed to read current host ip_forward state for OpenVMM " - "SSH forwarding" - ), - ) - original_ip_forward_value = ip_forward_result.stdout.strip() - if original_ip_forward_value not in ["0", "1"]: - raise LisaException( - "failed to parse current host ip_forward state for " - "OpenVMM SSH forwarding. " - f"stdout: {ip_forward_result.stdout.strip() or ''}. " - f"stderr: {ip_forward_result.stderr.strip() or ''}" + with host_context.ssh_forwarding_lock: + forwarding_interface, _ = self.host_node.tools[Ip].get_default_route_info() + host_interface = _get_tap_host_interface_name(network) + host_network = ipaddress.ip_interface(network.tap_host_cidr).network + guest_address = shlex.quote(guest_address) + guest_port = network.ssh_port + forwarded_port = network.forwarded_port + + if host_context.active_forwarding_count == 0: + ip_forward_result = self.host_node.execute( + "sysctl -n net.ipv4.ip_forward", + shell=True, + sudo=True, + no_info_log=True, + no_error_log=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + "failed to read current host ip_forward state for OpenVMM " + "SSH forwarding" + ), ) - host_context.original_ip_forward_value = original_ip_forward_value + original_ip_forward_value = ip_forward_result.stdout.strip() + if original_ip_forward_value not in ["0", "1"]: + raise LisaException( + "failed to parse current host ip_forward state for " + "OpenVMM SSH forwarding. " + f"stdout: {ip_forward_result.stdout.strip() or ''}. " + f"stderr: {ip_forward_result.stderr.strip() or ''}" + ) + host_context.original_ip_forward_value = original_ip_forward_value - host_context.active_forwarding_count += 1 - node_context.forwarding_interface = forwarding_interface - node_context.forwarded_port = forwarded_port - node_context.forwarding_enabled = True + host_context.active_forwarding_count += 1 + node_context.forwarding_interface = forwarding_interface + node_context.forwarded_port = forwarded_port + node_context.forwarding_enabled = True - commands = [ - "sysctl -w net.ipv4.ip_forward=1", - ( - "iptables -C FORWARD -i " - f"{shlex.quote(host_interface)} -o {shlex.quote(forwarding_interface)} " - "-j ACCEPT " - "|| " - "iptables -I FORWARD -i " - f"{shlex.quote(host_interface)} -o {shlex.quote(forwarding_interface)} " - "-j ACCEPT" - ), - ( - "iptables -C FORWARD -i " - f"{shlex.quote(host_interface)} ! -o " - f"{shlex.quote(forwarding_interface)} " - "-j ACCEPT " - "|| " - "iptables -I FORWARD -i " - f"{shlex.quote(host_interface)} ! -o " - f"{shlex.quote(forwarding_interface)} " - "-j ACCEPT" - ), - ( - "iptables -C FORWARD -i " - f"{shlex.quote(forwarding_interface)} -o {shlex.quote(host_interface)} " - "-m state --state RELATED,ESTABLISHED -j ACCEPT " - "|| " - "iptables -I FORWARD -i " - f"{shlex.quote(forwarding_interface)} -o {shlex.quote(host_interface)} " - "-m state --state RELATED,ESTABLISHED -j ACCEPT" - ), - ( - "iptables -C FORWARD ! -i " - f"{shlex.quote(forwarding_interface)} -o {shlex.quote(host_interface)} " - "-m state --state RELATED,ESTABLISHED -j ACCEPT " - "|| " - "iptables -I FORWARD ! -i " - f"{shlex.quote(forwarding_interface)} -o {shlex.quote(host_interface)} " - "-m state --state RELATED,ESTABLISHED -j ACCEPT" - ), - ( - "iptables -C FORWARD -i " - f"{shlex.quote(forwarding_interface)} -o {shlex.quote(host_interface)} " - f"-p tcp -d {guest_address} --dport {guest_port} -j ACCEPT " - "|| " - "iptables -I FORWARD -i " - f"{shlex.quote(forwarding_interface)} -o {shlex.quote(host_interface)} " - f"-p tcp -d {guest_address} --dport {guest_port} -j ACCEPT" - ), - ( - "iptables -t nat -C POSTROUTING -s " - f"{shlex.quote(str(host_network))} " - f"-o {shlex.quote(forwarding_interface)} " - "-j MASQUERADE || " - "iptables -t nat -I POSTROUTING -s " - f"{shlex.quote(str(host_network))} " - f"-o {shlex.quote(forwarding_interface)} " - "-j MASQUERADE" - ), - ( - "iptables -t nat -C PREROUTING -p tcp --dport " - f"{forwarded_port} -j DNAT --to-destination " - f"{guest_address}:{guest_port} " - "|| " - "iptables -t nat -I PREROUTING -p tcp --dport " - f"{forwarded_port} -j DNAT --to-destination " - f"{guest_address}:{guest_port}" - ), - ( - "iptables -t nat -C OUTPUT -p tcp --dport " - f"{forwarded_port} -j DNAT --to-destination " - f"{guest_address}:{guest_port} " - "|| " - "iptables -t nat -I OUTPUT -p tcp --dport " - f"{forwarded_port} -j DNAT --to-destination " - f"{guest_address}:{guest_port}" - ), - ] - try: - for command in commands: + rules = [ + ( + "", + "FORWARD -i " + f"{shlex.quote(host_interface)} -o " + f"{shlex.quote(forwarding_interface)} -j ACCEPT", + ), + ( + "", + "FORWARD -i " + f"{shlex.quote(host_interface)} ! -o " + f"{shlex.quote(forwarding_interface)} -j ACCEPT", + ), + ( + "", + "FORWARD -i " + f"{shlex.quote(forwarding_interface)} -o " + f"{shlex.quote(host_interface)} -m state --state " + "RELATED,ESTABLISHED -j ACCEPT", + ), + ( + "", + "FORWARD ! -i " + f"{shlex.quote(forwarding_interface)} -o " + f"{shlex.quote(host_interface)} -m state --state " + "RELATED,ESTABLISHED -j ACCEPT", + ), + ( + "", + "FORWARD ! -i " + f"{shlex.quote(host_interface)} -o " + f"{shlex.quote(host_interface)} " + f"-p tcp -d {guest_address} --dport {guest_port} -j ACCEPT", + ), + ( + "-t nat", + "POSTROUTING -s " + f"{shlex.quote(str(host_network))} -o " + f"{shlex.quote(forwarding_interface)} -j MASQUERADE", + ), + ( + "-t nat", + f"PREROUTING -p tcp --dport {forwarded_port} " + f"-j DNAT --to-destination {guest_address}:{guest_port}", + ), + ( + "-t nat", + f"OUTPUT -p tcp --dport {forwarded_port} " + f"-j DNAT --to-destination {guest_address}:{guest_port}", + ), + ] + + try: self.host_node.execute( - command, + "sysctl -w net.ipv4.ip_forward=1", shell=True, sudo=True, expected_exit_code=0, @@ -1399,15 +1735,48 @@ def _enable_ssh_forwarding( "failed to configure OpenVMM SSH forwarding" ), ) - except Exception: - try: - self._disable_ssh_forwarding_context(node_context, network) - except Exception as cleanup_identifier: - self._log.debug( - "failed to roll back OpenVMM SSH forwarding after setup " - f"error: {cleanup_identifier}" - ) - raise + for table_args, rule in rules: + self._ensure_ssh_forwarding_rule(node_context, table_args, rule) + except Exception: + try: + self._disable_ssh_forwarding_context(node_context, network) + except Exception as cleanup_identifier: + self._log.debug( + "failed to roll back OpenVMM SSH forwarding after setup " + f"error: {cleanup_identifier}" + ) + raise + + def _ensure_ssh_forwarding_rule( + self, + node_context: NodeContext, + table_args: str, + rule: str, + ) -> None: + table_prefix = f"{table_args} " if table_args else "" + check_command = f"iptables {table_prefix}-C {rule}" + insert_command = f"iptables {table_prefix}-I {rule}" + result = self.host_node.execute( + check_command, + shell=True, + sudo=True, + no_info_log=True, + no_error_log=True, + expected_exit_code=None, + ) + if result.exit_code == 0: + return + + self.host_node.execute( + insert_command, + shell=True, + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + "failed to configure OpenVMM SSH forwarding" + ), + ) + node_context.forwarding_rules_added.append(f"{table_args}|{rule}") def _disable_ssh_forwarding(self, node: Node) -> None: node_context = get_node_context(node) @@ -1418,7 +1787,7 @@ def _disable_ssh_forwarding(self, node: Node) -> None: def _disable_ssh_forwarding_context( self, - node_context: Any, + node_context: NodeContext, network: OpenVmmNetworkSchema, ) -> None: if ( @@ -1429,90 +1798,48 @@ def _disable_ssh_forwarding_context( return host_context = get_host_context(self.host_node) - guest_address = shlex.quote(node_context.guest_address) - guest_port = node_context.ssh_port - forwarded_port = node_context.forwarded_port - forwarding_interface = node_context.forwarding_interface - host_interface = _get_tap_host_interface_name(network) - host_network = ipaddress.ip_interface(network.tap_host_cidr).network - commands = [ - ( - "iptables -D FORWARD -i " - f"{shlex.quote(host_interface)} -o {shlex.quote(forwarding_interface)} " - "-j ACCEPT || true" - ), - ( - "iptables -D FORWARD -i " - f"{shlex.quote(host_interface)} ! -o " - f"{shlex.quote(forwarding_interface)} " - "-j ACCEPT || true" - ), - ( - "iptables -D FORWARD -i " - f"{shlex.quote(forwarding_interface)} -o {shlex.quote(host_interface)} " - "-m state --state RELATED,ESTABLISHED -j ACCEPT || true" - ), - ( - "iptables -D FORWARD ! -i " - f"{shlex.quote(forwarding_interface)} -o {shlex.quote(host_interface)} " - "-m state --state RELATED,ESTABLISHED -j ACCEPT || true" - ), - ( - "iptables -D FORWARD -i " - f"{shlex.quote(forwarding_interface)} -o {shlex.quote(host_interface)} " - f"-p tcp -d {guest_address} --dport {guest_port} -j ACCEPT || true" - ), - ( - "iptables -t nat -D PREROUTING -p tcp --dport " - f"{forwarded_port} -j DNAT --to-destination " - f"{guest_address}:{guest_port} || true" - ), - ( - "iptables -t nat -D OUTPUT -p tcp --dport " - f"{forwarded_port} -j DNAT --to-destination " - f"{guest_address}:{guest_port} || true" - ), - ( - "iptables -t nat -D POSTROUTING -s " - f"{shlex.quote(str(host_network))} " - f"-o {shlex.quote(forwarding_interface)} " - "-j MASQUERADE || true" - ), - ] - for command in commands: - self.host_node.execute( - command, - shell=True, - sudo=True, - expected_exit_code=0, - expected_exit_code_failure_message=( - "failed to remove OpenVMM SSH forwarding" - ), - ) + with host_context.ssh_forwarding_lock: + for rule_key in reversed(node_context.forwarding_rules_added): + table_args, rule = rule_key.split("|", 1) + table_prefix = f"{table_args} " if table_args else "" + command = f"iptables {table_prefix}-D {rule} || true" + self.host_node.execute( + command, + shell=True, + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + "failed to remove OpenVMM SSH forwarding" + ), + ) - if node_context.forwarding_enabled and host_context.active_forwarding_count > 0: - host_context.active_forwarding_count -= 1 + if ( + node_context.forwarding_enabled + and host_context.active_forwarding_count > 0 + ): + host_context.active_forwarding_count -= 1 - if ( - host_context.active_forwarding_count == 0 - and host_context.original_ip_forward_value - ): - self.host_node.execute( - "sysctl -w net.ipv4.ip_forward=" - f"{shlex.quote(host_context.original_ip_forward_value)}", - shell=True, - sudo=True, - expected_exit_code=0, - expected_exit_code_failure_message=( - "failed to restore host ip_forward state after OpenVMM " - "SSH forwarding" - ), - ) - host_context.original_ip_forward_value = "" + if ( + host_context.active_forwarding_count == 0 + and host_context.original_ip_forward_value + ): + self.host_node.execute( + "sysctl -w net.ipv4.ip_forward=" + f"{shlex.quote(host_context.original_ip_forward_value)}", + shell=True, + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + "failed to restore host ip_forward state after OpenVMM " + "SSH forwarding" + ), + ) + host_context.original_ip_forward_value = "" - node_context.forwarded_port = 0 - node_context.forwarding_enabled = False - node_context.forwarding_interface = "" + node_context.forwarded_port = 0 + node_context.forwarding_enabled = False + node_context.forwarding_interface = "" + node_context.forwarding_rules_added.clear() def _wait_for_process_exit(self, process_id: str, timeout: int = 60) -> None: try: @@ -1678,10 +2005,19 @@ def type_schema(cls) -> Type[schema.TypedSchema]: return OpenVmmGuestNodeSchema def cleanup(self) -> None: + passthrough_release_error: Optional[Exception] = None try: self._openvmm_controller.stop_node(self, wait=False) except Exception as identifier: self.log.debug(f"failed to stop OpenVMM guest during cleanup: {identifier}") + try: + self._openvmm_controller.release_device_passthrough(self) + except Exception as identifier: + passthrough_release_error = identifier + self.log.warning( + "failed to release OpenVMM passthrough devices during cleanup: " + f"{identifier}" + ) try: self._openvmm_controller.cleanup_node_artifacts(self) except Exception as identifier: @@ -1689,6 +2025,12 @@ def cleanup(self) -> None: f"failed to clean OpenVMM guest artifacts during cleanup: {identifier}" ) super().cleanup() + if passthrough_release_error: + raise LisaException( + "failed to release OpenVMM passthrough devices during cleanup. " + "Verify host PCI devices were restored before reusing this host: " + f"{passthrough_release_error}" + ) def _initialize(self, *args: Any, **kwargs: Any) -> None: self._provision() @@ -1766,6 +2108,10 @@ def _provision(self) -> None: ) node_context.console_log_file_path = str(working_path / "openvmm-console.log") node_context.ssh_port = runbook.network.ssh_port + self._openvmm_controller.set_device_passthrough_node_context( + node_context, + runbook, + ) self._openvmm_controller.launch(self, self.log) self._openvmm_controller.configure_connection(self, self.log) diff --git a/lisa/sut_orchestrator/openvmm/schema.py b/lisa/sut_orchestrator/openvmm/schema.py index d8c660bf33..686c38db4d 100644 --- a/lisa/sut_orchestrator/openvmm/schema.py +++ b/lisa/sut_orchestrator/openvmm/schema.py @@ -3,6 +3,7 @@ import ipaddress import re +import shlex from dataclasses import dataclass, field from typing import List, Optional @@ -10,7 +11,11 @@ from lisa import schema from lisa.secret import PATTERN_HEADTAIL, add_secret -from lisa.sut_orchestrator.util.schema import CloudInitSchema +from lisa.sut_orchestrator.util.schema import ( + CloudInitSchema, + DevicePassthroughSchema, + HostDevicePoolSchema, +) from lisa.util import LisaException from .. import OPENVMM @@ -20,6 +25,8 @@ OPENVMM_ADDRESS_MODE_STATIC = "static" OPENVMM_NETWORK_MODE_USER = "user" OPENVMM_NETWORK_MODE_TAP = "tap" +OPENVMM_CONNECTION_MODE_FORWARDED_PORT = "forwarded_port" +OPENVMM_CONNECTION_MODE_HOST_PROXY = "host_proxy" OPENVMM_SERIAL_MODE_STDERR = "stderr" OPENVMM_SERIAL_MODE_FILE = "file" # Keep raw disk growth opt-in so existing OpenVMM runbooks don't mutate @@ -29,6 +36,19 @@ OPENVMM_INTERFACE_NAME_PATTERN = re.compile(r"^[A-Za-z0-9_.-]+$") +def _decode_extra_args(value: object) -> List[str]: + if value is None: + return [] + if isinstance(value, str): + return shlex.split(value) + if isinstance(value, list): + return [str(item) for item in value if str(item)] + raise LisaException( + "OpenVMM extra_args must be a string or list of strings, " + f"not '{type(value).__name__}'" + ) + + @dataclass_json() @dataclass class OpenVmmInstallerSchema(schema.TypedSchema, schema.ExtendableSchemaMixin): @@ -79,6 +99,7 @@ def __post_init__(self) -> None: @dataclass class OpenVmmNetworkSchema: mode: str = OPENVMM_NETWORK_MODE_USER + connection_mode: str = OPENVMM_CONNECTION_MODE_FORWARDED_PORT address_mode: str = OPENVMM_ADDRESS_MODE_DISCOVER tap_name: str = "" bridge_name: str = "" @@ -140,6 +161,15 @@ def validate_tap_interface_names(self) -> None: self._validate_interface_name("bridge_name", self.bridge_name) def __post_init__(self) -> None: + if self.connection_mode not in [ + OPENVMM_CONNECTION_MODE_FORWARDED_PORT, + OPENVMM_CONNECTION_MODE_HOST_PROXY, + ]: + raise LisaException( + f"connection_mode '{self.connection_mode}' is not supported. " + f"Supported values: {OPENVMM_CONNECTION_MODE_FORWARDED_PORT}, " + f"{OPENVMM_CONNECTION_MODE_HOST_PROXY}" + ) if self.mode not in [ OPENVMM_NETWORK_MODE_USER, OPENVMM_NETWORK_MODE_TAP, @@ -162,7 +192,14 @@ def __post_init__(self) -> None: f"{OPENVMM_ADDRESS_MODE_STATIC}" ) - if self.forwarded_port: + if self.connection_mode == OPENVMM_CONNECTION_MODE_HOST_PROXY: + if self.mode != OPENVMM_NETWORK_MODE_TAP: + raise LisaException( + "host_proxy connection_mode is supported only with tap networking" + ) + self.forward_ssh_port = False + self.forwarded_port = 0 + elif self.forwarded_port: self.forward_ssh_port = True if self.forward_ssh_port: @@ -200,6 +237,12 @@ def __post_init__(self) -> None: ) +@dataclass_json() +@dataclass +class OpenVmmDevicePassthroughSchema(DevicePassthroughSchema): + managed: str = "" + + @dataclass_json() @dataclass class OpenVmmGuestNodeSchema(schema.GuestNode): @@ -225,7 +268,12 @@ class OpenVmmGuestNodeSchema(schema.GuestNode): openvmm_binary: str = "/usr/local/bin/openvmm" serial: OpenVmmSerialSchema = field(default_factory=OpenVmmSerialSchema) network: OpenVmmNetworkSchema = field(default_factory=OpenVmmNetworkSchema) - extra_args: List[str] = field(default_factory=list) + device_pools: Optional[List[HostDevicePoolSchema]] = None + device_passthrough: Optional[List[OpenVmmDevicePassthroughSchema]] = None + extra_args: List[str] = field( + default_factory=list, + metadata=schema.field_metadata(decoder=_decode_extra_args), + ) def __post_init__(self) -> None: add_secret(self.username, PATTERN_HEADTAIL) diff --git a/lisa/sut_orchestrator/ready.py b/lisa/sut_orchestrator/ready.py index 8e9b40f440..bb2ca9ca5f 100644 --- a/lisa/sut_orchestrator/ready.py +++ b/lisa/sut_orchestrator/ready.py @@ -81,7 +81,7 @@ def _deploy_environment(self, environment: Environment, log: Logger) -> None: pass def _delete_environment(self, environment: Environment, log: Logger) -> None: - if self._ready_runbook.reuse_dirty_env: + if self._ready_runbook.reuse_dirty_env and not self.runbook.guest_enabled: log.debug( f"Environment '{environment.name}' was marked as 'Deleted' " "because it was dirty. Now resetting it to 'Prepared' since " @@ -89,3 +89,8 @@ def _delete_environment(self, environment: Environment, log: Logger) -> None: "the environment." ) environment.status = EnvironmentStatus.Prepared + elif self._ready_runbook.reuse_dirty_env: + log.debug( + f"Environment '{environment.name}' was marked as 'Deleted'. " + "It won't be reused because guest_enabled is true." + ) diff --git a/lisa/tools/cargo.py b/lisa/tools/cargo.py index 11f95967ad..c06909c580 100644 --- a/lisa/tools/cargo.py +++ b/lisa/tools/cargo.py @@ -11,8 +11,6 @@ from lisa.tools.curl import Curl from lisa.tools.echo import Echo from lisa.tools.gcc import Gcc -from lisa.tools.ln import Ln -from lisa.tools.rm import Rm from lisa.util import LisaException, UnsupportedDistroException from lisa.util.process import ExecutableResult @@ -30,6 +28,13 @@ class Cargo(Tool): re.M, ) toolchain: str = "" + # Rustup mutates shared state under $HOME/.rustup. Baremetal jobs may reuse the + # same account concurrently, so wait long enough for another toolchain install. + RUSTUP_LOCK_TIMEOUT = 1800 + # The rustup installer downloads the full Rust toolchain. Baremetal lab + # network throughput can be slow enough that the default command timeout is + # too short even when the download is still making progress. + RUSTUP_INSTALL_TIMEOUT = 1800 @property def command(self) -> str: @@ -46,71 +51,113 @@ def dependencies(self) -> List[Type[Tool]]: def _initialize(self, *args: Any, **kwargs: Any) -> None: self._command = "cargo" + def _check_exists(self) -> bool: + exists, self._use_sudo = self.command_exists(self.command) + if exists and self._cargo_runs(self.command): + return True + + home_dir = self._get_home_dir() + if not home_dir: + return False + + cargo_bin = f"{home_dir}/.cargo/bin/cargo" + if self._cargo_runs(cargo_bin): + self._command = cargo_bin + self._use_sudo = False + rustup_bin = f"{home_dir}/.cargo/bin/rustup" + self._try_set_toolchain(rustup_bin) + return True + + return False + def _install(self) -> bool: node_os = self.node.os cargo_source_url = "https://sh.rustup.rs" if isinstance(node_os, CBLMariner) or isinstance(node_os, Ubuntu): self.__install_dependencies() + home_dir = self._get_home_dir() + if not home_dir: + raise LisaException("failure to grab $HOME path") + cargo_bin = f"{home_dir}/.cargo/bin/cargo" + rustup_bin = f"{home_dir}/.cargo/bin/rustup" + # install cargo/rust curl = self.node.tools[Curl] - result = curl.fetch( - arg="-sSf", - url=cargo_source_url, - execute_arg="-s -- -y", - shell=True, + install_command = ( + f"{shlex.quote(cargo_bin)} --version >/dev/null 2>&1 || " + f"{shlex.quote(curl.command)} -sSf " + f"{shlex.quote(cargo_source_url)} | sh -s -- -y" ) - result.assert_exit_code() - - echo = self.node.tools[Echo] - home_dir = echo.run( - "$HOME", + self.node.execute( + self.wrap_with_rustup_lock(install_command), shell=True, + timeout=self.RUSTUP_INSTALL_TIMEOUT, expected_exit_code=0, - expected_exit_code_failure_message="failure to grab $HOME path", - ).stdout.strip() - cargo_bin = f"{home_dir}/.cargo/bin/cargo" - rustup_bin = f"{home_dir}/.cargo/bin/rustup" - - ln = self.node.tools[Ln] - ln.create_link( - is_symbolic=True, - target=cargo_bin, - link="/usr/local/bin/cargo", - ) - ln.create_link( - is_symbolic=True, - target=rustup_bin, - link="/usr/local/bin/rustup", + expected_exit_code_failure_message="curl fetch failed", ) else: raise UnsupportedDistroException(node_os) self.node.execute( - f"{shlex.quote(rustup_bin)} default stable", + self.wrap_with_rustup_lock(f"{shlex.quote(rustup_bin)} default stable"), shell=True, expected_exit_code=0, expected_exit_code_failure_message="failed to set rustup stable toolchain", ) self.toolchain = self.__get_rust_toolchain(rustup_command=rustup_bin) self._log.debug(f"Rust toolchain: {self.toolchain}") - self._command = f"{home_dir}/.rustup/toolchains/{self.toolchain}/bin/cargo" + self._command = cargo_bin self._exists = None is_installed = self._check_exists() - if not is_installed: - self._command = cargo_bin - self._exists = None - is_installed = self._check_exists() + return is_installed - self.node.tools[Rm].remove_file( - "/usr/local/bin/cargo", - sudo=True, + def wrap_with_rustup_lock(self, command: str) -> str: + return ( + 'mkdir -p "$HOME/.rustup" && ' + f"flock -w {self.RUSTUP_LOCK_TIMEOUT} " + '"$HOME/.rustup/lisa-rustup.lock" ' + f"sh -c {shlex.quote(command)}" ) - self.node.tools[Rm].remove_file( - "/usr/local/bin/rustup", - sudo=True, + + def _get_home_dir(self) -> str: + result = self.node.execute( + "echo $HOME", + shell=True, + no_info_log=True, + no_error_log=True, + expected_exit_code=None, ) - return is_installed + if result.exit_code != 0: + return "" + return result.stdout.strip() + + def _cargo_runs(self, command: str) -> bool: + result = self.node.execute( + f"{shlex.quote(command)} --version", + shell=True, + no_info_log=True, + no_error_log=True, + expected_exit_code=None, + ) + return result.exit_code == 0 + + def _try_set_toolchain(self, rustup_command: str) -> None: + result = self.node.execute( + f"test -x {shlex.quote(rustup_command)}", + shell=True, + no_info_log=True, + no_error_log=True, + expected_exit_code=None, + ) + if result.exit_code != 0: + return + try: + self.toolchain = self.__get_rust_toolchain(rustup_command=rustup_command) + except LisaException as identifier: + self._log.debug( + f"failed to detect rust toolchain from {rustup_command}: {identifier}" + ) def __install_dependencies(self) -> None: node_os: Posix = cast(Posix, self.node.os) diff --git a/lisa/tools/openvmm.py b/lisa/tools/openvmm.py index 06f0cc4d48..a1f5404541 100644 --- a/lisa/tools/openvmm.py +++ b/lisa/tools/openvmm.py @@ -13,6 +13,7 @@ VERSION_PATTERN = re.compile(r"openvmm(?:\.exe)?\s+(?P.+)") OPENVMM_NETWORK_BACKEND_CONSOMME = "consomme" +OPENVMM_DEFAULT_SCSI_CONTROLLER = "lisa_scsi0" _COMMAND_NOT_FOUND_MARKERS = ( "command not found", @@ -21,6 +22,10 @@ ) +def _new_str_list() -> List[str]: + return [] + + def is_missing_command_output(output: str) -> bool: normalized_output = output.lower() return any(marker in normalized_output for marker in _COMMAND_NOT_FOUND_MARKERS) @@ -32,7 +37,7 @@ class OpenVmmLaunchConfig: with_hv: bool = True hypervisor: str = "mshv" disk_img_path: str = "" - dvd_disk_paths: List[str] = field(default_factory=list) + dvd_disk_paths: List[str] = field(default_factory=_new_str_list) processors: int = 1 memory_mb: int = 1024 network_mode: str = "user" @@ -40,7 +45,7 @@ class OpenVmmLaunchConfig: network_cidr: str = "" serial_mode: str = "file" serial_path: str = "" - extra_args: List[str] = field(default_factory=list) + extra_args: List[str] = field(default_factory=_new_str_list) stdout_path: str = "" stderr_path: str = "" @@ -103,11 +108,26 @@ def build_command(self, config: OpenVmmLaunchConfig) -> str: args.append("--uefi") args.extend(["--uefi-firmware", config.uefi_firmware_path]) + if config.disk_img_path or config.dvd_disk_paths: + args.extend(["--vmbus-scsi", f"id={OPENVMM_DEFAULT_SCSI_CONTROLLER}"]) + if config.disk_img_path: - args.extend(["--disk", f"file:{config.disk_img_path}"]) + args.extend( + [ + "--disk", + f"file:{config.disk_img_path}," + f"on={OPENVMM_DEFAULT_SCSI_CONTROLLER},lun=0", + ] + ) - for dvd_disk_path in config.dvd_disk_paths: - args.extend(["--disk", f"file:{dvd_disk_path},dvd"]) + for lun, dvd_disk_path in enumerate(config.dvd_disk_paths, start=1): + args.extend( + [ + "--disk", + f"file:{dvd_disk_path},on={OPENVMM_DEFAULT_SCSI_CONTROLLER}," + f"lun={lun},dvd", + ] + ) if config.network_mode == "user": network_backend = OPENVMM_NETWORK_BACKEND_CONSOMME diff --git a/lisa/util/shell.py b/lisa/util/shell.py index 1aa9ef7982..7bde154094 100644 --- a/lisa/util/shell.py +++ b/lisa/util/shell.py @@ -11,7 +11,18 @@ from functools import partial from pathlib import Path, PurePath, PureWindowsPath from time import sleep -from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union, cast +from typing import ( + Any, + Callable, + Dict, + List, + Mapping, + Optional, + Sequence, + Tuple, + Union, + cast, +) import paramiko import spur @@ -165,20 +176,21 @@ def try_connect( connection_info: schema.ConnectionInfo, ssh_timeout: int = 300, sock: Optional[Any] = None, + sock_factory: Optional[Callable[[], Any]] = None, ) -> Any: - # spur always run a posix command and will fail on Windows. - # So try with paramiko firstly. - paramiko_client = paramiko.SSHClient() - - # Use base policy, do nothing on host key. The host key shouldn't be saved - # locally, or make any warning message. The IP addresses in cloud may be - # reused by different servers. If they are saved, there will be conflict - # error in paramiko. - paramiko_client.set_missing_host_key_policy(paramiko.MissingHostKeyPolicy()) - # wait for ssh port to be ready timeout_start = time.time() while time.time() < timeout_start + ssh_timeout: + # spur always run a posix command and will fail on Windows. + # So try with paramiko firstly. + paramiko_client = paramiko.SSHClient() + current_sock = sock_factory() if sock_factory else sock + + # Use base policy, do nothing on host key. The host key shouldn't be saved + # locally, or make any warning message. The IP addresses in cloud may be + # reused by different servers. If they are saved, there will be conflict + # error in paramiko. + paramiko_client.set_missing_host_key_policy(paramiko.MissingHostKeyPolicy()) try: paramiko_client.connect( hostname=connection_info.address, @@ -187,7 +199,7 @@ def try_connect( password=connection_info.password, key_filename=connection_info.private_key_file, banner_timeout=10, - sock=sock, + sock=current_sock, ) stdin, stdout, _ = paramiko_client.exec_command("cmd\n") @@ -218,6 +230,7 @@ def try_connect( return stdout except SSHException as e: + paramiko_client.close() # socket is open, but SSH service not responded if ( str(e) == "Error reading SSH protocol banner" @@ -225,10 +238,14 @@ def try_connect( ): sleep(1) continue - except (NoValidConnectionsError, ConnectionResetError, TimeoutError): + except (NoValidConnectionsError, ConnectionResetError, TimeoutError, EOFError): + paramiko_client.close() # ssh service is not ready sleep(1) continue + except Exception: + paramiko_client.close() + raise # raise exception if ssh service is not ready raise LisaException(f"ssh connection cannot be established: {connection_info}") @@ -262,10 +279,15 @@ def _minimize_shell(shell: spur.ssh.SshShell) -> None: class SshShell(InitializableMixin): - def __init__(self, connection_info: schema.ConnectionInfo) -> None: + def __init__( + self, + connection_info: schema.ConnectionInfo, + proxy_jump_boxes: Optional[List[schema.ConnectionInfo]] = None, + ) -> None: super().__init__() self.is_remote = True self.connection_info = connection_info + self._proxy_jump_boxes = proxy_jump_boxes or [] self._inner_shell: Optional[spur.SshShell] = None self._jump_boxes: List[Any] = [] self._jump_box_sock: Any = None @@ -279,24 +301,44 @@ def __init__(self, connection_info: schema.ConnectionInfo) -> None: paramiko_logger.setLevel(logging.WARN) def _initialize(self, *args: Any, **kwargs: Any) -> None: - is_ready, tcp_error_code = wait_tcp_port_ready( - self.connection_info.address, self.connection_info.port - ) - if not is_ready: - raise TcpConnectionException( - self.connection_info.address, - self.connection_info.port, - tcp_error_code, + jump_boxes_runbook = self._get_jump_boxes() + if not jump_boxes_runbook: + is_ready, tcp_error_code = wait_tcp_port_ready( + self.connection_info.address, self.connection_info.port ) + if not is_ready: + raise TcpConnectionException( + self.connection_info.address, + self.connection_info.port, + tcp_error_code, + ) - sock = self._establish_jump_boxes( - address=self.connection_info.address, - port=self.connection_info.port, - ) + sock_factory: Optional[Callable[[], Any]] = None + sock = None + if jump_boxes_runbook: + + def _create_jump_box_sock() -> Any: + self._close_jump_boxes() + return self._establish_jump_boxes( + address=self.connection_info.address, + port=self.connection_info.port, + ) + + sock_factory = _create_jump_box_sock + else: + sock = self._establish_jump_boxes( + address=self.connection_info.address, + port=self.connection_info.port, + ) try: - stdout = try_connect(self.connection_info, sock=sock) + stdout = try_connect( + self.connection_info, + sock=sock, + sock_factory=sock_factory, + ) except Exception as e: + self._close_jump_boxes() raise LisaException( "failed to connect SSH port of the VM. It might be due to one of the " "following reasons: 1. Port 22 is not open. 2. The VM denies other " @@ -679,7 +721,7 @@ def _purepath_to_str( return path def _establish_jump_boxes(self, address: str, port: int) -> Any: - jump_boxes_runbook = development.get_jump_boxes() + jump_boxes_runbook = self._get_jump_boxes() sock: Any = None is_trace_enabled = development.is_trace_enabled() if is_trace_enabled: @@ -704,12 +746,14 @@ def _establish_jump_boxes(self, address: str, port: int) -> Any: if index < len(jump_boxes_runbook) - 1: next_hop = jump_boxes_runbook[index + 1] dest_address = ( - next_hop.private_address - if next_hop.private_address + getattr(next_hop, "private_address", "") + if getattr(next_hop, "private_address", "") else next_hop.address ) dest_port = ( - next_hop.private_port if next_hop.private_port else next_hop.port + getattr(next_hop, "private_port", 0) + if getattr(next_hop, "private_port", 0) + else next_hop.port ) else: dest_address = address @@ -728,6 +772,9 @@ def _establish_jump_boxes(self, address: str, port: int) -> Any: return sock + def _get_jump_boxes(self) -> List[Any]: + return [*development.get_jump_boxes(), *self._proxy_jump_boxes] + def _open_jump_box_channel( self, client: paramiko.SSHClient, @@ -741,7 +788,7 @@ def _open_jump_box_channel( sock = transport.open_channel( kind="direct-tcpip", - src_addr=(src_address, src_port), + src_addr=("127.0.0.1", 0), dest_addr=(dest_address, dest_port), ) diff --git a/selftests/runners/test_lisa_runner.py b/selftests/runners/test_lisa_runner.py index f4e66f74fc..554322bab5 100644 --- a/selftests/runners/test_lisa_runner.py +++ b/selftests/runners/test_lisa_runner.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. from pathlib import Path +from types import SimpleNamespace from typing import Any, List, Optional, Union, cast from unittest import TestCase from unittest.mock import MagicMock, patch @@ -304,6 +305,36 @@ def test_merge_req_platform_type_checked(self) -> None: test_results=test_results, ) + def test_merge_req_guest_platform_type_checked(self) -> None: + envs = load_environments(None) + runner = generate_runner(None) + platform = test_platform.generate_platform() + platform.runbook.guest_enabled = True + platform.runbook.guests = [SimpleNamespace(type="guest-platform")] + runner.platform = platform + runner._guest_enabled = True + + test_results = test_testsuite.generate_cases_result() + for test_result in test_results: + metadata = test_result.runtime_data.metadata + metadata.requirement = simple_requirement( + supported_platform_type=["guest-platform"] + ) + + runner._merge_test_requirements( + test_results=test_results, + existing_environments=envs, + platform_type=constants.PLATFORM_MOCK, + ) + + self.verify_test_results( + expected_test_order=["mock_ut1", "mock_ut2", "mock_ut3"], + expected_envs=["", "", ""], + expected_status=[TestStatus.QUEUED, TestStatus.QUEUED, TestStatus.QUEUED], + expected_message=["", "", ""], + test_results=test_results, + ) + def test_fit_a_predefined_env(self) -> None: # predefined env can run case in below condition. # 1. with predefined env of 1 simple node, so ut2 don't need a new env @@ -782,7 +813,7 @@ def verify_test_results( ) # compare it's begin with actual_messages = [ - test_results[index].message[0 : len(expected)] + test_results[index].message[: len(expected)] for index, expected in enumerate(expected_message) ] self.assertListEqual( diff --git a/selftests/test_environment.py b/selftests/test_environment.py index b8665172fc..dd20d7fa29 100644 --- a/selftests/test_environment.py +++ b/selftests/test_environment.py @@ -254,6 +254,40 @@ def test_create_from_runbook_cap(self) -> None: env_cap.nodes[0].network_interface.nic_count, ) + def test_node_cleanup_uses_guest_snapshot(self) -> None: + parent = node.LocalNode( + index=0, + runbook=schema.LocalNode(), + logger_name="node", + ) + cleanup_counter: List[str] = [] + guest = node.LocalNode( + index=0, + runbook=schema.LocalNode(), + logger_name="g", + parent=parent, + ) + + def cleanup_guest() -> None: + cleanup_counter.append(guest.name) + parent.guests.append( + node.LocalNode( + index=len(parent.guests), + runbook=schema.LocalNode(), + logger_name="g", + parent=parent, + ) + ) + node.LocalNode.cleanup(guest) + + cast(Any, guest).cleanup = cleanup_guest + parent.guests.append(guest) + + parent.cleanup() + + self.assertEqual(1, len(cleanup_counter)) + self.assertEqual(1, len(parent.guests)) + def test_create_from_requirement(self) -> None: requirement = simple_requirement(min_count=2) env_requirement = requirement.environment diff --git a/selftests/test_openvmm_installer.py b/selftests/test_openvmm_installer.py index f64df4d819..085a489650 100644 --- a/selftests/test_openvmm_installer.py +++ b/selftests/test_openvmm_installer.py @@ -9,7 +9,7 @@ from lisa.sut_orchestrator.openvmm.installer import OpenVmmSourceInstaller from lisa.sut_orchestrator.openvmm.schema import OpenVmmSourceInstallerSchema -from lisa.tools import Cargo, Git, Ln +from lisa.tools import Cargo, Git, Ln, Rm class _ToolMap: @@ -32,22 +32,34 @@ class OpenVmmInstallerTestCase(TestCase): def test_source_installer_uses_host_paths_for_remote_commands(self) -> None: package_installs: List[List[str]] = [] linux = Ubuntu(package_installs) + + def _wrap_with_rustup_lock(command: str) -> str: + return f"locked({command})" + cargo = SimpleNamespace( exists=True, - command="/home/test/.cargo/bin/cargo", + command="cargo", toolchain="stable", + wrap_with_rustup_lock=_wrap_with_rustup_lock, ) git = SimpleNamespace( - clone=MagicMock(return_value="/tmp/work/openvmm-src"), + clone=MagicMock(return_value="/tmp/work/openvmm"), get_current_commit_hash=MagicMock(return_value="abc1234"), ) ln = SimpleNamespace(create_link=MagicMock()) + rm = SimpleNamespace(remove_directory=MagicMock()) executed_commands: List[Dict[str, Any]] = [] def _execute(command: str, **kwargs: Any) -> SimpleNamespace: executed_commands.append({"command": command, **kwargs}) if command == "echo $HOME": return SimpleNamespace(stdout="/home/test\n", stderr="", exit_code=0) + if command == "command -v cargo": + return SimpleNamespace( + stdout="/home/test/.cargo/bin/cargo\n", + stderr="", + exit_code=0, + ) if command == "/usr/local/bin/openvmm --version": return SimpleNamespace(stdout="openvmm 1.0.0\n", stderr="", exit_code=0) return SimpleNamespace(stdout="", stderr="", exit_code=0) @@ -58,7 +70,7 @@ def _execute(command: str, **kwargs: Any) -> SimpleNamespace: execute=_execute, get_pure_path=PurePosixPath, get_str_path=str, - tools=_ToolMap({Cargo: cargo, Git: git, Ln: ln}), + tools=_ToolMap({Cargo: cargo, Git: git, Ln: ln, Rm: rm}), ) runbook = OpenVmmSourceInstallerSchema( repo="https://github.com/microsoft/openvmm.git", @@ -88,6 +100,20 @@ def _execute(command: str, **kwargs: Any) -> SimpleNamespace: }, restore_call["update_envs"], ) + rust_src_call = next( + command + for command in executed_commands + if "component add rust-src" in command["command"] + ) + self.assertTrue(rust_src_call["command"].startswith("locked(")) + rm.remove_directory.assert_called_once_with("/tmp/work/openvmm") + git.clone.assert_called_once_with( + url="https://github.com/microsoft/openvmm.git", + cwd=PurePosixPath("/tmp/work"), + dir_name="openvmm", + ref="", + auth_token="", + ) install_dir_call = next( command for command in executed_commands @@ -99,7 +125,76 @@ def _execute(command: str, **kwargs: Any) -> SimpleNamespace: for command in executed_commands if command["command"].startswith("cp ") ) - self.assertIn( - "/tmp/work/openvmm-src/target/release/openvmm", copy_call["command"] - ) + self.assertIn("/tmp/work/openvmm/target/release/openvmm", copy_call["command"]) self.assertNotIn("\\", copy_call["command"]) + + def test_source_installer_refreshes_stale_source_checkout(self) -> None: + package_installs: List[List[str]] = [] + linux = Ubuntu(package_installs) + + def _wrap_with_rustup_lock(command: str) -> str: + return f"locked({command})" + + cargo = SimpleNamespace( + exists=True, + command="cargo", + toolchain="stable", + wrap_with_rustup_lock=_wrap_with_rustup_lock, + ) + git = SimpleNamespace( + clone=MagicMock(return_value="/tmp/work/openvmm"), + get_current_commit_hash=MagicMock(return_value="abc1234"), + ) + ln = SimpleNamespace(create_link=MagicMock()) + rm = SimpleNamespace(remove_directory=MagicMock()) + executed_commands: List[Dict[str, Any]] = [] + + def _execute(command: str, **kwargs: Any) -> SimpleNamespace: + executed_commands.append({"command": command, **kwargs}) + if command == "echo $HOME": + return SimpleNamespace(stdout="/home/test\n", stderr="", exit_code=0) + if command == "command -v cargo": + return SimpleNamespace( + stdout="/home/test/.cargo/bin/cargo\n", + stderr="", + exit_code=0, + ) + if command == "/usr/local/bin/openvmm --version": + return SimpleNamespace(stdout="openvmm 1.0.0\n", stderr="", exit_code=0) + return SimpleNamespace(stdout="", stderr="", exit_code=0) + + node = SimpleNamespace( + os=linux, + working_path=PurePosixPath("/tmp/work"), + execute=_execute, + get_pure_path=PurePosixPath, + get_str_path=str, + tools=_ToolMap({Cargo: cargo, Git: git, Ln: ln, Rm: rm}), + ) + runbook = OpenVmmSourceInstallerSchema( + repo="https://github.com/microsoft/openvmm.git", + install_path="/usr/local/bin/openvmm", + ) + installer = OpenVmmSourceInstaller( + runbook=runbook, + node=cast(Any, node), + log=MagicMock(), + ) + + version = installer.install() + + self.assertEqual("openvmm 1.0.0", version) + rm.remove_directory.assert_called_once_with("/tmp/work/openvmm") + git.clone.assert_called_once_with( + url="https://github.com/microsoft/openvmm.git", + cwd=PurePosixPath("/tmp/work"), + dir_name="openvmm", + ref="", + auth_token="", + ) + restore_calls = [ + command + for command in executed_commands + if "xflowey restore-packages" in command["command"] + ] + self.assertEqual(1, len(restore_calls)) diff --git a/selftests/test_openvmm_node.py b/selftests/test_openvmm_node.py index fd3a428a8e..e1cafde91d 100644 --- a/selftests/test_openvmm_node.py +++ b/selftests/test_openvmm_node.py @@ -15,6 +15,7 @@ from lisa.sut_orchestrator.openvmm.context import NodeContext from lisa.sut_orchestrator.openvmm.node import OpenVmmController, OpenVmmGuestNode from lisa.sut_orchestrator.openvmm.schema import ( + OPENVMM_CONNECTION_MODE_HOST_PROXY, OPENVMM_NETWORK_MODE_TAP, OpenVmmGuestNodeSchema, OpenVmmNetworkSchema, @@ -293,10 +294,15 @@ def test_enable_ssh_forwarding_allows_openvmm_guest_subnet_routing( bridge_name = "ovmbr1" guest_address = "10.0.1.2" tap_host_cidr = "10.0.1.1/24" - execute_result = SimpleNamespace(exit_code=0, stderr="", stdout="0") + + def _execute(command: str, *args: Any, **kwargs: Any) -> Any: + if command.startswith("iptables") and " -C " in command: + return SimpleNamespace(exit_code=1, stderr="", stdout="") + return SimpleNamespace(exit_code=0, stderr="", stdout="0") + host_node = SimpleNamespace( is_remote=True, - execute=MagicMock(return_value=execute_result), + execute=MagicMock(side_effect=_execute), tools={Ip: SimpleNamespace(get_default_route_info=lambda: ("eth0", ""))}, ) controller = OpenVmmController(cast(Any, host_node), MagicMock()) @@ -334,6 +340,39 @@ def test_enable_ssh_forwarding_allows_openvmm_guest_subnet_routing( for command in commands ) ) + self.assertTrue( + any( + f"iptables -C FORWARD ! -i {bridge_name} -o {bridge_name} " + f"-p tcp -d {guest_address} --dport 22 -j ACCEPT" in command + for command in commands + ) + ) + self.assertTrue( + any( + f"iptables -I FORWARD -i {bridge_name} -o eth0 -j ACCEPT" in command + for command in commands + ) + ) + self.assertTrue( + any( + f"iptables -I FORWARD -i {bridge_name} ! -o eth0 -j ACCEPT" in command + for command in commands + ) + ) + self.assertTrue( + any( + f"iptables -I FORWARD ! -i eth0 -o {bridge_name} " + "-m state --state RELATED,ESTABLISHED -j ACCEPT" in command + for command in commands + ) + ) + self.assertTrue( + any( + f"iptables -I FORWARD ! -i {bridge_name} -o {bridge_name} " + f"-p tcp -d {guest_address} --dport 22 -j ACCEPT" in command + for command in commands + ) + ) self.assertTrue( any( f"iptables -D FORWARD -i {bridge_name} -o eth0 -j ACCEPT" in command @@ -353,12 +392,68 @@ def test_enable_ssh_forwarding_allows_openvmm_guest_subnet_routing( for command in commands ) ) + self.assertTrue( + any( + f"iptables -D FORWARD ! -i {bridge_name} -o {bridge_name} " + f"-p tcp -d {guest_address} --dport 22 -j ACCEPT" in command + for command in commands + ) + ) self.assertFalse( any( f"iptables -C FORWARD -i {bridge_name} -j ACCEPT" in command for command in commands ) ) + self.assertFalse( + any( + f"FORWARD -i eth0 -o {bridge_name} -p tcp -d {guest_address}" in command + for command in commands + ) + ) + + def test_configure_connection_uses_host_proxy_mode(self) -> None: + host_connection = schema.ConnectionInfo( + address="10.0.0.10", + port=22, + username="hostuser", + password="hostpass", + ) + host_node = SimpleNamespace(is_remote=True, _connection_info=host_connection) + controller = OpenVmmController(cast(Any, host_node), MagicMock()) + network = OpenVmmNetworkSchema( + mode=OPENVMM_NETWORK_MODE_TAP, + connection_mode=OPENVMM_CONNECTION_MODE_HOST_PROXY, + tap_name="tap0", + ) + node = SimpleNamespace( + runbook=OpenVmmGuestNodeSchema( + uefi=OpenVmmUefiSchema(firmware_path="/tmp/MSVM.fd"), + disk_img="/tmp/guest.raw", + username="root", + password="guestpass", + network=network, + ), + set_connection_info=MagicMock(), + ) + node_context = NodeContext(guest_address="10.0.0.2") + + with patch( + "lisa.sut_orchestrator.openvmm.node.get_node_context", + return_value=node_context, + ), patch.object( + controller, "_resolve_guest_address", return_value="10.0.0.2" + ), patch.object( + controller, "_wait_for_guest_ssh_from_host" + ) as wait_from_host: + controller.configure_connection(cast(Any, node), MagicMock()) + + wait_from_host.assert_called_once() + node.set_connection_info.assert_called_once() + kwargs = node.set_connection_info.call_args.kwargs + self.assertEqual("10.0.0.2", kwargs["address"]) + self.assertFalse(kwargs["use_public_address"]) + self.assertEqual([host_connection], kwargs["proxy_jump_boxes"]) def test_create_node_cloud_init_iso_skips_root_resize_for_non_raw_disk( self, diff --git a/selftests/test_openvmm_schema.py b/selftests/test_openvmm_schema.py index c84946d5fa..b889213c84 100644 --- a/selftests/test_openvmm_schema.py +++ b/selftests/test_openvmm_schema.py @@ -7,7 +7,10 @@ from marshmallow import ValidationError from lisa.sut_orchestrator.openvmm.schema import ( + OPENVMM_CONNECTION_MODE_HOST_PROXY, + OPENVMM_NETWORK_MODE_TAP, OPENVMM_NETWORK_MODE_USER, + OpenVmmGuestNodeSchema, OpenVmmNetworkSchema, ) @@ -35,3 +38,29 @@ def test_network_schema_rejects_invalid_ssh_port(self) -> None: "ssh_port": 0, } ) + + def test_guest_schema_splits_extra_args_string(self) -> None: + guest_schema = cast(Any, OpenVmmGuestNodeSchema).schema() + guest = guest_schema.load( + { + "uefi": {"firmware_path": "/firmware"}, + "disk_img": "/disk.raw", + "extra_args": "--foo 'bar baz'", + } + ) + + self.assertEqual(["--foo", "bar baz"], guest.extra_args) + + def test_host_proxy_connection_mode_disables_forwarded_port(self) -> None: + network_schema = cast(Any, OpenVmmNetworkSchema).schema() + network = network_schema.load( + { + "mode": OPENVMM_NETWORK_MODE_TAP, + "connection_mode": OPENVMM_CONNECTION_MODE_HOST_PROXY, + "tap_name": "tap0", + "forwarded_port": 60022, + } + ) + + self.assertFalse(network.forward_ssh_port) + self.assertEqual(0, network.forwarded_port) diff --git a/selftests/test_platform.py b/selftests/test_platform.py index ffe627c9ba..0f321185cc 100644 --- a/selftests/test_platform.py +++ b/selftests/test_platform.py @@ -9,6 +9,7 @@ from dataclasses_json import dataclass_json import lisa +import lisa.sut_orchestrator.ready # noqa: F401 from lisa import schema, search_space from lisa.environment import ( Environment, @@ -97,6 +98,7 @@ def _deploy_environment(self, environment: Environment, log: Logger) -> None: environment.create_node_from_requirement(node_requirement=node_space) for node in environment.nodes.list(): node._is_initialized = True + node.test_connection = lambda: True self.test_data.deployed_envs.append(environment.name) if self._mock_runbook.deployed_status not in [ EnvironmentStatus.Deployed, @@ -257,6 +259,47 @@ def test_prepared_env_deleted_not_ready(self) -> None: platform.delete_environment(env) self.assertEqual(EnvironmentStatus.Deleted, env.status) + def test_ready_guest_enabled_dirty_env_is_not_reused(self) -> None: + runbook = schema.load_by_type( + schema.Platform, + { + constants.TYPE: constants.PLATFORM_READY, + "guest_enabled": True, + constants.PLATFORM_READY: {"reuse_dirty_env": True}, + }, + ) + platform = load_platform([runbook]) + self.addCleanup(plugin_manager.unregister, plugin=platform) + platform.initialize() + env = load_environments(generate_env_runbook(local=True))["customized_0"] + + platform.prepare_environment(env) + env.status = EnvironmentStatus.Deployed + env.mark_dirty() + platform.delete_environment(env) + + self.assertEqual(EnvironmentStatus.Deleted, env.status) + + def test_ready_dirty_env_is_reused_by_default(self) -> None: + runbook = schema.load_by_type( + schema.Platform, + { + constants.TYPE: constants.PLATFORM_READY, + constants.PLATFORM_READY: {"reuse_dirty_env": True}, + }, + ) + platform = load_platform([runbook]) + self.addCleanup(plugin_manager.unregister, plugin=platform) + platform.initialize() + env = load_environments(generate_env_runbook(local=True))["customized_0"] + + platform.prepare_environment(env) + platform.deploy_environment(env) + env.mark_dirty() + platform.delete_environment(env) + + self.assertEqual(EnvironmentStatus.Prepared, env.status) + def test_initialize_guest_nodes_copies_parent_capability(self) -> None: runbook = schema.load_by_type( schema.Platform,