diff --git a/tests/chaos/conftest.py b/tests/chaos/conftest.py index 87734d3813..bfbbe680ca 100644 --- a/tests/chaos/conftest.py +++ b/tests/chaos/conftest.py @@ -43,6 +43,7 @@ utility_daemonset_for_custom_tests, wait_for_node_status, ) +from utilities.storage import construct_datavolume_source_dict from utilities.virt import VirtualMachineForTests, running_vm LOGGER = logging.getLogger(__name__) @@ -93,16 +94,18 @@ def chaos_dv_rhel9( artifactory_config_map_chaos_namespace_scope_module, ): yield DataVolume( - source="http", + source_dict=construct_datavolume_source_dict( + source="http", + url=rhel9_http_image_url, + secret_name=artifactory_secret_chaos_namespace_scope_module.name, + cert_configmap_name=artifactory_config_map_chaos_namespace_scope_module.name, + ), name="chaos-dv", api_name="storage", namespace=chaos_namespace.name, - url=rhel9_http_image_url, size=Images.Rhel.DEFAULT_DV_SIZE, storage_class=request.param["storage_class"], client=admin_client, - secret=artifactory_secret_chaos_namespace_scope_module, - cert_configmap=artifactory_config_map_chaos_namespace_scope_module.name, ) diff --git a/tests/chaos/snapshot/conftest.py b/tests/chaos/snapshot/conftest.py index d8ce5b2c3a..393223f6b6 100644 --- a/tests/chaos/snapshot/conftest.py +++ b/tests/chaos/snapshot/conftest.py @@ -8,6 +8,7 @@ TIMEOUT_8MIN, TIMEOUT_10MIN, ) +from utilities.storage import construct_datavolume_source_dict from utilities.virt import VirtualMachineForTests @@ -21,16 +22,18 @@ def chaos_dv_rhel9_for_snapshot( artifactory_config_map_chaos_namespace_scope_module, ): yield DataVolume( - source="http", + source_dict=construct_datavolume_source_dict( + source="http", + url=rhel9_http_image_url, + secret_name=artifactory_secret_chaos_namespace_scope_module.name, + cert_configmap_name=artifactory_config_map_chaos_namespace_scope_module.name, + ), name="chaos-dv", api_name="storage", namespace=chaos_namespace.name, - url=rhel9_http_image_url, size=Images.Rhel.DEFAULT_DV_SIZE, storage_class=[*storage_class_matrix_snapshot_matrix__function__][0], client=admin_client, - secret=artifactory_secret_chaos_namespace_scope_module, - cert_configmap=artifactory_config_map_chaos_namespace_scope_module.name, ) diff --git a/tests/conftest.py b/tests/conftest.py index 233fd03adb..a186cc87bd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -132,7 +132,7 @@ UNPRIVILEGED_PASSWORD, UNPRIVILEGED_USER, ) -from utilities.constants.storage import StorageClassNames +from utilities.constants.storage import BIND_IMMEDIATE_ANNOTATION, StorageClassNames from utilities.constants.timeouts import ( TIMEOUT_3MIN, TIMEOUT_4MIN, @@ -200,6 +200,7 @@ from utilities.sanity import cluster_sanity from utilities.ssp import get_data_import_crons, get_ssp_resource from utilities.storage import ( + construct_datavolume_source_dict, create_or_update_data_source, data_volume, get_default_storage_class, @@ -2518,13 +2519,15 @@ def dvs_for_upgrade( client=admin_client, name=f"dv-for-product-upgrade-{storage_class}", namespace=golden_images_namespace_name, - source="http", + source_dict=construct_datavolume_source_dict( + source="http", + url=rhel_latest_os_params["rhel_image_path"], + secret_name=artifactory_secret.name, + cert_configmap_name=artifactory_config_map.name, + ), storage_class=storage_class, - secret=artifactory_secret, - cert_configmap=artifactory_config_map.name, - url=rhel_latest_os_params["rhel_image_path"], size=rhel_latest_os_params["rhel_dv_size"], - bind_immediate_annotation=True, + annotations=BIND_IMMEDIATE_ANNOTATION, api_name="storage", ) dv.create() diff --git a/tests/data_protection/oadp/conftest.py b/tests/data_protection/oadp/conftest.py index d378a63a19..e2b1e987ba 100644 --- a/tests/data_protection/oadp/conftest.py +++ b/tests/data_protection/oadp/conftest.py @@ -29,6 +29,7 @@ ) from utilities.storage import ( check_upload_virtctl_result, + construct_datavolume_source_dict, create_dv, create_vm_from_dv, get_downloaded_artifact, @@ -48,10 +49,12 @@ def imported_dv_in_progress_second_namespace( with create_dv( dv_name="imported-dv", namespace=namespace_for_backup2.name, + source="http", url=rhel9_http_image_url, size=Images.Rhel.DEFAULT_DV_SIZE, storage_class=storage_class_for_snapshot, client=namespace_for_backup2.client, + use_artifactory=True, ) as dv: yield dv @@ -165,16 +168,18 @@ def windows_vm_with_data_volume_template( name="oadp-windows-dv", namespace=namespace_for_backup.name, storage_class=snapshot_storage_class_name_scope_module, - source="registry", - url=( - f"{get_test_artifact_server_url(schema='registry')}/" - f"{py_config['latest_windows_os_dict'][CONTAINER_DISK_IMAGE_PATH_STR]}" + source_dict=construct_datavolume_source_dict( + source="registry", + url=( + f"{get_test_artifact_server_url(schema='registry')}/" + f"{py_config['latest_windows_os_dict'][CONTAINER_DISK_IMAGE_PATH_STR]}" + ), + secret_name=artifactory_secret.name, + cert_configmap_name=artifactory_config_map.name, ), size=Images.Windows.CONTAINER_DISK_DV_SIZE, client=admin_client, api_name="storage", - secret=artifactory_secret, - cert_configmap=artifactory_config_map.name, ) dv.to_dict() @@ -299,7 +304,8 @@ def cloned_rhel_dv(imported_dv_second_namespace): dv_name="cloned-dv", namespace=imported_dv_second_namespace.namespace, size=imported_dv_second_namespace.size, - source_pvc=imported_dv_second_namespace.name, + source_pvc_name=imported_dv_second_namespace.name, + source_pvc_namespace=imported_dv_second_namespace.namespace, storage_class=imported_dv_second_namespace.storage_class, client=imported_dv_second_namespace.client, ) as cdv: diff --git a/tests/infrastructure/golden_images/update_boot_source/test_ssp_common_templates_boot_sources.py b/tests/infrastructure/golden_images/update_boot_source/test_ssp_common_templates_boot_sources.py index 827a6d6323..1b23f867a4 100644 --- a/tests/infrastructure/golden_images/update_boot_source/test_ssp_common_templates_boot_sources.py +++ b/tests/infrastructure/golden_images/update_boot_source/test_ssp_common_templates_boot_sources.py @@ -16,6 +16,7 @@ DEFAULT_FEDORA_REGISTRY_URL, OS_FLAVOR_FEDORA, ) +from utilities.constants.storage import BIND_IMMEDIATE_ANNOTATION from utilities.constants.timeouts import ( TIMEOUT_5MIN, TIMEOUT_5SEC, @@ -23,6 +24,7 @@ from utilities.infra import ( validate_os_info_vmi_vs_linux_os, ) +from utilities.storage import construct_datavolume_source_dict from utilities.virt import VirtualMachineForTestsFromTemplate, running_vm LOGGER = logging.getLogger(__name__) @@ -119,11 +121,10 @@ def imported_fedora_dv(admin_client, golden_images_namespace, fedora_data_source name=fedora_data_source.name, namespace=golden_images_namespace.name, api_name="storage", - source="registry", + source_dict=construct_datavolume_source_dict(source="registry", url=DEFAULT_FEDORA_REGISTRY_URL), size=Images.Fedora.DEFAULT_DV_SIZE, storage_class=py_config["default_storage_class"], - url=DEFAULT_FEDORA_REGISTRY_URL, - bind_immediate_annotation=True, + annotations=BIND_IMMEDIATE_ANNOTATION, ) as dv: dv.wait_for_dv_success() yield dv diff --git a/tests/infrastructure/golden_images/update_boot_source/test_ssp_data_sources.py b/tests/infrastructure/golden_images/update_boot_source/test_ssp_data_sources.py index f1f9a92832..9c67333220 100644 --- a/tests/infrastructure/golden_images/update_boot_source/test_ssp_data_sources.py +++ b/tests/infrastructure/golden_images/update_boot_source/test_ssp_data_sources.py @@ -27,12 +27,14 @@ from utilities.constants.hco import DATA_SOURCE_NAME from utilities.constants.images import DEFAULT_FEDORA_REGISTRY_URL from utilities.constants.pytest import QUARANTINED +from utilities.constants.storage import BIND_IMMEDIATE_ANNOTATION from utilities.constants.timeouts import ( TIMEOUT_5MIN, TIMEOUT_10MIN, ) from utilities.exceptions import ResourceValueError from utilities.ssp import wait_for_condition_message_value +from utilities.storage import construct_datavolume_source_dict LOGGER = logging.getLogger(__name__) @@ -51,14 +53,16 @@ def dv_for_data_source(name, data_source, admin_client): client=admin_client, name=name, namespace=data_source.namespace, - # underlying OS is not relevant - url=get_http_image_url(image_directory=Images.Cirros.DIR, image_name=Images.Cirros.QCOW2_IMG), - source="http", - secret=artifactory_secret, - cert_configmap=artifactory_config_map.name, + source_dict=construct_datavolume_source_dict( + # underlying OS is not relevant + source="http", + url=get_http_image_url(image_directory=Images.Cirros.DIR, image_name=Images.Cirros.QCOW2_IMG), + secret_name=artifactory_secret.name, + cert_configmap_name=artifactory_config_map.name, + ), size=Images.Cirros.DEFAULT_DV_SIZE, storage_class=py_config["default_storage_class"], - bind_immediate_annotation=True, + annotations=BIND_IMMEDIATE_ANNOTATION, api_name="storage", ) as dv: dv.wait_for_dv_success() diff --git a/tests/infrastructure/instance_types/conftest.py b/tests/infrastructure/instance_types/conftest.py index 0fe82b1859..1c1bd40f28 100644 --- a/tests/infrastructure/instance_types/conftest.py +++ b/tests/infrastructure/instance_types/conftest.py @@ -30,6 +30,7 @@ from utilities.constants.os_matrix import CONTAINER_DISK_IMAGE_PATH_STR from utilities.constants.timeouts import TIMEOUT_15MIN from utilities.storage import ( + construct_datavolume_source_dict, create_dummy_first_consumer_pod, data_volume_template_with_source_ref_dict, generate_data_source_dict, @@ -158,13 +159,17 @@ def latest_windows_data_volume( name="latest-windows", namespace=windows_test_images_namespace_role_binding.namespace, api_name="storage", - source="registry", + source_dict=construct_datavolume_source_dict( + source="registry", + url=( + f"{get_test_artifact_server_url(schema='registry')}/" + f"{py_config['latest_windows_os_dict'][CONTAINER_DISK_IMAGE_PATH_STR]}" + ), + secret_name=windows_namespace_artifactory_secret_and_configmap["secret"].name, + cert_configmap_name=windows_namespace_artifactory_secret_and_configmap["config_map"].name, + ), size=Images.Windows.CONTAINER_DISK_DV_SIZE, storage_class=default_sc.name, - url=f"{get_test_artifact_server_url(schema='registry')}/" - f"{py_config['latest_windows_os_dict'][CONTAINER_DISK_IMAGE_PATH_STR]}", - secret=windows_namespace_artifactory_secret_and_configmap["secret"], - cert_configmap=windows_namespace_artifactory_secret_and_configmap["config_map"].name, ) if windows_data_volume.exists: yield windows_data_volume diff --git a/tests/infrastructure/instance_types/supported_os/conftest.py b/tests/infrastructure/instance_types/supported_os/conftest.py index 067ab562c1..7664973c3d 100644 --- a/tests/infrastructure/instance_types/supported_os/conftest.py +++ b/tests/infrastructure/instance_types/supported_os/conftest.py @@ -18,7 +18,7 @@ CONTAINER_DISK_IMAGE_PATH_STR, DATA_SOURCE_STR, ) -from utilities.storage import get_test_artifact_server_url +from utilities.storage import construct_datavolume_source_dict, get_test_artifact_server_url from utilities.virt import VirtualMachineForTests @@ -103,12 +103,14 @@ def windows_data_volume_template( name=f"{os_matrix_key}-dv", namespace=namespace.name, api_name="storage", - source="registry", + source_dict=construct_datavolume_source_dict( + source="registry", + url=f"{get_test_artifact_server_url(schema='registry')}/{os_params[CONTAINER_DISK_IMAGE_PATH_STR]}", + secret_name=secret.name, + cert_configmap_name=cert.name, + ), size=Images.Windows.CONTAINER_DISK_DV_SIZE, storage_class=py_config["default_storage_class"], - url=f"{get_test_artifact_server_url(schema='registry')}/{os_params[CONTAINER_DISK_IMAGE_PATH_STR]}", - secret=secret, - cert_configmap=cert.name, ) win_dv.to_dict() yield win_dv diff --git a/tests/observability/metrics/utils.py b/tests/observability/metrics/utils.py index 60dbf3c9ca..4452f17814 100644 --- a/tests/observability/metrics/utils.py +++ b/tests/observability/metrics/utils.py @@ -60,6 +60,7 @@ TIMEOUT_40MIN, ) from utilities.monitoring import get_metrics_value +from utilities.storage import construct_datavolume_source_dict from utilities.virt import VirtualMachineForTests, running_vm LOGGER = logging.getLogger(__name__) @@ -661,12 +662,14 @@ def create_windows11_wsl2_vm( name=dv_name, namespace=namespace, api_name="storage", - source=REGISTRY_STR, + source_dict=construct_datavolume_source_dict( + source=REGISTRY_STR, + url=f"{get_test_artifact_server_url(schema=REGISTRY_STR)}/docker-local/windows-qe/win_11:virtio", + secret_name=artifactory_secret.name, + cert_configmap_name=artifactory_config_map.name, + ), size=Images.Windows.CONTAINER_DISK_DV_SIZE, storage_class=storage_class, - url=f"{get_test_artifact_server_url(schema=REGISTRY_STR)}/docker-local/windows-qe/win_11:virtio", - secret=artifactory_secret, - cert_configmap=artifactory_config_map.name, ) dv.to_dict() base_preference = VirtualMachineClusterPreference(client=client, name=windows_preference_name) diff --git a/tests/scale/test_scale_benchmark.py b/tests/scale/test_scale_benchmark.py index 773eba28a6..78e74c7a4d 100644 --- a/tests/scale/test_scale_benchmark.py +++ b/tests/scale/test_scale_benchmark.py @@ -46,7 +46,7 @@ create_ns, ) from utilities.must_gather import run_must_gather -from utilities.storage import generate_data_source_dict, get_test_artifact_server_url +from utilities.storage import construct_datavolume_source_dict, generate_data_source_dict, get_test_artifact_server_url from utilities.virt import ( VirtualMachineForTestsFromTemplate, verify_vm_migrated, @@ -274,12 +274,14 @@ def _delete_resources(): namespace=golden_images_namespace.name, storage_class=SCALE_STORAGE_TYPES[storage_type], api_name="storage", - url=f"{get_test_artifact_server_url()}{dv_info['url']}", + source_dict=construct_datavolume_source_dict( + source="http", + url=f"{get_test_artifact_server_url()}{dv_info['url']}", + secret_name=artifactory_secret.name, + cert_configmap_name=artifactory_config_map.name, + ), size=dv_info["size"], client=admin_client, - source="http", - secret=artifactory_secret, - cert_configmap=artifactory_config_map.name, ) golden_images_scale_dv.deploy() dvs_list.append(golden_images_scale_dv) diff --git a/tests/storage/cdi_clone/test_clone.py b/tests/storage/cdi_clone/test_clone.py index 3ecbdc7458..678f1b1f35 100644 --- a/tests/storage/cdi_clone/test_clone.py +++ b/tests/storage/cdi_clone/test_clone.py @@ -85,7 +85,8 @@ def test_successful_clone_of_large_image( dv_name="dv-target", namespace=namespace.name, size=data_volume_multi_storage_scope_function.size, - source_pvc=data_volume_multi_storage_scope_function.name, + source_pvc_name=data_volume_multi_storage_scope_function.name, + source_pvc_namespace=data_volume_multi_storage_scope_function.namespace, storage_class=data_volume_multi_storage_scope_function.storage_class, client=namespace.client, ) as cdv: @@ -173,7 +174,8 @@ def test_successful_vm_from_cloned_dv_windows( dv_name="dv-target", namespace=data_volume_multi_storage_scope_function.namespace, size=data_volume_multi_storage_scope_function.size, - source_pvc=data_volume_multi_storage_scope_function.name, + source_pvc_name=data_volume_multi_storage_scope_function.name, + source_pvc_namespace=data_volume_multi_storage_scope_function.namespace, storage_class=data_volume_multi_storage_scope_function.storage_class, ) as cdv: cdv.wait_for_dv_success(timeout=WINDOWS_CLONE_TIMEOUT) @@ -198,7 +200,7 @@ def test_successful_vm_from_cloned_dv_windows( ), pytest.param( { - "dv_name": "dv-source-win", + "dv_name": f"dv-source-{OS_FLAVOR_WINDOWS}", "image": f"{Images.Windows.DIR}/{Images.Windows.WIN11_IMG}", "dv_size": Images.Windows.DEFAULT_DV_SIZE, }, @@ -220,11 +222,12 @@ def test_successful_snapshot_clone( dv_name="dv-target", namespace=namespace, size=data_volume_snapshot_capable_storage_scope_function.size, - source_pvc=data_volume_snapshot_capable_storage_scope_function.name, + source_pvc_name=data_volume_snapshot_capable_storage_scope_function.name, + source_pvc_namespace=data_volume_snapshot_capable_storage_scope_function.namespace, storage_class=storage_class, ) as cdv: cdv.wait_for_dv_success() - if OS_FLAVOR_WINDOWS not in data_volume_snapshot_capable_storage_scope_function.url.split("/")[-1]: + if OS_FLAVOR_WINDOWS not in data_volume_snapshot_capable_storage_scope_function.name: with create_vm_from_dv( client=unprivileged_client, dv=cdv, diff --git a/tests/storage/cdi_import/conftest.py b/tests/storage/cdi_import/conftest.py index e405be127e..d0873d0b76 100644 --- a/tests/storage/cdi_import/conftest.py +++ b/tests/storage/cdi_import/conftest.py @@ -7,6 +7,7 @@ import pytest from ocp_resources.datavolume import DataVolume +from ocp_resources.resource import Resource from pytest_testconfig import py_config from tests.storage.cdi_import.utils import wait_dv_and_get_importer, wait_for_multus_network_status @@ -25,7 +26,7 @@ from utilities.constants.timeouts import TIMEOUT_1MIN from utilities.infra import NON_EXIST_URL from utilities.network import network_device, network_nad -from utilities.storage import create_dv, sc_volume_binding_mode_is_wffc +from utilities.storage import construct_datavolume_source_dict, create_dv, sc_volume_binding_mode_is_wffc from utilities.virt import VirtualMachineForTests LOGGER = logging.getLogger(__name__) @@ -64,6 +65,7 @@ def dv_non_exist_url(namespace, storage_class_name_scope_module): with create_dv( dv_name=f"cnv-876-{storage_class_name_scope_module}", namespace=namespace.name, + source="http", url=NON_EXIST_URL, size=DEFAULT_DV_SIZE, storage_class=storage_class_name_scope_module, @@ -83,12 +85,13 @@ def dv_from_http_import( with create_dv( dv_name=f"{request.param.get('dv_name', 'http-dv')}-{storage_class_name_scope_module}", namespace=namespace.name, + source="http", url=get_file_url( url=images_internal_http_server[request.param.get("source", HTTP)], file_name=request.param["file_name"], ), content_type=request.param.get("content_type", DataVolume.ContentType.KUBEVIRT), - cert_configmap=request.param.get("configmap_name"), + cert_configmap_name=request.param.get("configmap_name"), size=request.param.get("size", DEFAULT_DV_SIZE), volume_mode=request.param.get("volume_mode"), storage_class=storage_class_name_scope_module, @@ -119,7 +122,7 @@ def created_blank_dv_list(unprivileged_client, namespace, storage_class_name_sco for dv_index in range(number_of_dvs): dv = DataVolume( client=unprivileged_client, - source="blank", + source_dict=construct_datavolume_source_dict(source="blank"), name=f"dv-{dv_index}", namespace=namespace.name, size=Images.Fedora.DEFAULT_DV_SIZE, @@ -173,10 +176,9 @@ def dvs_and_vms_from_public_registry(unprivileged_client, namespace, storage_cla for name in ("dv1", "dv2", "dv3"): dv = DataVolume( client=unprivileged_client, - source=REGISTRY_STR, + source_dict=construct_datavolume_source_dict(source=REGISTRY_STR, url=QUAY_FEDORA_CONTAINER_IMAGE), name=f"import-public-registry-quay-{name}", namespace=namespace.name, - url=QUAY_FEDORA_CONTAINER_IMAGE, size=Images.Fedora.DEFAULT_DV_SIZE, storage_class=storage_class_name_scope_function, api_name="storage", @@ -216,7 +218,7 @@ def importer_pod_annotations(admin_client, namespace, linux_nad): url=QUAY_FEDORA_CONTAINER_IMAGE, size=Images.Fedora.DEFAULT_DV_SIZE, storage_class=py_config["default_storage_class"], - multus_annotation=linux_nad.name, + annotations={f"{Resource.ApiGroup.K8S_V1_CNI_CNCF_IO}/networks": linux_nad.name}, client=namespace.client, ) as dv: importer_pod = wait_dv_and_get_importer(dv=dv, admin_client=admin_client) diff --git a/tests/storage/cdi_import/test_import_http.py b/tests/storage/cdi_import/test_import_http.py index 39b5705122..ada0213c9c 100644 --- a/tests/storage/cdi_import/test_import_http.py +++ b/tests/storage/cdi_import/test_import_http.py @@ -75,6 +75,7 @@ def test_empty_url(namespace, storage_class_name_scope_module, unprivileged_clie client=unprivileged_client, dv_name=f"cnv-674-{storage_class_name_scope_module}", namespace=namespace.name, + source="http", url="", size=DEFAULT_DV_SIZE, storage_class=storage_class_name_scope_module, @@ -179,10 +180,11 @@ def test_successful_import_basic_auth( client=admin_client, dv_name="import-http-dv", namespace=namespace.name, + source="http", url=get_file_url(url=images_internal_http_server["http_auth"], file_name=file_name), content_type=content_type, size=DEFAULT_DV_SIZE, - secret=internal_http_secret, + secret_name=internal_http_secret.name, storage_class=storage_class_name_scope_module, ) as dv: dv.wait_for_dv_success() @@ -265,7 +267,7 @@ def test_certconfigmap_incorrect_cert( "source": HTTP, "image": ALPINE_QCOW2_IMG, "dv_size": DEFAULT_DV_SIZE, - "cert_configmap": "wrong_name", + "cert_configmap_name": "wrong_name", "wait": False, }, marks=pytest.mark.polarion("CNV-2815"), diff --git a/tests/storage/cross_cluster_live_migration/conftest.py b/tests/storage/cross_cluster_live_migration/conftest.py index fb7958f3e1..1b5a344718 100644 --- a/tests/storage/cross_cluster_live_migration/conftest.py +++ b/tests/storage/cross_cluster_live_migration/conftest.py @@ -49,6 +49,7 @@ from utilities.constants.timeouts import TIMEOUT_1MIN, TIMEOUT_30SEC from utilities.infra import create_ns, get_hyperconverged_resource from utilities.storage import ( + construct_datavolume_source_dict, data_volume_template_with_source_ref_dict, write_file, ) @@ -532,12 +533,14 @@ def vm_for_cclm_windows_with_instance_type( name="dv-windows", namespace=remote_cluster_source_test_namespace.name, api_name="storage", - source="registry", + source_dict=construct_datavolume_source_dict( + source="registry", + url=f"{get_test_artifact_server_url(schema='registry')}/{WINDOWS_2022[CONTAINER_DISK_IMAGE_PATH_STR]}", + secret_name=remote_cluster_artifactory_secret_scope_class.name, + cert_configmap_name=remote_cluster_artifactory_config_map_scope_class.name, + ), size=Images.Windows.CONTAINER_DISK_DV_SIZE, storage_class=remote_cluster_source_storage_class, - url=f"{get_test_artifact_server_url(schema='registry')}/{WINDOWS_2022[CONTAINER_DISK_IMAGE_PATH_STR]}", - secret=remote_cluster_artifactory_secret_scope_class, - cert_configmap=remote_cluster_artifactory_config_map_scope_class.name, ) dv.to_dict() dv.res["metadata"].pop("namespace", None) diff --git a/tests/storage/golden_image/test_golden_image.py b/tests/storage/golden_image/test_golden_image.py index 2e16a46328..e7099b9b83 100644 --- a/tests/storage/golden_image/test_golden_image.py +++ b/tests/storage/golden_image/test_golden_image.py @@ -158,8 +158,8 @@ def test_regular_user_cant_clone_dv_in_ns( namespace=golden_images_namespace, source=PVC, size=golden_image_data_volume_scope_module.size, - source_pvc=golden_image_data_volume_scope_module.pvc.name, - source_namespace=golden_images_namespace, + source_pvc_name=golden_image_data_volume_scope_module.pvc.name, + source_pvc_namespace=golden_images_namespace, client=unprivileged_client, storage_class=storage_class, ): diff --git a/tests/storage/hpp/utils.py b/tests/storage/hpp/utils.py index d1d40ffb31..6df7d81e62 100644 --- a/tests/storage/hpp/utils.py +++ b/tests/storage/hpp/utils.py @@ -108,10 +108,12 @@ def cirros_dv_on_hpp(dv_name, storage_class, namespace): with create_dv( dv_name=dv_name, namespace=namespace.name, + source="http", url=get_http_image_url(image_directory=Images.Cirros.DIR, image_name=Images.Cirros.QCOW2_IMG), size=Images.Cirros.DEFAULT_DV_SIZE, storage_class=storage_class, client=namespace.client, + use_artifactory=True, ) as dv: yield dv diff --git a/tests/storage/online_resize/conftest.py b/tests/storage/online_resize/conftest.py index e1c013a4d1..3ede35b277 100644 --- a/tests/storage/online_resize/conftest.py +++ b/tests/storage/online_resize/conftest.py @@ -70,7 +70,8 @@ def second_rhel_dv_for_online_resize(rhel_dv_for_online_resize, unprivileged_cli client=unprivileged_client, size=rhel_dv_for_online_resize.size, storage_class=rhel_dv_for_online_resize.storage_class, - source_pvc=rhel_dv_for_online_resize.name, + source_pvc_name=rhel_dv_for_online_resize.name, + source_pvc_namespace=rhel_dv_for_online_resize.namespace, ) as rhel_dv: yield rhel_dv diff --git a/tests/storage/online_resize/test_online_resize.py b/tests/storage/online_resize/test_online_resize.py index 1569b1c5d4..528878836d 100644 --- a/tests/storage/online_resize/test_online_resize.py +++ b/tests/storage/online_resize/test_online_resize.py @@ -105,7 +105,8 @@ def test_disk_expand_then_clone_fail( client=unprivileged_client, size=RHEL_DV_SIZE, storage_class=rhel_dv_for_online_resize.storage_class, - source_pvc=rhel_dv_for_online_resize.name, + source_pvc_name=rhel_dv_for_online_resize.name, + source_pvc_namespace=rhel_dv_for_online_resize.namespace, ) as dv: for sample in TimeoutSampler( wait_timeout=TIMEOUT_1MIN, @@ -150,7 +151,8 @@ def test_disk_expand_then_clone_success( client=unprivileged_client, size=rhel_dv_for_online_resize.pvc.instance.spec.resources.requests.storage, storage_class=rhel_dv_for_online_resize.storage_class, - source_pvc=rhel_dv_for_online_resize.name, + source_pvc_name=rhel_dv_for_online_resize.name, + source_pvc_namespace=rhel_dv_for_online_resize.namespace, ) as cdv: cdv.wait_for_condition( condition=DataVolume.Condition.Type.READY, diff --git a/tests/storage/restricted_namespace_cloning/conftest.py b/tests/storage/restricted_namespace_cloning/conftest.py index 374b64d5b7..530fd16213 100644 --- a/tests/storage/restricted_namespace_cloning/conftest.py +++ b/tests/storage/restricted_namespace_cloning/conftest.py @@ -35,7 +35,7 @@ from utilities.constants.pytest import UNPRIVILEGED_USER from utilities.constants.storage import PVC from utilities.infra import create_ns -from utilities.storage import create_dv, get_dv_size_from_datasource +from utilities.storage import construct_datavolume_source_dict, create_dv, get_dv_size_from_datasource from utilities.virt import VirtualMachineForTests, running_vm @@ -98,10 +98,12 @@ def data_volume_clone_settings(destination_namespace, dv_cloned_from_datasource) dv = DataVolume( name=f"{TARGET_DV}-{storage_class}", namespace=destination_namespace.name, - source=PVC, + source_dict=construct_datavolume_source_dict( + source=PVC, + source_pvc_name=dv_cloned_from_datasource.name, + source_pvc_namespace=dv_cloned_from_datasource.namespace, + ), size=dv_cloned_from_datasource.size, - source_pvc=dv_cloned_from_datasource.name, - source_namespace=dv_cloned_from_datasource.namespace, storage_class=storage_class, api_name="storage", ) @@ -284,8 +286,8 @@ def dv_cloned_by_unprivileged_user_in_the_same_namespace( namespace=namespace, source=PVC, size=dv_cloned_from_datasource.size, - source_pvc=dv_cloned_from_datasource.pvc.name, - source_namespace=namespace, + source_pvc_name=dv_cloned_from_datasource.pvc.name, + source_pvc_namespace=namespace, client=unprivileged_client, storage_class=storage_class_name_scope_module, ) as cdv: @@ -307,8 +309,8 @@ def dv_destination_cloned_from_pvc( namespace=destination_namespace.name, source=PVC, size=dv_cloned_from_datasource.size, - source_pvc=dv_cloned_from_datasource.pvc.name, - source_namespace=dv_cloned_from_datasource.namespace, + source_pvc_name=dv_cloned_from_datasource.pvc.name, + source_pvc_namespace=dv_cloned_from_datasource.namespace, client=unprivileged_client, storage_class=storage_class_name_scope_module, ) as cdv: diff --git a/tests/storage/restricted_namespace_cloning/test_restricted_namespace_cloning.py b/tests/storage/restricted_namespace_cloning/test_restricted_namespace_cloning.py index 0339262d56..2b8ec46a63 100644 --- a/tests/storage/restricted_namespace_cloning/test_restricted_namespace_cloning.py +++ b/tests/storage/restricted_namespace_cloning/test_restricted_namespace_cloning.py @@ -142,8 +142,8 @@ def test_user_permissions_negative( namespace=destination_namespace.name, storage_class=storage_class_name_scope_module, size=dv_cloned_from_datasource.size, - source_pvc=dv_cloned_from_datasource.pvc.name, - source_namespace=dv_cloned_from_datasource.namespace, + source_pvc_name=dv_cloned_from_datasource.pvc.name, + source_pvc_namespace=dv_cloned_from_datasource.namespace, unprivileged_client=unprivileged_client, ) @@ -171,8 +171,8 @@ def test_unprivileged_user_clone_same_namespace_negative( namespace=namespace.name, storage_class=storage_class_name_scope_module, size=dv_cloned_from_datasource.size, - source_pvc=dv_cloned_from_datasource.pvc.name, - source_namespace=dv_cloned_from_datasource.namespace, + source_pvc_name=dv_cloned_from_datasource.pvc.name, + source_pvc_namespace=dv_cloned_from_datasource.namespace, unprivileged_client=unprivileged_client, ) @@ -203,7 +203,7 @@ def test_user_permissions_only_for_dst_ns_negative( namespace=destination_namespace.name, storage_class=storage_class_name_scope_module, size=dv_cloned_from_datasource.size, - source_pvc=dv_cloned_from_datasource.pvc.name, - source_namespace=dv_cloned_from_datasource.namespace, + source_pvc_name=dv_cloned_from_datasource.pvc.name, + source_pvc_namespace=dv_cloned_from_datasource.namespace, unprivileged_client=unprivileged_client, ) diff --git a/tests/storage/restricted_namespace_cloning/utils.py b/tests/storage/restricted_namespace_cloning/utils.py index 589eb37644..d6fd10fd50 100644 --- a/tests/storage/restricted_namespace_cloning/utils.py +++ b/tests/storage/restricted_namespace_cloning/utils.py @@ -31,8 +31,8 @@ def create_dv_negative( namespace: str, storage_class: str, size: str, - source_pvc: str, - source_namespace: str, + source_pvc_name: str, + source_pvc_namespace: str, unprivileged_client: DynamicClient, ) -> None: with pytest.raises( @@ -44,8 +44,8 @@ def create_dv_negative( namespace=namespace, source=PVC, size=size, - source_pvc=source_pvc, - source_namespace=source_namespace, + source_pvc_name=source_pvc_name, + source_pvc_namespace=source_pvc_namespace, client=unprivileged_client, storage_class=storage_class, ): diff --git a/tests/storage/storage_migration/conftest.py b/tests/storage/storage_migration/conftest.py index f94987771d..7d79adac9d 100644 --- a/tests/storage/storage_migration/conftest.py +++ b/tests/storage/storage_migration/conftest.py @@ -31,6 +31,7 @@ from utilities.constants.timeouts import TIMEOUT_2MIN, TIMEOUT_5SEC from utilities.infra import create_ns from utilities.storage import ( + construct_datavolume_source_dict, create_dv, data_volume_template_with_source_ref_dict, virtctl_volume, @@ -169,13 +170,15 @@ def vm_for_storage_class_migration_from_template_with_dv( dv = DataVolume( name="dv-rhel-imported", namespace=namespace.name, - source="http", - url=rhel_latest_os_params["rhel_image_path"], + source_dict=construct_datavolume_source_dict( + source="http", + url=rhel_latest_os_params["rhel_image_path"], + secret_name=artifactory_secret_scope_module.name, + cert_configmap_name=artifactory_config_map_scope_module.name, + ), size=Images.Rhel.DEFAULT_DV_SIZE, storage_class=source_storage_class, api_name="storage", - secret=artifactory_secret_scope_module, - cert_configmap=artifactory_config_map_scope_module.name, ) dv.to_dict() with VirtualMachineForTests( @@ -365,14 +368,16 @@ def windows_vm_with_vtpm_for_storage_migration( name="windows-11-dv", namespace=namespace.name, storage_class=source_storage_class, - source="http", - # Using WSL image to avoid the issue of the Windows VM not being able to boot - url=get_http_image_url(image_directory=Images.Windows.DIR, image_name=Images.Windows.WIN11_WSL2_IMG), + source_dict=construct_datavolume_source_dict( + source="http", + # Using WSL image to avoid the issue of the Windows VM not being able to boot + url=get_http_image_url(image_directory=Images.Windows.DIR, image_name=Images.Windows.WIN11_WSL2_IMG), + secret_name=artifactory_secret_scope_module.name, + cert_configmap_name=artifactory_config_map_scope_module.name, + ), size=Images.Windows.DEFAULT_DV_SIZE, client=unprivileged_client, api_name="storage", - secret=artifactory_secret_scope_module, - cert_configmap=artifactory_config_map_scope_module.name, ) dv.to_dict() with VirtualMachineForTests( diff --git a/tests/storage/test_cdi_certificate.py b/tests/storage/test_cdi_certificate.py index 10b05177c2..5ea0e247c5 100644 --- a/tests/storage/test_cdi_certificate.py +++ b/tests/storage/test_cdi_certificate.py @@ -229,7 +229,8 @@ def test_import_clone_after_certs_renewal( dv_name="dv-target", namespace=namespace.name, size=data_volume_multi_storage_scope_module.size, - source_pvc=data_volume_multi_storage_scope_module.name, + source_pvc_name=data_volume_multi_storage_scope_module.name, + source_pvc_namespace=data_volume_multi_storage_scope_module.namespace, storage_class=data_volume_multi_storage_scope_module.storage_class, ) as cdv: cdv.wait_for_dv_success(timeout=TIMEOUT_3MIN) diff --git a/tests/storage/test_cdi_resources.py b/tests/storage/test_cdi_resources.py index c41eea1410..9227aba404 100644 --- a/tests/storage/test_cdi_resources.py +++ b/tests/storage/test_cdi_resources.py @@ -233,7 +233,8 @@ def test_cloner_pods_cdi_label( dv_name="dv-target", namespace=data_volume_without_snapshot_capability_scope_function.namespace, size=data_volume_without_snapshot_capability_scope_function.size, - source_pvc=data_volume_without_snapshot_capability_scope_function.name, + source_pvc_name=data_volume_without_snapshot_capability_scope_function.name, + source_pvc_namespace=data_volume_without_snapshot_capability_scope_function.namespace, storage_class=data_volume_without_snapshot_capability_scope_function.storage_class, ) as cdv: cdv.wait_for_status(status=DataVolume.Status.CLONE_IN_PROGRESS, timeout=TIMEOUT_10MIN) diff --git a/tests/storage/test_priority_class.py b/tests/storage/test_priority_class.py index 91fa1294d9..623f41b8be 100644 --- a/tests/storage/test_priority_class.py +++ b/tests/storage/test_priority_class.py @@ -8,6 +8,7 @@ from tests.storage.utils import get_importer_pod from utilities.artifactory import get_test_artifact_server_url from utilities.constants import Images +from utilities.storage import construct_datavolume_source_dict from utilities.virt import VirtualMachineForTests, fedora_vm_body @@ -31,10 +32,12 @@ def priority_class(request, admin_client): @pytest.fixture() def dv_dict(namespace, priority_class, unprivileged_client): dv = DataVolume( - source="http", + source_dict=construct_datavolume_source_dict( + source="http", + url=f"{get_test_artifact_server_url()}{RHEL_LATEST['image_path']}", + ), name="priority-dv", namespace=namespace.name, - url=f"{get_test_artifact_server_url()}{RHEL_LATEST['image_path']}", size=RHEL_LATEST["dv_size"], storage_class=py_config["default_storage_class"], client=unprivileged_client, diff --git a/tests/storage/test_wffc.py b/tests/storage/test_wffc.py index a3d6c5195f..7fe48be8bc 100644 --- a/tests/storage/test_wffc.py +++ b/tests/storage/test_wffc.py @@ -16,6 +16,7 @@ add_dv_to_vm, check_disk_count_in_vm, check_upload_virtctl_result, + construct_datavolume_source_dict, create_dv, create_vm_from_dv, data_volume_template_with_source_ref_dict, @@ -60,7 +61,7 @@ def blank_dv_template_wffc_scope_function(request, namespace, wffc_storage_class blank_dv_template = DataVolume( name=f"dv-{request.param['dv_name']}", namespace=namespace.name, - source="blank", + source_dict=construct_datavolume_source_dict(source="blank"), size=DEFAULT_BLANK_DV_SIZE, storage_class=wffc_storage_class_name_scope_module, api_name="storage", diff --git a/tests/storage/utils.py b/tests/storage/utils.py index 2e73e6cac6..8fbfc3994a 100644 --- a/tests/storage/utils.py +++ b/tests/storage/utils.py @@ -43,6 +43,7 @@ from utilities.ssp import validate_os_info_vmi_vs_windows_os from utilities.storage import ( PodWithPVC, + construct_datavolume_source_dict, create_dv, get_containers_for_pods_with_pvc, ) @@ -77,7 +78,7 @@ def import_image_to_dv( dv_name=dv_name, namespace=configmap.namespace, url=url, - cert_configmap=configmap.name, + cert_configmap_name=configmap.name, storage_class=py_config["default_storage_class"], client=client, ) as dv: @@ -360,13 +361,15 @@ def create_windows19_vm(dv_name, namespace, client, vm_name, cpu_model, storage_ name=dv_name, namespace=namespace, storage_class=storage_class, - source="http", - url=get_http_image_url(image_directory=Images.Windows.UEFI_WIN_DIR, image_name=Images.Windows.WIN2k19_IMG), + source_dict=construct_datavolume_source_dict( + source="http", + url=get_http_image_url(image_directory=Images.Windows.UEFI_WIN_DIR, image_name=Images.Windows.WIN2k19_IMG), + secret_name=artifactory_secret.name, + cert_configmap_name=artifactory_config_map.name, + ), size=Images.Windows.DEFAULT_DV_SIZE, client=client, api_name="storage", - secret=artifactory_secret, - cert_configmap=artifactory_config_map.name, ) dv.to_dict() with VirtualMachineForTestsFromTemplate( diff --git a/tests/storage/vm_export/utils.py b/tests/storage/vm_export/utils.py index 9755440ae0..4b91d45ed5 100644 --- a/tests/storage/vm_export/utils.py +++ b/tests/storage/vm_export/utils.py @@ -15,6 +15,7 @@ from pyhelper_utils.shell import run_command from pytest_testconfig import config as py_config +from utilities.constants.storage import BIND_IMMEDIATE_ANNOTATION from utilities.constants.timeouts import TIMEOUT_1MIN from utilities.storage import create_dv @@ -57,7 +58,7 @@ def create_blank_dv_by_specific_user( size="1Gi", storage_class=py_config["default_storage_class"], consume_wffc=False, - bind_immediate=True, + annotations=BIND_IMMEDIATE_ANNOTATION, client=client, ) as dv: dv.wait_for_dv_success(timeout=TIMEOUT_1MIN) diff --git a/tests/utils.py b/tests/utils.py index b6a5489ab3..b9bc63c0e3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -68,6 +68,7 @@ ExecCommandOnPod, ) from utilities.os_utils import get_windows_container_disk_path +from utilities.storage import construct_datavolume_source_dict from utilities.virt import ( VirtualMachineForTests, fedora_vm_body, @@ -569,14 +570,16 @@ def create_cirros_vm( dv = DataVolume( name=dv_name, namespace=namespace, - source="http", - url=get_http_image_url(image_directory=Images.Cirros.DIR, image_name=Images.Cirros.QCOW2_IMG), + source_dict=construct_datavolume_source_dict( + source="http", + url=get_http_image_url(image_directory=Images.Cirros.DIR, image_name=Images.Cirros.QCOW2_IMG), + secret_name=artifactory_secret.name, + cert_configmap_name=artifactory_config_map.name, + ), storage_class=storage_class, size=Images.Cirros.DEFAULT_DV_SIZE, api_name="storage", volume_mode=volume_mode, - secret=artifactory_secret, - cert_configmap=artifactory_config_map.name, ) dv.to_dict() dv_metadata = dv.res["metadata"] @@ -736,13 +739,15 @@ def create_windows2022_dv_from_registry( name=dv_name, namespace=namespace, storage_class=storage_class, - source="registry", - url=f"{get_test_artifact_server_url(schema='registry')}/{get_windows_container_disk_path(os_value=WIN_2K22)}", + source_dict=construct_datavolume_source_dict( + source="registry", + url=f"{get_test_artifact_server_url(schema='registry')}/{get_windows_container_disk_path(os_value=WIN_2K22)}", + secret_name=artifactory_secret.name, + cert_configmap_name=artifactory_config_map.name, + ), size=Images.Windows.CONTAINER_DISK_DV_SIZE, client=client, api_name="storage", - secret=artifactory_secret, - cert_configmap=artifactory_config_map.name, ) dv.to_dict() diff --git a/tests/virt/cluster/longevity_tests/utils.py b/tests/virt/cluster/longevity_tests/utils.py index 83c7a95da3..7786b532c8 100644 --- a/tests/virt/cluster/longevity_tests/utils.py +++ b/tests/virt/cluster/longevity_tests/utils.py @@ -32,7 +32,7 @@ TIMEOUT_60MIN, ) from utilities.constants.virt import WIN_10 -from utilities.storage import get_test_artifact_server_url +from utilities.storage import construct_datavolume_source_dict, get_test_artifact_server_url from utilities.virt import ( VirtualMachineForTests, VirtualMachineForTestsFromTemplate, @@ -365,13 +365,15 @@ def create_multi_dvs(namespace, client, dv_params): name=dv_name, client=client, namespace=namespace_name, - source="http", + source_dict=construct_datavolume_source_dict( + source="http", + url=f"{get_test_artifact_server_url()}{dv[dv_name].get('image_path')}", + secret_name=artifactory_secret.name, + cert_configmap_name=artifactory_config_map.name, + ), size=dv[dv_name].get("dv_size"), storage_class=dv[dv_name].get("storage_class"), - url=f"{get_test_artifact_server_url()}{dv[dv_name].get('image_path')}", api_name="storage", - secret=artifactory_secret, - cert_configmap=artifactory_config_map.name, ) yield from deploy_and_wait_for_dvs(dv_dict=dvs) diff --git a/tests/virt/cluster/vm_cloning/test_vm_cloning.py b/tests/virt/cluster/vm_cloning/test_vm_cloning.py index 134b3e8560..963a54ea62 100644 --- a/tests/virt/cluster/vm_cloning/test_vm_cloning.py +++ b/tests/virt/cluster/vm_cloning/test_vm_cloning.py @@ -23,6 +23,7 @@ from utilities.storage import ( add_dv_to_vm, check_disk_count_in_vm, + construct_datavolume_source_dict, ) from utilities.virt import ( VirtualMachineForCloning, @@ -54,7 +55,7 @@ def dummy_dv_dict_for_vm_cloning(client, namespace): name="dummy-dv-for-clone", client=client, namespace=namespace.name, - source="blank", + source_dict=construct_datavolume_source_dict(source="blank"), size="10Gi", storage_class=py_config["default_storage_class"], api_name="storage", diff --git a/tests/virt/upgrade/conftest.py b/tests/virt/upgrade/conftest.py index dd9cc05b5a..34e21b751f 100644 --- a/tests/virt/upgrade/conftest.py +++ b/tests/virt/upgrade/conftest.py @@ -258,11 +258,13 @@ def windows_vm( client=admin_client, dv_name=latest_windows_dict["os_version"], namespace=py_config["golden_images_namespace"], + source="http", url=f"{get_test_artifact_server_url()}{latest_windows_dict['image_path']}", storage_class=py_config["default_storage_class"], access_modes=py_config["default_access_mode"], volume_mode=py_config["default_volume_mode"], size=latest_windows_dict["dv_size"], + use_artifactory=True, ) as dv: dv.wait_for_dv_success(timeout=TIMEOUT_30MIN) with DataSource( diff --git a/tests/virt/upgrade_custom/vgpu/conftest.py b/tests/virt/upgrade_custom/vgpu/conftest.py index f787e12493..a55504b5d4 100644 --- a/tests/virt/upgrade_custom/vgpu/conftest.py +++ b/tests/virt/upgrade_custom/vgpu/conftest.py @@ -44,11 +44,13 @@ def rhel_data_volume( client=admin_client, dv_name=RHEL_LATEST_OS, namespace=py_config["golden_images_namespace"], + source="http", url=f"{get_test_artifact_server_url()}{RHEL_LATEST['image_path']}", storage_class=py_config["default_storage_class"], access_modes=py_config["default_access_mode"], volume_mode=py_config["default_volume_mode"], size=RHEL_LATEST["dv_size"], + use_artifactory=True, ) as dv: dv.wait_for_dv_success(timeout=TIMEOUT_30MIN) yield dv diff --git a/tests/virt/utils.py b/tests/virt/utils.py index 2506e385fd..3adf5fe4f4 100644 --- a/tests/virt/utils.py +++ b/tests/virt/utils.py @@ -459,10 +459,12 @@ def get_or_create_golden_image_data_source( with create_dv( dv_name=data_source_name, namespace=golden_images_namespace.name, + source="http", storage_class=py_config["default_storage_class"], url=f"{get_test_artifact_server_url()}{os_dict['image_path']}", size=os_dict["dv_size"], client=admin_client, + use_artifactory=True, ) as dv: dv.wait_for_dv_success(timeout=TIMEOUT_30MIN) yield from create_or_update_data_source(admin_client=admin_client, dv=dv) diff --git a/utilities/oadp.py b/utilities/oadp.py index b735e7a26f..bd3d15d654 100644 --- a/utilities/oadp.py +++ b/utilities/oadp.py @@ -33,6 +33,7 @@ get_pod_by_name_prefix, unique_name, ) +from utilities.storage import construct_datavolume_source_dict from utilities.virt import VirtualMachineForTests, running_vm LOGGER = logging.getLogger(__name__) @@ -159,17 +160,19 @@ def create_rhel_vm( dv = DataVolume( name=dv_name, namespace=namespace, - source="http", - url=get_http_image_url( - image_directory=Images.Rhel.DIR, - image_name=rhel_image, + source_dict=construct_datavolume_source_dict( + source="http", + url=get_http_image_url( + image_directory=Images.Rhel.DIR, + image_name=rhel_image, + ), + secret_name=artifactory_secret.name, + cert_configmap_name=artifactory_config_map.name, ), storage_class=storage_class, size=Images.Rhel.DEFAULT_DV_SIZE, api_name="storage", volume_mode=volume_mode, - secret=artifactory_secret, - cert_configmap=artifactory_config_map.name, ) dv.to_dict() dv_metadata = dv.res["metadata"] diff --git a/utilities/storage.py b/utilities/storage.py index 3fe4d506ed..469bb596a6 100644 --- a/utilities/storage.py +++ b/utilities/storage.py @@ -40,6 +40,7 @@ from utilities.constants.images import OS_FLAVOR_WINDOWS from utilities.constants.networking import POD_CONTAINER_SPEC from utilities.constants.storage import ( + BIND_IMMEDIATE_ANNOTATION, CDI_LABEL, HOTPLUG_DISK_SERIAL, ) @@ -109,76 +110,177 @@ def create_dummy_first_consumer_pod(volume_mode=DataVolume.VolumeMode.FILE, dv=N ) +def construct_datavolume_source_dict( + source: str, + url: str | None = None, + secret_name: str | None = None, + cert_configmap_name: str | None = None, + source_pvc_name: str | None = None, + source_pvc_namespace: str | None = None, +) -> dict[str, Any]: + """ + Build a DataVolume source_dict. + + Args: + source: Source type ("http", "registry", "pvc", "blank", "upload"). + url: URL for http/registry sources. + secret_name: Optional Secret name for authentication (http/registry sources). + cert_configmap_name: Optional ConfigMap name for TLS certificates (http/registry sources). + source_pvc_name: PVC name for pvc source type. + source_pvc_namespace: Namespace of the source PVC. + + Returns: + dict[str, Any]: The constructed source_dict for DataVolume. + """ + if source == "http": + if not utilities.infra.url_excluded_from_validation(url): + validate_file_exists_in_url(url=url) + source_spec: dict[str, Any] = {"http": {"url": url}} + elif source == "registry": + source_spec = {"registry": {"url": url}} + elif source == "pvc": + pvc_spec: dict[str, Any] = {"name": source_pvc_name} + if source_pvc_namespace is not None: + pvc_spec["namespace"] = source_pvc_namespace + source_spec = {"pvc": pvc_spec} + elif source == "blank": + source_spec = {"blank": {}} + elif source == "upload": + source_spec = {"upload": {}} + else: + raise ValueError(f"Unsupported source type: {source}") + + if source in ("http", "registry"): + if secret_name: + source_spec[source]["secretRef"] = secret_name + if cert_configmap_name: + source_spec[source]["certConfigMap"] = cert_configmap_name + + return source_spec + + @contextmanager def create_dv( - dv_name, - namespace, - storage_class, - client, - volume_mode=None, - url=None, - source="http", - content_type=None, - size="5Gi", - secret=None, - cert_configmap=None, - hostpath_node=None, - access_modes=None, - source_pvc=None, - source_namespace=None, - multus_annotation=None, - teardown=True, - consume_wffc=True, - bind_immediate=None, - preallocation=None, - api_name="storage", - source_ref=None, -): + dv_name: str, + namespace: str, + client: DynamicClient, + storage_class: str | None = None, + access_modes: str | None = None, + volume_mode: str | None = None, + url: str | None = None, + source: str | None = None, + content_type: str | None = None, + size: str = "5Gi", + secret_name: str | None = None, + cert_configmap_name: str | None = None, + source_pvc_name: str | None = None, + source_pvc_namespace: str | None = None, + annotations: dict[str, str] | None = None, + teardown: bool = True, + consume_wffc: bool = True, + preallocation: bool | None = None, + api_name: str = "storage", + source_ref: dict[str, Any] | None = None, + source_dict: dict[str, Any] | None = None, + use_artifactory: bool = False, +) -> Generator[DataVolume]: + """ + Create and manage a DataVolume with optional Artifactory resource lifecycle. + + Context manager that constructs a DataVolume from either a pre-built ``source_dict``/``source_ref`` + or by building one via ``construct_datavolume_source_dict`` from the ``source`` parameter. + When ``use_artifactory`` is True for http/registry sources, creates namespace-scoped + Artifactory Secret and ConfigMap resources that are cleaned up on exit. + + Args: + dv_name: Name for the DataVolume resource. + namespace: Target Kubernetes namespace. + client: Kubernetes dynamic client. + storage_class: StorageClass name. + access_modes: PVC access mode (e.g. "ReadWriteOnce"). + volume_mode: PVC volume mode ("Block" or "Filesystem"). + url: Source URL for http/registry sources. + source: Source type ("http", "registry", "pvc", "blank", "upload"). + Used to construct ``source_dict`` when neither ``source_dict`` nor ``source_ref`` is provided. + content_type: CDI content type (e.g. "kubevirt"). + size: PVC size (default "5Gi"). + secret_name: Pre-existing Secret name for source authentication. + cert_configmap_name: Pre-existing ConfigMap name for TLS certificates. + source_pvc_name: PVC name for clone sources. + source_pvc_namespace: Namespace of the source PVC for clone sources. + annotations: Annotations dict to apply to the DataVolume. + teardown: Whether to delete the DataVolume on context exit. + consume_wffc: Whether to create a dummy consumer pod for WaitForFirstConsumer storage classes. + preallocation: Whether to preallocate the target PVC. + api_name: CDI API group name (default "storage"). + source_ref: Pre-built sourceRef dict for DataVolume. + source_dict: Pre-built source dict for DataVolume. + use_artifactory: Whether to create Artifactory Secret/ConfigMap for http/registry sources. + + Yields: + DataVolume: The created DataVolume resource. + + Raises: + ValueError: If ``source`` is not provided when ``source_dict`` and ``source_ref`` are both None. + """ artifactory_secret = None - cert_created = None + artifactory_config_map = None - if source_ref: - source = None - if source in ("http", "https"): - if not utilities.infra.url_excluded_from_validation(url): - # Make sure URL exists - validate_file_exists_in_url(url=url) - if not secret: - secret = utilities.artifactory.get_artifactory_secret(namespace=namespace, client=client) - artifactory_secret = secret - if not cert_configmap: - cert_created = utilities.artifactory.get_artifactory_config_map(namespace=namespace, client=client) - cert_configmap = cert_created.name - - with DataVolume( - source=source, - name=dv_name, - namespace=namespace, - url=url, - content_type=content_type, - size=size, - storage_class=storage_class, - cert_configmap=cert_configmap, - volume_mode=volume_mode, - hostpath_node=hostpath_node, - access_modes=access_modes, - secret=secret, - client=client, - source_pvc=source_pvc, - source_namespace=source_namespace, - bind_immediate_annotation=bind_immediate, - multus_annotation=multus_annotation, - teardown=teardown, - preallocation=preallocation, - api_name=api_name, - source_ref=source_ref, - ) as dv: - if sc_volume_binding_mode_is_wffc(sc=storage_class, client=client) and consume_wffc: - create_dummy_first_consumer_pod(dv=dv) - yield dv - utilities.artifactory.cleanup_artifactory_secret_and_config_map( - artifactory_secret=artifactory_secret, artifactory_config_map=cert_created - ) + try: + if source_dict is None and source_ref is None: + if not source: + raise ValueError("'source' is required when 'source_dict' and 'source_ref' are not provided") + + LOGGER.info("No 'source_dict' or 'source_ref' provided - will construct the 'source_dict'") + + if source in ("http", "registry") and use_artifactory: + LOGGER.info(f"Creating artifactory resources for DV '{dv_name}' in namespace '{namespace}'") + LOGGER.info(f"DV source is '{source}' with url: {url}") + + if not secret_name: + artifactory_secret = utilities.artifactory.get_artifactory_secret( + namespace=namespace, client=client + ) + secret_name = artifactory_secret.name + if not cert_configmap_name: + artifactory_config_map = utilities.artifactory.get_artifactory_config_map( + namespace=namespace, client=client + ) + cert_configmap_name = artifactory_config_map.name + + source_dict = construct_datavolume_source_dict( + source=source, + url=url, + secret_name=secret_name, + cert_configmap_name=cert_configmap_name, + source_pvc_name=source_pvc_name, + source_pvc_namespace=source_pvc_namespace, + ) + + with DataVolume( + name=dv_name, + namespace=namespace, + client=client, + content_type=content_type, + size=size, + storage_class=storage_class, + access_modes=access_modes, + volume_mode=volume_mode, + annotations=annotations, + teardown=teardown, + preallocation=preallocation, + api_name=api_name, + source_ref=source_ref, + source_dict=source_dict, + ) as dv: + if storage_class and sc_volume_binding_mode_is_wffc(sc=storage_class, client=client) and consume_wffc: + create_dummy_first_consumer_pod(dv=dv) + yield dv + + finally: + utilities.artifactory.cleanup_artifactory_secret_and_config_map( + artifactory_secret=artifactory_secret, artifactory_config_map=artifactory_config_map + ) def data_volume( @@ -260,13 +362,14 @@ def data_volume( "access_modes": params_dict.get("access_modes"), "volume_mode": params_dict.get("volume_mode"), "consume_wffc": consume_wffc, - "bind_immediate": bind_immediate, + "annotations": BIND_IMMEDIATE_ANNOTATION if bind_immediate else None, "preallocation": params_dict.get("preallocation", None), "url": url, "client": client, + "use_artifactory": source == "http", } - if params_dict.get("cert_configmap"): - dv_kwargs["cert_configmap"] = params_dict.get("cert_configmap") + if params_dict.get("cert_configmap_name"): + dv_kwargs["cert_configmap_name"] = params_dict["cert_configmap_name"] # Create dv with create_dv(**{k: v for k, v in dv_kwargs.items() if v is not None}) as dv: if params_dict.get("wait", True): @@ -554,12 +657,14 @@ def data_volume_template_dict( dv = DataVolume( name=target_dv_name, namespace=target_dv_namespace, - source="pvc", + source_dict=construct_datavolume_source_dict( + source="pvc", + source_pvc_name=source_dv.name, + source_pvc_namespace=source_dv.namespace, + ), storage_class=storage_class or source_dv_pvc_spec.storageClassName, volume_mode=volume_mode or source_dv_pvc_spec.volumeMode, size=size or source_dv.size, - source_pvc=source_dv.name, - source_namespace=source_dv.namespace, api_name=source_dv.api_name, ) dv.to_dict() diff --git a/utilities/unittests/conftest.py b/utilities/unittests/conftest.py index 716551300f..6259758bfb 100644 --- a/utilities/unittests/conftest.py +++ b/utilities/unittests/conftest.py @@ -13,8 +13,6 @@ from ocp_resources import resource from pytest_testconfig import config as py_config -import utilities - os.environ["OPENSHIFT_VIRTUALIZATION_TEST_IMAGES_ARCH"] = "amd64" # Add utilities to Python path for imports @@ -34,6 +32,7 @@ def _mock_get_client(*args: Any, **kwargs: Any) -> MagicMock: # type: ignore[mi # Note: utilities.hco is mocked here but test_hco.py will clear it and import real module mock_hco = MagicMock() mock_infra = MagicMock() +mock_virt = MagicMock() mock_data_collector = MagicMock() mock_data_collector.get_data_collector_base_directory = MagicMock(return_value="/tmp/data") mock_data_collector.get_data_collector_base = MagicMock(return_value="/tmp/data/") @@ -44,12 +43,16 @@ def _mock_get_client(*args: Any, **kwargs: Any) -> MagicMock: # type: ignore[mi sys.modules["utilities.hco"] = mock_hco sys.modules["utilities.infra"] = mock_infra +sys.modules["utilities.virt"] = mock_virt sys.modules["utilities.data_collector"] = mock_data_collector sys.modules["jira"] = mock_jira +import utilities + # Also set them as attributes of the utilities module for tests that need them utilities.hco = mock_hco # type: ignore[attr-defined] utilities.infra = mock_infra # type: ignore[attr-defined] +utilities.virt = mock_virt # type: ignore[attr-defined] utilities.data_collector = mock_data_collector # type: ignore[attr-defined] diff --git a/utilities/unittests/test_storage.py b/utilities/unittests/test_storage.py new file mode 100644 index 0000000000..f85a753ce2 --- /dev/null +++ b/utilities/unittests/test_storage.py @@ -0,0 +1,149 @@ +"""Unit tests for construct_datavolume_source_dict in utilities/storage.py""" + +import importlib +import sys +from unittest.mock import patch + +import pytest + +# Other test modules (test_hco, test_ssp) mock utilities.storage in sys.modules. +# Clear the mock and reimport the real module to test actual behavior. +if "utilities.storage" in sys.modules: + del sys.modules["utilities.storage"] + +import utilities.storage + +importlib.reload(utilities.storage) + +from utilities.storage import construct_datavolume_source_dict + + +class TestConstructDatavolumeSourceDictHttp: + @patch("utilities.storage.validate_file_exists_in_url") + @patch("utilities.infra.url_excluded_from_validation", return_value=False) + def test_http_source(self, mock_excluded, mock_validate): + result = construct_datavolume_source_dict(source="http", url="https://example.com/image.qcow2") + assert result == {"http": {"url": "https://example.com/image.qcow2"}} + mock_validate.assert_called_once_with(url="https://example.com/image.qcow2") + + @patch("utilities.storage.validate_file_exists_in_url") + @patch("utilities.infra.url_excluded_from_validation", return_value=True) + def test_http_source_excluded_from_validation(self, mock_excluded, mock_validate): + result = construct_datavolume_source_dict(source="http", url="https://internal.example.com/image.qcow2") + assert result == {"http": {"url": "https://internal.example.com/image.qcow2"}} + mock_validate.assert_not_called() + + @patch("utilities.storage.validate_file_exists_in_url") + @patch("utilities.infra.url_excluded_from_validation", return_value=False) + def test_http_source_with_secret(self, mock_excluded, mock_validate): + result = construct_datavolume_source_dict( + source="http", + url="https://example.com/image.qcow2", + secret_name="my-secret", + ) + assert result == {"http": {"url": "https://example.com/image.qcow2", "secretRef": "my-secret"}} + + @patch("utilities.storage.validate_file_exists_in_url") + @patch("utilities.infra.url_excluded_from_validation", return_value=False) + def test_http_source_with_cert_configmap(self, mock_excluded, mock_validate): + result = construct_datavolume_source_dict( + source="http", + url="https://example.com/image.qcow2", + cert_configmap_name="my-cert-cm", + ) + assert result == {"http": {"url": "https://example.com/image.qcow2", "certConfigMap": "my-cert-cm"}} + + @patch("utilities.storage.validate_file_exists_in_url") + @patch("utilities.infra.url_excluded_from_validation", return_value=False) + def test_http_source_with_secret_and_cert(self, mock_excluded, mock_validate): + result = construct_datavolume_source_dict( + source="http", + url="https://example.com/image.qcow2", + secret_name="my-secret", + cert_configmap_name="my-cert-cm", + ) + assert result == { + "http": {"url": "https://example.com/image.qcow2", "secretRef": "my-secret", "certConfigMap": "my-cert-cm"} + } + + +class TestConstructDatavolumeSourceDictRegistry: + def test_registry_source(self): + result = construct_datavolume_source_dict(source="registry", url="docker://registry.example.com/image:latest") + assert result == {"registry": {"url": "docker://registry.example.com/image:latest"}} + + def test_registry_source_with_secret(self): + result = construct_datavolume_source_dict( + source="registry", + url="docker://registry.example.com/image:latest", + secret_name="registry-secret", + ) + assert result == { + "registry": {"url": "docker://registry.example.com/image:latest", "secretRef": "registry-secret"} + } + + def test_registry_source_with_cert_configmap(self): + result = construct_datavolume_source_dict( + source="registry", + url="docker://registry.example.com/image:latest", + cert_configmap_name="registry-cert-cm", + ) + assert result == { + "registry": {"url": "docker://registry.example.com/image:latest", "certConfigMap": "registry-cert-cm"} + } + + def test_registry_source_with_secret_and_cert(self): + result = construct_datavolume_source_dict( + source="registry", + url="docker://registry.example.com/image:latest", + secret_name="registry-secret", + cert_configmap_name="registry-cert-cm", + ) + assert result == { + "registry": { + "url": "docker://registry.example.com/image:latest", + "secretRef": "registry-secret", + "certConfigMap": "registry-cert-cm", + } + } + + +class TestConstructDatavolumeSourceDictPvc: + def test_pvc_source_with_namespace(self): + result = construct_datavolume_source_dict( + source="pvc", + source_pvc_name="my-pvc", + source_pvc_namespace="my-namespace", + ) + assert result == {"pvc": {"name": "my-pvc", "namespace": "my-namespace"}} + + def test_pvc_source_without_namespace(self): + result = construct_datavolume_source_dict(source="pvc", source_pvc_name="my-pvc") + assert result == {"pvc": {"name": "my-pvc"}} + assert "namespace" not in result["pvc"] + + def test_pvc_source_with_empty_namespace(self): + result = construct_datavolume_source_dict( + source="pvc", + source_pvc_name="my-pvc", + source_pvc_namespace="", + ) + assert result == {"pvc": {"name": "my-pvc", "namespace": ""}} + + +class TestConstructDatavolumeSourceDictBlank: + def test_blank_source(self): + result = construct_datavolume_source_dict(source="blank") + assert result == {"blank": {}} + + +class TestConstructDatavolumeSourceDictUpload: + def test_upload_source(self): + result = construct_datavolume_source_dict(source="upload") + assert result == {"upload": {}} + + +class TestConstructDatavolumeSourceDictUnsupported: + def test_unsupported_source_raises_value_error(self): + with pytest.raises(ValueError, match="Unsupported source type: ftp"): + construct_datavolume_source_dict(source="ftp")