[Storage] Use source_dict for DataVolume source specification, refactor create_dv

tests/chaos/conftest.py

@@ -43,6 +43,7 @@
     utility_daemonset_for_custom_tests,
     wait_for_node_status,
 )
+from utilities.storage import construct_datavolume_source_dict
 from utilities.virt import VirtualMachineForTests, running_vm
 
 LOGGER = logging.getLogger(__name__)
@@ -93,16 +94,18 @@ def chaos_dv_rhel9(
     artifactory_config_map_chaos_namespace_scope_module,
 ):
     yield DataVolume(
-        source="http",
+        source_dict=construct_datavolume_source_dict(
+            source="http",
+            url=rhel9_http_image_url,
+            secret_name=artifactory_secret_chaos_namespace_scope_module.name,
+            cert_configmap_name=artifactory_config_map_chaos_namespace_scope_module.name,
+        ),
         name="chaos-dv",
         api_name="storage",
         namespace=chaos_namespace.name,
-        url=rhel9_http_image_url,
         size=Images.Rhel.DEFAULT_DV_SIZE,
         storage_class=request.param["storage_class"],
         client=admin_client,
-        secret=artifactory_secret_chaos_namespace_scope_module,
-        cert_configmap=artifactory_config_map_chaos_namespace_scope_module.name,
     )
 
 

tests/chaos/snapshot/conftest.py

@@ -8,6 +8,7 @@
     TIMEOUT_8MIN,
     TIMEOUT_10MIN,
 )
+from utilities.storage import construct_datavolume_source_dict
 from utilities.virt import VirtualMachineForTests
 
 
@@ -21,16 +22,18 @@ def chaos_dv_rhel9_for_snapshot(
     artifactory_config_map_chaos_namespace_scope_module,
 ):
     yield DataVolume(
-        source="http",
+        source_dict=construct_datavolume_source_dict(
+            source="http",
+            url=rhel9_http_image_url,
+            secret_name=artifactory_secret_chaos_namespace_scope_module.name,
+            cert_configmap_name=artifactory_config_map_chaos_namespace_scope_module.name,
+        ),
         name="chaos-dv",
         api_name="storage",
         namespace=chaos_namespace.name,
-        url=rhel9_http_image_url,
         size=Images.Rhel.DEFAULT_DV_SIZE,
         storage_class=[*storage_class_matrix_snapshot_matrix__function__][0],
         client=admin_client,
-        secret=artifactory_secret_chaos_namespace_scope_module,
-        cert_configmap=artifactory_config_map_chaos_namespace_scope_module.name,
     )
 
 

tests/conftest.py

@@ -132,7 +132,7 @@
     UNPRIVILEGED_PASSWORD,
     UNPRIVILEGED_USER,
 )
-from utilities.constants.storage import StorageClassNames
+from utilities.constants.storage import BIND_IMMEDIATE_ANNOTATION, StorageClassNames
 from utilities.constants.timeouts import (
     TIMEOUT_3MIN,
     TIMEOUT_4MIN,
@@ -200,6 +200,7 @@
 from utilities.sanity import cluster_sanity
 from utilities.ssp import get_data_import_crons, get_ssp_resource
 from utilities.storage import (
+    construct_datavolume_source_dict,
     create_or_update_data_source,
     data_volume,
     get_default_storage_class,
@@ -2518,13 +2519,15 @@ def dvs_for_upgrade(
             client=admin_client,
             name=f"dv-for-product-upgrade-{storage_class}",
             namespace=golden_images_namespace_name,
-            source="http",
+            source_dict=construct_datavolume_source_dict(
+                source="http",
+                url=rhel_latest_os_params["rhel_image_path"],
+                secret_name=artifactory_secret.name,
+                cert_configmap_name=artifactory_config_map.name,
+            ),
             storage_class=storage_class,
-            secret=artifactory_secret,
-            cert_configmap=artifactory_config_map.name,
-            url=rhel_latest_os_params["rhel_image_path"],
             size=rhel_latest_os_params["rhel_dv_size"],
-            bind_immediate_annotation=True,
+            annotations=BIND_IMMEDIATE_ANNOTATION,
             api_name="storage",
         )
         dv.create()

tests/data_protection/oadp/conftest.py

@@ -29,6 +29,7 @@
 )
 from utilities.storage import (
     check_upload_virtctl_result,
+    construct_datavolume_source_dict,
     create_dv,
     create_vm_from_dv,
     get_downloaded_artifact,
@@ -48,10 +49,12 @@ def imported_dv_in_progress_second_namespace(
     with create_dv(
         dv_name="imported-dv",
         namespace=namespace_for_backup2.name,
+        source="http",
         url=rhel9_http_image_url,
         size=Images.Rhel.DEFAULT_DV_SIZE,
         storage_class=storage_class_for_snapshot,
         client=namespace_for_backup2.client,
+        use_artifactory=True,
     ) as dv:
         yield dv
 
@@ -165,16 +168,18 @@ def windows_vm_with_data_volume_template(
             name="oadp-windows-dv",
             namespace=namespace_for_backup.name,
             storage_class=snapshot_storage_class_name_scope_module,
-            source="registry",
-            url=(
-                f"{get_test_artifact_server_url(schema='registry')}/"
-                f"{py_config['latest_windows_os_dict'][CONTAINER_DISK_IMAGE_PATH_STR]}"
+            source_dict=construct_datavolume_source_dict(
+                source="registry",
+                url=(
+                    f"{get_test_artifact_server_url(schema='registry')}/"
+                    f"{py_config['latest_windows_os_dict'][CONTAINER_DISK_IMAGE_PATH_STR]}"
+                ),
+                secret_name=artifactory_secret.name,
+                cert_configmap_name=artifactory_config_map.name,
             ),
             size=Images.Windows.CONTAINER_DISK_DV_SIZE,
             client=admin_client,
             api_name="storage",
-            secret=artifactory_secret,
-            cert_configmap=artifactory_config_map.name,
         )
         dv.to_dict()
 
@@ -299,7 +304,8 @@ def cloned_rhel_dv(imported_dv_second_namespace):
         dv_name="cloned-dv",
         namespace=imported_dv_second_namespace.namespace,
         size=imported_dv_second_namespace.size,
-        source_pvc=imported_dv_second_namespace.name,
+        source_pvc_name=imported_dv_second_namespace.name,
+        source_pvc_namespace=imported_dv_second_namespace.namespace,
         storage_class=imported_dv_second_namespace.storage_class,
         client=imported_dv_second_namespace.client,
     ) as cdv:

tests/infrastructure/golden_images/update_boot_source/test_ssp_common_templates_boot_sources.py

@@ -16,13 +16,15 @@
     DEFAULT_FEDORA_REGISTRY_URL,
     OS_FLAVOR_FEDORA,
 )
+from utilities.constants.storage import BIND_IMMEDIATE_ANNOTATION
 from utilities.constants.timeouts import (
     TIMEOUT_5MIN,
     TIMEOUT_5SEC,
 )
 from utilities.infra import (
     validate_os_info_vmi_vs_linux_os,
 )
+from utilities.storage import construct_datavolume_source_dict
 from utilities.virt import VirtualMachineForTestsFromTemplate, running_vm
 
 LOGGER = logging.getLogger(__name__)
@@ -119,11 +121,10 @@ def imported_fedora_dv(admin_client, golden_images_namespace, fedora_data_source
         name=fedora_data_source.name,
         namespace=golden_images_namespace.name,
         api_name="storage",
-        source="registry",
+        source_dict=construct_datavolume_source_dict(source="registry", url=DEFAULT_FEDORA_REGISTRY_URL),
         size=Images.Fedora.DEFAULT_DV_SIZE,
         storage_class=py_config["default_storage_class"],
-        url=DEFAULT_FEDORA_REGISTRY_URL,
-        bind_immediate_annotation=True,
+        annotations=BIND_IMMEDIATE_ANNOTATION,
     ) as dv:
         dv.wait_for_dv_success()
         yield dv

tests/infrastructure/golden_images/update_boot_source/test_ssp_data_sources.py

@@ -27,12 +27,14 @@
 from utilities.constants.hco import DATA_SOURCE_NAME
 from utilities.constants.images import DEFAULT_FEDORA_REGISTRY_URL
 from utilities.constants.pytest import QUARANTINED
+from utilities.constants.storage import BIND_IMMEDIATE_ANNOTATION
 from utilities.constants.timeouts import (
     TIMEOUT_5MIN,
     TIMEOUT_10MIN,
 )
 from utilities.exceptions import ResourceValueError
 from utilities.ssp import wait_for_condition_message_value
+from utilities.storage import construct_datavolume_source_dict
 
 LOGGER = logging.getLogger(__name__)
 
@@ -51,14 +53,16 @@ def dv_for_data_source(name, data_source, admin_client):
         client=admin_client,
         name=name,
         namespace=data_source.namespace,
-        # underlying OS is not relevant
-        url=get_http_image_url(image_directory=Images.Cirros.DIR, image_name=Images.Cirros.QCOW2_IMG),
-        source="http",
-        secret=artifactory_secret,
-        cert_configmap=artifactory_config_map.name,
+        source_dict=construct_datavolume_source_dict(
+            # underlying OS is not relevant
+            source="http",
+            url=get_http_image_url(image_directory=Images.Cirros.DIR, image_name=Images.Cirros.QCOW2_IMG),
+            secret_name=artifactory_secret.name,
+            cert_configmap_name=artifactory_config_map.name,
+        ),
         size=Images.Cirros.DEFAULT_DV_SIZE,
         storage_class=py_config["default_storage_class"],
-        bind_immediate_annotation=True,
+        annotations=BIND_IMMEDIATE_ANNOTATION,
         api_name="storage",
     ) as dv:
         dv.wait_for_dv_success()

tests/infrastructure/instance_types/conftest.py

@@ -30,6 +30,7 @@
 from utilities.constants.os_matrix import CONTAINER_DISK_IMAGE_PATH_STR
 from utilities.constants.timeouts import TIMEOUT_15MIN
 from utilities.storage import (
+    construct_datavolume_source_dict,
     create_dummy_first_consumer_pod,
     data_volume_template_with_source_ref_dict,
     generate_data_source_dict,
@@ -158,13 +159,17 @@ def latest_windows_data_volume(
         name="latest-windows",
         namespace=windows_test_images_namespace_role_binding.namespace,
         api_name="storage",
-        source="registry",
+        source_dict=construct_datavolume_source_dict(
+            source="registry",
+            url=(
+                f"{get_test_artifact_server_url(schema='registry')}/"
+                f"{py_config['latest_windows_os_dict'][CONTAINER_DISK_IMAGE_PATH_STR]}"
+            ),
+            secret_name=windows_namespace_artifactory_secret_and_configmap["secret"].name,
+            cert_configmap_name=windows_namespace_artifactory_secret_and_configmap["config_map"].name,
+        ),
         size=Images.Windows.CONTAINER_DISK_DV_SIZE,
         storage_class=default_sc.name,
-        url=f"{get_test_artifact_server_url(schema='registry')}/"
-        f"{py_config['latest_windows_os_dict'][CONTAINER_DISK_IMAGE_PATH_STR]}",
-        secret=windows_namespace_artifactory_secret_and_configmap["secret"],
-        cert_configmap=windows_namespace_artifactory_secret_and_configmap["config_map"].name,
     )
     if windows_data_volume.exists:
         yield windows_data_volume

tests/infrastructure/instance_types/supported_os/conftest.py

@@ -18,7 +18,7 @@
     CONTAINER_DISK_IMAGE_PATH_STR,
     DATA_SOURCE_STR,
 )
-from utilities.storage import get_test_artifact_server_url
+from utilities.storage import construct_datavolume_source_dict, get_test_artifact_server_url
 from utilities.virt import VirtualMachineForTests
 
 
@@ -103,12 +103,14 @@ def windows_data_volume_template(
         name=f"{os_matrix_key}-dv",
         namespace=namespace.name,
         api_name="storage",
-        source="registry",
+        source_dict=construct_datavolume_source_dict(
+            source="registry",
+            url=f"{get_test_artifact_server_url(schema='registry')}/{os_params[CONTAINER_DISK_IMAGE_PATH_STR]}",
+            secret_name=secret.name,
+            cert_configmap_name=cert.name,
+        ),
         size=Images.Windows.CONTAINER_DISK_DV_SIZE,
         storage_class=py_config["default_storage_class"],
-        url=f"{get_test_artifact_server_url(schema='registry')}/{os_params[CONTAINER_DISK_IMAGE_PATH_STR]}",
-        secret=secret,
-        cert_configmap=cert.name,
     )
     win_dv.to_dict()
     yield win_dv

tests/observability/metrics/utils.py

@@ -60,6 +60,7 @@
     TIMEOUT_40MIN,
 )
 from utilities.monitoring import get_metrics_value
+from utilities.storage import construct_datavolume_source_dict
 from utilities.virt import VirtualMachineForTests, running_vm
 
 LOGGER = logging.getLogger(__name__)
@@ -661,12 +662,14 @@ def create_windows11_wsl2_vm(
         name=dv_name,
         namespace=namespace,
         api_name="storage",
-        source=REGISTRY_STR,
+        source_dict=construct_datavolume_source_dict(
+            source=REGISTRY_STR,
+            url=f"{get_test_artifact_server_url(schema=REGISTRY_STR)}/docker-local/windows-qe/win_11:virtio",
+            secret_name=artifactory_secret.name,
+            cert_configmap_name=artifactory_config_map.name,
+        ),
         size=Images.Windows.CONTAINER_DISK_DV_SIZE,
         storage_class=storage_class,
-        url=f"{get_test_artifact_server_url(schema=REGISTRY_STR)}/docker-local/windows-qe/win_11:virtio",
-        secret=artifactory_secret,
-        cert_configmap=artifactory_config_map.name,
     )
     dv.to_dict()
     base_preference = VirtualMachineClusterPreference(client=client, name=windows_preference_name)

tests/scale/test_scale_benchmark.py

@@ -46,7 +46,7 @@
     create_ns,
 )
 from utilities.must_gather import run_must_gather
-from utilities.storage import generate_data_source_dict, get_test_artifact_server_url
+from utilities.storage import construct_datavolume_source_dict, generate_data_source_dict, get_test_artifact_server_url
 from utilities.virt import (
     VirtualMachineForTestsFromTemplate,
     verify_vm_migrated,
@@ -274,12 +274,14 @@ def _delete_resources():
                 namespace=golden_images_namespace.name,
                 storage_class=SCALE_STORAGE_TYPES[storage_type],
                 api_name="storage",
-                url=f"{get_test_artifact_server_url()}{dv_info['url']}",
+                source_dict=construct_datavolume_source_dict(
+                    source="http",
+                    url=f"{get_test_artifact_server_url()}{dv_info['url']}",
+                    secret_name=artifactory_secret.name,
+                    cert_configmap_name=artifactory_config_map.name,
+                ),
                 size=dv_info["size"],
                 client=admin_client,
-                source="http",
-                secret=artifactory_secret,
-                cert_configmap=artifactory_config_map.name,
             )
             golden_images_scale_dv.deploy()
             dvs_list.append(golden_images_scale_dv)

tests/storage/cdi_clone/test_clone.py

@@ -85,7 +85,8 @@ def test_successful_clone_of_large_image(
         dv_name="dv-target",
         namespace=namespace.name,
         size=data_volume_multi_storage_scope_function.size,
-        source_pvc=data_volume_multi_storage_scope_function.name,
+        source_pvc_name=data_volume_multi_storage_scope_function.name,
+        source_pvc_namespace=data_volume_multi_storage_scope_function.namespace,
         storage_class=data_volume_multi_storage_scope_function.storage_class,
         client=namespace.client,
     ) as cdv:
@@ -173,7 +174,8 @@ def test_successful_vm_from_cloned_dv_windows(
         dv_name="dv-target",
         namespace=data_volume_multi_storage_scope_function.namespace,
         size=data_volume_multi_storage_scope_function.size,
-        source_pvc=data_volume_multi_storage_scope_function.name,
+        source_pvc_name=data_volume_multi_storage_scope_function.name,
+        source_pvc_namespace=data_volume_multi_storage_scope_function.namespace,
         storage_class=data_volume_multi_storage_scope_function.storage_class,
     ) as cdv:
         cdv.wait_for_dv_success(timeout=WINDOWS_CLONE_TIMEOUT)
@@ -198,7 +200,7 @@ def test_successful_vm_from_cloned_dv_windows(
         ),
         pytest.param(
             {
-                "dv_name": "dv-source-win",
+                "dv_name": f"dv-source-{OS_FLAVOR_WINDOWS}",
                 "image": f"{Images.Windows.DIR}/{Images.Windows.WIN11_IMG}",
                 "dv_size": Images.Windows.DEFAULT_DV_SIZE,
             },
@@ -220,11 +222,12 @@ def test_successful_snapshot_clone(
         dv_name="dv-target",
         namespace=namespace,
         size=data_volume_snapshot_capable_storage_scope_function.size,
-        source_pvc=data_volume_snapshot_capable_storage_scope_function.name,
+        source_pvc_name=data_volume_snapshot_capable_storage_scope_function.name,
+        source_pvc_namespace=data_volume_snapshot_capable_storage_scope_function.namespace,
         storage_class=storage_class,
     ) as cdv:
         cdv.wait_for_dv_success()
-        if OS_FLAVOR_WINDOWS not in data_volume_snapshot_capable_storage_scope_function.url.split("/")[-1]:
+        if OS_FLAVOR_WINDOWS not in data_volume_snapshot_capable_storage_scope_function.name:
             with create_vm_from_dv(
                 client=unprivileged_client,
                 dv=cdv,