Skip to content

use dmabuf option when the ibperf binaries have them configured #235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ahskabir wants to merge 4 commits into
base: main
Choose a base branch
from
Open

use dmabuf option when the ibperf binaries have them configured #235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
37cd488
use dmabuf option when the ibperf binaries have them configured
ahskabir Jun 12, 2026
de6cc87
using shdl handle instead of phdl handle for the check API
ahskabir Jul 7, 2026
3b0595e
the dmabuf method uses shdl instead of phdl.
ahskabir Jul 8, 2026
6cef5d0
guard against possible None value of head_node
ahskabir Jul 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 94 additions & 5 deletions cvs/lib/ibperf_lib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@

from cvs.lib.utils_lib import *

from cvs.lib import globals

log = globals.log


def detect_rocm_path(phdl, config_rocm_path):
"""
Expand Down Expand Up @@ -64,6 +68,39 @@ def detect_rocm_path(phdl, config_rocm_path):
return '/opt/rocm'


def check_perftest_dmabuf_support(shdl, binary_path, head_node):

@cijohnson cijohnson Jul 24, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We no need to pass head_node to this function, because shdl isa Pssh instance with only one node in the list(head_node) refer https://github.com/ROCm/cvs/pull/235/changes#diff-fc999b79285f07a1d04220513c8cc33f07b30514861fdf7baea8058c53a85bd8R170

"""
Check if the given perftest binary on the cluster node supports the
--use_rocm_dmabuf flag by executing the binary's help output and
grepping for the flag.

Args:
phdl: Pssh handle used to execute commands on one or more cluster nodes.
binary_path (str): Full path to the perftest binary on the node.

Returns:
bool: True if the binary supports --use_rocm_dmabuf on any host,
False otherwise.
"""
cmd = f'{binary_path} --help 2>&1 | grep use_rocm_dmabuf | wc -l'
log.info("Checking dmabuf support: %s", cmd)
dmabuf_available = False

# if head_node is None, that would be a programming error
# but we still cover the edge case
if head_node is None:
log.warning("ibperf dmabuf check failed: head_node is None, check programming and re-run")
return dmabuf_available
try:
dmabuf_check_out = shdl.exec(cmd)
dmabuf_available = int(dmabuf_check_out[head_node].strip()) > 0

@cijohnson cijohnson Jul 24, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

dmabuf_check_out = shdl.exec(cmd)
for node in dmabuf_check_out .keys():
dmabuf_available = int(dmabuf_check_out [node].strip()) > 0

We can use above logic, without head_node argument, the for loop will iterate once as the dmabuf_check_out will have ouptput from one node(head_node)

log.info("dmabuf available in ibperf build" if dmabuf_available else "dmabuf not available in ibperf build")
return dmabuf_available
except Exception as e:
log.warning(f"ibperf dmabuf check failed: {e}")
return dmabuf_available


def get_ib_bw_pps(phdl, msg_size, cmd):
res_dict = {}

Expand Down Expand Up @@ -177,6 +214,7 @@ def verify_expected_lat(lat_test, msg_size, res_dict, expected_res):


def run_ib_perf_bw_test(
shdl,
phdl,
bw_test,
gpu_numa_dict,
Expand All @@ -189,6 +227,7 @@ def run_ib_perf_bw_test(
port_no=1516,
duration=60,
rocm_path='',
cluster_node_list=None,

@cijohnson cijohnson Jul 24, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We no need to pass cluster_node_list , because shdl is connection handle to head node only

):
app_port = port_no
result_dict = {}
Expand All @@ -201,6 +240,11 @@ def run_ib_perf_bw_test(
log.info(f'Setting LD_LIBRARY_PATH to {rocm_path}/lib for perftest binaries')
phdl.exec(f'echo "export LD_LIBRARY_PATH={rocm_path}/lib:$LD_LIBRARY_PATH" >> /tmp/ib_cmds_file.txt')
server_addr = None

# if cluster_node_list is None that would be programming error
if cluster_node_list:
head_node = cluster_node_list[0]
dmabuf_supported = check_perftest_dmabuf_support(shdl, f'{app_path}/{bw_test}', head_node)
for node in bck_nic_dict.keys():
result_dict[node] = {}
cmd_dict[node] = []
Expand All @@ -215,7 +259,15 @@ def run_ib_perf_bw_test(
for gpu_no in range(0, 8):
card_no = 'card' + str(gpu_no)
rdma_dev = gpu_nic_dict[node][card_no]['rdma_dev']
cmd = f'numactl --physcpubind={gpu_numa_dict[node][card_no]["local_cpulist"]} --localalloc {app_path}/{bw_test} -d {rdma_dev} --use_rocm={gpu_no} -x {gid_index} --report_gbits -b -F -D {duration} -p {port_no} -s {msg_size} -q {qp_count} > /tmp/ib_perf_{inst_count}_logs & 2>&1'
if dmabuf_supported:

@cijohnson cijohnson Jul 24, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can simplify like below to avoid code duplication in if else

`
cmd_parts = [
"numactl",
f'--physcpubind={gpu_numa_dict[node][card_no]["local_cpulist"]}',
"--localalloc",
f"{app_path}/{bw_test}",
"-d", rdma_dev,
f"--use_rocm={gpu_no}",
*(["--use_rocm_dmabuf"] if dmabuf_supported else []),
"-x", str(gid_index),
"--report_gbits", "-b", "-F",
"-D", str(duration),
"-p", str(port_no),
"-s", str(msg_size),
"-q", str(qp_count),
server_addr,
]

cmd = " ".join(cmd_parts) + f" > /tmp/ib_perf_{inst_count}_logs 2>&1 &"
`

cmd = f'numactl --physcpubind={gpu_numa_dict[node][card_no]["local_cpulist"]} --localalloc {app_path}/{bw_test} \
-d {rdma_dev} --use_rocm={gpu_no} --use_rocm_dmabuf -x {gid_index} --report_gbits -b -F -D {duration} -p {port_no} \
-s {msg_size} -q {qp_count} > /tmp/ib_perf_{inst_count}_logs & 2>&1'
else:
cmd = f'numactl --physcpubind={gpu_numa_dict[node][card_no]["local_cpulist"]} --localalloc {app_path}/{bw_test} \
-d {rdma_dev} --use_rocm={gpu_no} -x {gid_index} --report_gbits -b -F -D {duration} -p {port_no} \
-s {msg_size} -q {qp_count} > /tmp/ib_perf_{inst_count}_logs & 2>&1'

cmd_dict[node].append(f'echo "{cmd}" >> /tmp/ib_cmds_file.txt')
inst_count = inst_count + 1
port_no = port_no + 1
Expand All @@ -228,7 +280,14 @@ def run_ib_perf_bw_test(
for gpu_no in range(0, 8):
card_no = 'card' + str(gpu_no)
rdma_dev = gpu_nic_dict[node][card_no]['rdma_dev']
cmd = f'numactl --physcpubind={gpu_numa_dict[node][card_no]["local_cpulist"]} --localalloc {app_path}/{bw_test} -d {rdma_dev} --use_rocm={gpu_no} -x {gid_index} --report_gbits -b -F -D {duration} -p {port_no} -s {msg_size} -q {qp_count} {server_addr} > /tmp/ib_perf_{inst_count}_logs & 2>&1'
if dmabuf_supported:
cmd = f'numactl --physcpubind={gpu_numa_dict[node][card_no]["local_cpulist"]} --localalloc {app_path}/{bw_test} \
-d {rdma_dev} --use_rocm={gpu_no} --use_rocm_dmabuf -x {gid_index} --report_gbits -b -F -D {duration} -p {port_no} \
-s {msg_size} -q {qp_count} {server_addr} > /tmp/ib_perf_{inst_count}_logs & 2>&1'
else:
cmd = f'numactl --physcpubind={gpu_numa_dict[node][card_no]["local_cpulist"]} --localalloc {app_path}/{bw_test} \
-d {rdma_dev} --use_rocm={gpu_no} -x {gid_index} --report_gbits -b -F -D {duration} -p {port_no} \
-s {msg_size} -q {qp_count} {server_addr} > /tmp/ib_perf_{inst_count}_logs & 2>&1'
cmd_dict[node].append(f'echo "{cmd}" >> /tmp/ib_cmds_file.txt')
inst_count = inst_count + 1
port_no = port_no + 1
Expand Down Expand Up @@ -269,7 +328,18 @@ def run_ib_perf_bw_test(


def run_ib_perf_lat_test(
phdl, lat_test, gpu_numa_dict, gpu_nic_dict, bck_nic_dict, app_path, msg_size, gid_index, port_no=1516, rocm_path=''
shdl,
phdl,
lat_test,
gpu_numa_dict,
gpu_nic_dict,
bck_nic_dict,
app_path,
msg_size,
gid_index,
port_no=1516,
rocm_path='',
cluster_node_list=None,

@cijohnson cijohnson Jul 24, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We no need to pass cluster_node_list , because shdl is connection handle to head node only

):
app_port = port_no
result_dict = {}
Expand All @@ -282,6 +352,10 @@ def run_ib_perf_lat_test(
log.info(f'Setting LD_LIBRARY_PATH to {rocm_path}/lib for perftest binaries')
phdl.exec(f'echo "export LD_LIBRARY_PATH={rocm_path}/lib:$LD_LIBRARY_PATH" >> /tmp/ib_cmds_file.txt')
server_addr = None
# if cluster_node_list is None that would be programming error
if cluster_node_list:
head_node = cluster_node_list[0]
dmabuf_supported = check_perftest_dmabuf_support(shdl, f'{app_path}/{lat_test}', head_node)
for node in bck_nic_dict.keys():
result_dict[node] = {}
cmd_dict[node] = []
Expand All @@ -296,7 +370,15 @@ def run_ib_perf_lat_test(
for gpu_no in range(0, 8):
card_no = 'card' + str(gpu_no)
rdma_dev = gpu_nic_dict[node][card_no]['rdma_dev']
cmd = f'numactl --physcpubind={gpu_numa_dict[node][card_no]["local_cpulist"]} --localalloc {app_path}/{lat_test} -d {rdma_dev} --use_rocm={gpu_no} -x {gid_index} --report_gbits -F -p {port_no} -s {msg_size} > /tmp/ib_perf_{inst_count}_logs & 2>&1'
if dmabuf_supported:
cmd = f'numactl --physcpubind={gpu_numa_dict[node][card_no]["local_cpulist"]} --localalloc {app_path}/{lat_test} \
-d {rdma_dev} --use_rocm={gpu_no} --use_rocm_dmabuf -x {gid_index} --report_gbits -F -p {port_no} \
-s {msg_size} > /tmp/ib_perf_{inst_count}_logs & 2>&1'
else:
cmd = f'numactl --physcpubind={gpu_numa_dict[node][card_no]["local_cpulist"]} --localalloc {app_path}/{lat_test} \
-d {rdma_dev} --use_rocm={gpu_no} -x {gid_index} --report_gbits -F -p {port_no} \
-s {msg_size} > /tmp/ib_perf_{inst_count}_logs & 2>&1'

cmd_dict[node].append(f'echo "{cmd}" >> /tmp/ib_cmds_file.txt')
inst_count = inst_count + 1
port_no = port_no + 1
Expand All @@ -309,7 +391,14 @@ def run_ib_perf_lat_test(
for gpu_no in range(0, 8):
card_no = 'card' + str(gpu_no)
rdma_dev = gpu_nic_dict[node][card_no]['rdma_dev']
cmd = f'numactl --physcpubind={gpu_numa_dict[node][card_no]["local_cpulist"]} --localalloc {app_path}/{lat_test} -d {rdma_dev} --use_rocm={gpu_no} -x {gid_index} --report_gbits -F -p {port_no} -s {msg_size} {server_addr} > /tmp/ib_perf_{inst_count}_logs & 2>&1'
if dmabuf_supported:
cmd = f'numactl --physcpubind={gpu_numa_dict[node][card_no]["local_cpulist"]} --localalloc {app_path}/{lat_test} \
-d {rdma_dev} --use_rocm={gpu_no} --use_rocm_dmabuf -x {gid_index} --report_gbits -F -p {port_no} \
-s {msg_size} {server_addr} > /tmp/ib_perf_{inst_count}_logs & 2>&1'
else:
cmd = f'numactl --physcpubind={gpu_numa_dict[node][card_no]["local_cpulist"]} --localalloc {app_path}/{lat_test} \
-d {rdma_dev} --use_rocm={gpu_no} -x {gid_index} --report_gbits -F -p {port_no} \
-s {msg_size} {server_addr} > /tmp/ib_perf_{inst_count}_logs & 2>&1'
cmd_dict[node].append(f'echo "{cmd}" >> /tmp/ib_cmds_file.txt')
inst_count = inst_count + 1
port_no = port_no + 1
Expand Down
10 changes: 8 additions & 2 deletions cvs/tests/ibperf/ib_perf_bw_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ def vpc_node_list(cluster_dict):


@pytest.mark.parametrize("bw_test", ["ib_write_bw", "ib_read_bw", "ib_send_bw"])
def test_ib_bw_perf(phdl, bw_test, config_dict):
def test_ib_bw_perf(shdl, phdl, bw_test, config_dict, cluster_dict):
# Get IB_backend_nics for each node
# Get the NIC to GPU mapping dict
# Generate the command list for all nodes
Expand All @@ -230,13 +230,15 @@ def test_ib_bw_perf(phdl, bw_test, config_dict):
bck_nic_dict[node][rdma_dev] = rdma_nic_dict[node][rdma_dev]

rocm_path = ibperf_lib.detect_rocm_path(phdl, config_dict.get('rocm_dir', ''))
node_list = list(cluster_dict['node_dict'].keys())
for msg_size in config_dict['msg_size_list']:
ib_bw_dict[bw_test][msg_size] = {}
for qp_count in config_dict['qp_count_list']:
# Log a message to Dmesg to create a timestamp record
start_time = phdl.exec('date +"%a %b %e %H:%M"')
phdl.exec(f'echo "Starting Test {bw_test} for {msg_size} and QP count {qp_count}" | sudo -n tee /dev/kmsg')
ib_bw_dict[bw_test][msg_size][qp_count] = ibperf_lib.run_ib_perf_bw_test(
shdl,
phdl,
bw_test,
gpu_numa_dict,
Expand All @@ -249,6 +251,7 @@ def test_ib_bw_perf(phdl, bw_test, config_dict):
int(config_dict['port_no']),
int(config_dict['duration']),
rocm_path=rocm_path,
cluster_node_list=node_list,
)
end_time = phdl.exec('date +"%a %b %e %H:%M"')
verify_dmesg_for_errors(phdl, start_time, end_time, till_end_flag=True)
Expand All @@ -267,7 +270,7 @@ def test_ib_bw_perf(phdl, bw_test, config_dict):


@pytest.mark.parametrize("lat_test", ["ib_write_lat", "ib_send_lat"])
def test_ib_lat_perf(phdl, lat_test, config_dict):
def test_ib_lat_perf(shdl, phdl, lat_test, config_dict, cluster_dict):
globals.error_list = []
ib_lat_dict[lat_test] = {}

Expand All @@ -290,12 +293,14 @@ def test_ib_lat_perf(phdl, lat_test, config_dict):
log.info(f'%%%%%% gpu_numa_dict %%%%% {gpu_numa_dict}')

rocm_path = ibperf_lib.detect_rocm_path(phdl, config_dict.get('rocm_dir', ''))
node_list = list(cluster_dict['node_dict'].keys())
for msg_size in config_dict['msg_size_list']:
ib_lat_dict[lat_test][msg_size] = {}
# Log a message to Dmesg to create a timestamp record
start_time = phdl.exec('date +"%a %b %e %H:%M"')
phdl.exec(f'echo "Starting Test {lat_test} for {msg_size}" | sudo -n tee /dev/kmsg')
ib_lat_dict[lat_test][msg_size] = ibperf_lib.run_ib_perf_lat_test(
shdl,
phdl,
lat_test,
gpu_numa_dict,
Expand All @@ -306,6 +311,7 @@ def test_ib_lat_perf(phdl, lat_test, config_dict):
config_dict['gid_index'],
int(config_dict['port_no']),
rocm_path=rocm_path,
cluster_node_list=node_list,
)
end_time = phdl.exec('date +"%a %b %e %H:%M"')
verify_dmesg_for_errors(phdl, start_time, end_time, till_end_flag=True)
Expand Down
Loading