From 1347f3baa4370f1778c85f4408822ca69cf0f613 Mon Sep 17 00:00:00 2001 From: Mikael Huppunen Date: Mon, 22 Jun 2026 11:45:32 +0300 Subject: [PATCH 1/6] Gathering gpu memory reports from tasks --- memory_report.cpp | 44 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/memory_report.cpp b/memory_report.cpp index 3673ec136b..e748ead381 100644 --- a/memory_report.cpp +++ b/memory_report.cpp @@ -271,15 +271,47 @@ void report_memory_consumption( P::tstep, P::t, sum_mem[2]/GiB, sum_mem[5]/GiB); logFile << reportstring; - logFile << writeVerbose; - - MPI_Comm_free(&interComm); - MPI_Comm_free(&nodeComm); - #ifdef USE_GPU + const double KiB = 1024.0; // TODO: Clear duplicate output // (local_cells_capacity, ghost_cells_capacity, local_cells_size, ghost_cells_size) - gpu_reportMemory(mem[3], mem[4], mem[0], mem[1]); + int gpuReportedUsage = gpu_reportMemory(mem[3], mem[4], mem[0], mem[1]); + int sumGpu = 0; + struct { + int value; + int rank; + } minGpu, gpuMemUsageLoc, maxGpu; + gpuMemUsageLoc.value = gpuReportedUsage; + gpuMemUsageLoc.rank = rank; + + MPI_Reduce(&gpuReportedUsage, &sumGpu, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); + MPI_Reduce(&gpuMemUsageLoc, &maxGpu, 1, MPI_2INT, MPI_MAXLOC, 0, MPI_COMM_WORLD); + MPI_Reduce(&gpuMemUsageLoc, &minGpu, 1, MPI_2INT, MPI_MINLOC, 0, MPI_COMM_WORLD); + + snprintf(reportstring,512, "(MEM) tstep %i t %.3g %-21s (GiB/rank; avg, min, max, sum): %-8.3g %-8.3g %-8.3g %-8.3g min rank %i max rank %i\n", + P::tstep, P::t, "Total reported GPU memory usage", (double)sumGpu/((double)nProcs*KiB), (double)minGpu.value/KiB, (double)maxGpu.value/KiB, (double)sumGpu/KiB, minGpu.rank, maxGpu.rank); + logFile << reportstring; + + size_t free_byte; + size_t total_byte; + CHK_ERR( gpuMemGetInfo( &free_byte, &total_byte) ); + gpuReportedUsage = (total_byte-free_byte)/(1024*1024); + sumGpu = 0; + gpuMemUsageLoc.value = gpuReportedUsage; + gpuMemUsageLoc.rank = rank; + + MPI_Reduce(&gpuReportedUsage, &sumGpu, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); + MPI_Reduce(&gpuMemUsageLoc, &maxGpu, 1, MPI_2INT, MPI_MAXLOC, 0, MPI_COMM_WORLD); + MPI_Reduce(&gpuMemUsageLoc, &minGpu, 1, MPI_2INT, MPI_MINLOC, 0, MPI_COMM_WORLD); + + snprintf(reportstring,512, "(MEM) tstep %i t %.3g %-21s (GiB/rank; avg, min, max, sum): %-8.3g %-8.3g %-8.3g %-8.3g min rank %i max rank %i\n", + P::tstep, P::t, "Total reported GPU hardware usage", (double)sumGpu/((double)nProcs*KiB), (double)minGpu.value/KiB, (double)maxGpu.value/KiB, (double)sumGpu/KiB, minGpu.rank, maxGpu.rank); + logFile << reportstring; #endif + + logFile << writeVerbose; + + MPI_Comm_free(&interComm); + MPI_Comm_free(&nodeComm); } From 5ec7a9f2927a34be77878a91eb520595c0b0f8f5 Mon Sep 17 00:00:00 2001 From: Mikael Huppunen Date: Wed, 24 Jun 2026 11:56:53 +0300 Subject: [PATCH 2/6] Improved formatting --- memory_report.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memory_report.cpp b/memory_report.cpp index e748ead381..5ebe550851 100644 --- a/memory_report.cpp +++ b/memory_report.cpp @@ -288,7 +288,7 @@ void report_memory_consumption( MPI_Reduce(&gpuMemUsageLoc, &maxGpu, 1, MPI_2INT, MPI_MAXLOC, 0, MPI_COMM_WORLD); MPI_Reduce(&gpuMemUsageLoc, &minGpu, 1, MPI_2INT, MPI_MINLOC, 0, MPI_COMM_WORLD); - snprintf(reportstring,512, "(MEM) tstep %i t %.3g %-21s (GiB/rank; avg, min, max, sum): %-8.3g %-8.3g %-8.3g %-8.3g min rank %i max rank %i\n", + snprintf(reportstring,512, "(MEM) tstep %i t %.3g %-21s (GiB/rank; avg, min, max, sum): %-8.3g %-8.3g %-8.3g %-8.3g min rank %i max rank %i\n", P::tstep, P::t, "Total reported GPU memory usage", (double)sumGpu/((double)nProcs*KiB), (double)minGpu.value/KiB, (double)maxGpu.value/KiB, (double)sumGpu/KiB, minGpu.rank, maxGpu.rank); logFile << reportstring; From 6bbfb4d5dc1babfeff8b996abe3d43818f78c8e3 Mon Sep 17 00:00:00 2001 From: Mikael Huppunen Date: Thu, 25 Jun 2026 12:49:13 +0300 Subject: [PATCH 3/6] Printing GPU memory report after load balance --- vlasiator.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/vlasiator.cpp b/vlasiator.cpp index acbb9ad59f..4ffb69c067 100644 --- a/vlasiator.cpp +++ b/vlasiator.cpp @@ -1134,6 +1134,13 @@ int simulate(int argn,char* args[]) { // Make sure the ionosphere communicator is up-to-date, in case inner boundary cells // moved. SBC::ionosphereGrid.updateIonosphereCommunicator(mpiGrid, technical.view(), fsgrid); + + #ifdef USE_GPU + phiprof::Timer memTimer {"memory-report"}; + memTimer.start(); + report_memory_consumption(mpiGrid); + memTimer.stop(); + #endif } //get local cells From 6dc9c01140fc7323ec00eab61cf6f02c9b9f1e05 Mon Sep 17 00:00:00 2001 From: Mikael Huppunen Date: Thu, 25 Jun 2026 16:22:26 +0300 Subject: [PATCH 4/6] Getting pancake capacity --- arch/gpu_base.cpp | 30 +++++++++++++++++++++++++++++- arch/gpu_base.hpp | 1 + 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/arch/gpu_base.cpp b/arch/gpu_base.cpp index f3631e38ba..314ddfb8da 100644 --- a/arch/gpu_base.cpp +++ b/arch/gpu_base.cpp @@ -260,6 +260,32 @@ __host__ uint gpu_getAllocationCount() { return allocationCount; } +std::size_t get_preloaded_pool_size() { + const char* preload = std::getenv("LD_PRELOAD"); + if (!preload){ + return 0; + } + + std::string preloadString(preload); + std::size_t position = preloadString.find("libmpipancake.so"); + if (position == std::string::npos){ + return 0; + } + + void* handle = dlopen(preloadString.c_str(), RTLD_NOW | RTLD_NOLOAD); + if (!handle){ + return 0; + } + + using get_pool_size_function = std::size_t(*)(); + auto poolSizeFunction = reinterpret_cast(dlsym(handle, "get_pool_size")); + if (!poolSizeFunction){ + return 0; + } + + return poolSizeFunction(); +} + /* Memory reporting function */ @@ -297,12 +323,13 @@ int gpu_reportMemory(const size_t local_cells_capacity, const size_t ghost_cells // Remote neighbor contribution buffers are in unified memory but deallocated after each use size_t memoryManagerCapacity = gpuMemoryManager.totalGpuAllocation(); + size_t pancakeCapacity = get_preloaded_pool_size(); size_t free_byte ; size_t total_byte ; CHK_ERR( gpuMemGetInfo( &free_byte, &total_byte) ); size_t used_mb = (total_byte-free_byte)/(1024*1024); - size_t sum_mb = (miniBuffers+batchBuffers+vlasovBuffers+accBuffers+transBuffers+local_cells_capacity+ghost_cells_capacity+memoryManagerCapacity)/(1024*1024); + size_t sum_mb = (miniBuffers+batchBuffers+vlasovBuffers+accBuffers+transBuffers+local_cells_capacity+ghost_cells_capacity+memoryManagerCapacity+pancakeCapacity)/(1024*1024); size_t local_req_mb = local_cells_size/(1024*1024); size_t ghost_req_mb = ghost_cells_size/(1024*1024); @@ -317,6 +344,7 @@ int gpu_reportMemory(const size_t local_cells_capacity, const size_t ghost_cells logFile<<" Local cells: "< Date: Thu, 2 Jul 2026 14:11:15 +0300 Subject: [PATCH 5/6] No longer printing GPU memory report after load balance --- vlasiator.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/vlasiator.cpp b/vlasiator.cpp index 4ffb69c067..acbb9ad59f 100644 --- a/vlasiator.cpp +++ b/vlasiator.cpp @@ -1134,13 +1134,6 @@ int simulate(int argn,char* args[]) { // Make sure the ionosphere communicator is up-to-date, in case inner boundary cells // moved. SBC::ionosphereGrid.updateIonosphereCommunicator(mpiGrid, technical.view(), fsgrid); - - #ifdef USE_GPU - phiprof::Timer memTimer {"memory-report"}; - memTimer.start(); - report_memory_consumption(mpiGrid); - memTimer.stop(); - #endif } //get local cells From 26f81a6648b4c70ad34f02080e89dd57269c9c2e Mon Sep 17 00:00:00 2001 From: Mikael Huppunen Date: Fri, 3 Jul 2026 11:21:24 +0300 Subject: [PATCH 6/6] Subtracting GPU memory from CPU free memory count --- memory_report.cpp | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/memory_report.cpp b/memory_report.cpp index 5ebe550851..40434036d6 100644 --- a/memory_report.cpp +++ b/memory_report.cpp @@ -108,6 +108,33 @@ uint64_t get_node_free_memory(){ fclose(in_file); } + #ifdef USE_GPU + // On systems where GPU HBM is exposed as coherent NUMA memory (e.g. GH200), + // /proc/meminfo MemFree includes GPU free memory. Subtract free pages in + // Movable zones, which is where GPU HBM appears on such systems. + // On conventional systems Movable zones are empty, making this a no-op. + uint64_t movableFreeBytes = 0; + FILE *zoneinfo = fopen("/proc/zoneinfo", "r"); + if (zoneinfo) { + char line[256]; + bool inMovable = false; + while (fgets(line, sizeof(line), zoneinfo)) { + if (strncmp(line, "Node ", 5) == 0) { + inMovable = strstr(line, "Movable") != NULL; + } else if (inMovable) { + unsigned long pages = 0; + if (sscanf(line, " nr_free_pages %lu", &pages) == 1) { + movableFreeBytes += (uint64_t)pages * sysconf(_SC_PAGESIZE); + } + } + } + fclose(zoneinfo); + } + if (movableFreeBytes < mem_proc_free) { + mem_proc_free -= movableFreeBytes; + } + #endif + return mem_proc_free; } /*! Measures memory consumption and writes it into logfile.