Gathering GPU memory report from all tasks

arch/gpu_base.cpp

@@ -260,6 +260,32 @@ __host__ uint gpu_getAllocationCount() {
    return allocationCount;
 }
 
+std::size_t get_preloaded_pool_size() {
+   const char* preload = std::getenv("LD_PRELOAD");
+   if (!preload){
+      return 0;
+   }
+
+   std::string preloadString(preload);
+   std::size_t position = preloadString.find("libmpipancake.so");
+   if (position == std::string::npos){
+      return 0;
+   }
+
+   void* handle = dlopen(preloadString.c_str(), RTLD_NOW | RTLD_NOLOAD);
+   if (!handle){
+      return 0;
+   }
+
+   using get_pool_size_function = std::size_t(*)();
+   auto poolSizeFunction = reinterpret_cast<get_pool_size_function>(dlsym(handle, "get_pool_size"));
+   if (!poolSizeFunction){
+      return 0;
+   }
+
+   return poolSizeFunction();
+}
+
 /*
    Memory reporting function
 */
@@ -297,12 +323,13 @@ int gpu_reportMemory(const size_t local_cells_capacity, const size_t ghost_cells
    // Remote neighbor contribution buffers are in unified memory but deallocated after each use
 
    size_t memoryManagerCapacity = gpuMemoryManager.totalGpuAllocation();
+   size_t pancakeCapacity = get_preloaded_pool_size();
 
    size_t free_byte ;
    size_t total_byte ;
    CHK_ERR( gpuMemGetInfo( &free_byte, &total_byte) );
    size_t used_mb = (total_byte-free_byte)/(1024*1024);
-   size_t sum_mb = (miniBuffers+batchBuffers+vlasovBuffers+accBuffers+transBuffers+local_cells_capacity+ghost_cells_capacity+memoryManagerCapacity)/(1024*1024);
+   size_t sum_mb = (miniBuffers+batchBuffers+vlasovBuffers+accBuffers+transBuffers+local_cells_capacity+ghost_cells_capacity+memoryManagerCapacity+pancakeCapacity)/(1024*1024);
    size_t local_req_mb = local_cells_size/(1024*1024);
    size_t ghost_req_mb = ghost_cells_size/(1024*1024);
 
@@ -317,6 +344,7 @@ int gpu_reportMemory(const size_t local_cells_capacity, const size_t ghost_cells
       logFile<<"     Local cells:           "<<local_cells_capacity/(1024*1024)<<" Mbytes"<<std::endl;
       logFile<<"     Ghost cells:           "<<ghost_cells_capacity/(1024*1024)<<" Mbytes"<<std::endl;
       logFile<<"     Memory manager:        "<<memoryManagerCapacity/(1024*1024)<<" Mbytes"<<std::endl;
+      logFile<<"     MPI Pancake:           "<<pancakeCapacity/(1024*1024)<<" Mbytes"<<std::endl;
       if (local_req_mb || ghost_req_mb) {
          logFile<<"     Local cells required:  "<<local_req_mb<<" Mbytes"<<std::endl;
          logFile<<"     Ghost cells required:  "<<ghost_req_mb<<" Mbytes"<<std::endl;

arch/gpu_base.hpp

@@ -80,6 +80,7 @@ uint gpu_getThread();
 uint gpu_getMaxThreads();
 int gpu_getDevice();
 uint gpu_getAllocationCount();
+std::size_t get_preloaded_pool_size();
 int gpu_reportMemory(const size_t local_cap=0, const size_t ghost_cap=0, const size_t local_size=0, const size_t ghost_size=0);
 
 unsigned int nextPowerOfTwo(unsigned int n);

memory_report.cpp

@@ -108,6 +108,33 @@ uint64_t get_node_free_memory(){
       fclose(in_file);
    }
 
+   #ifdef USE_GPU
+   // On systems where GPU HBM is exposed as coherent NUMA memory (e.g. GH200),
+   // /proc/meminfo MemFree includes GPU free memory. Subtract free pages in
+   // Movable zones, which is where GPU HBM appears on such systems.
+   // On conventional systems Movable zones are empty, making this a no-op.
+   uint64_t movableFreeBytes = 0;
+   FILE *zoneinfo = fopen("/proc/zoneinfo", "r");
+   if (zoneinfo) {
+      char line[256];
+      bool inMovable = false;
+      while (fgets(line, sizeof(line), zoneinfo)) {
+         if (strncmp(line, "Node ", 5) == 0) {
+            inMovable = strstr(line, "Movable") != NULL;
+         } else if (inMovable) {
+            unsigned long pages = 0;
+            if (sscanf(line, " nr_free_pages %lu", &pages) == 1) {
+               movableFreeBytes += (uint64_t)pages * sysconf(_SC_PAGESIZE);
+            }
+         }
+      }
+      fclose(zoneinfo);
+   }
+   if (movableFreeBytes < mem_proc_free) {
+      mem_proc_free -= movableFreeBytes;
+   }
+   #endif
+
    return mem_proc_free;
 }
 /*! Measures memory consumption and writes it into logfile.
@@ -271,15 +298,47 @@ void report_memory_consumption(
          P::tstep, P::t, sum_mem[2]/GiB, sum_mem[5]/GiB);
    logFile << reportstring;
 
-   logFile << writeVerbose;
-
-   MPI_Comm_free(&interComm);
-   MPI_Comm_free(&nodeComm);
-
    #ifdef USE_GPU
+   const double KiB = 1024.0;
    // TODO: Clear duplicate output
    // (local_cells_capacity, ghost_cells_capacity, local_cells_size, ghost_cells_size)
-   gpu_reportMemory(mem[3], mem[4], mem[0], mem[1]);
+   int gpuReportedUsage = gpu_reportMemory(mem[3], mem[4], mem[0], mem[1]);
+   int sumGpu = 0;
+   struct {
+      int value;
+      int rank;
+   } minGpu, gpuMemUsageLoc, maxGpu;
+   gpuMemUsageLoc.value = gpuReportedUsage;
+   gpuMemUsageLoc.rank = rank;
+
+   MPI_Reduce(&gpuReportedUsage, &sumGpu, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+   MPI_Reduce(&gpuMemUsageLoc, &maxGpu, 1, MPI_2INT, MPI_MAXLOC, 0, MPI_COMM_WORLD);
+   MPI_Reduce(&gpuMemUsageLoc, &minGpu, 1, MPI_2INT, MPI_MINLOC, 0, MPI_COMM_WORLD);
+
+   snprintf(reportstring,512, "(MEM) tstep %i t %.3g %-21s   (GiB/rank; avg, min, max, sum): %-8.3g %-8.3g %-8.3g %-8.3g min rank %i max rank %i\n",
+         P::tstep, P::t, "Total reported GPU memory usage",  (double)sumGpu/((double)nProcs*KiB), (double)minGpu.value/KiB, (double)maxGpu.value/KiB, (double)sumGpu/KiB, minGpu.rank, maxGpu.rank);
+   logFile << reportstring;
+
+   size_t free_byte;
+   size_t total_byte;
+   CHK_ERR( gpuMemGetInfo( &free_byte, &total_byte) );
+   gpuReportedUsage = (total_byte-free_byte)/(1024*1024);
+   sumGpu = 0;
+   gpuMemUsageLoc.value = gpuReportedUsage;
+   gpuMemUsageLoc.rank = rank;
+
+   MPI_Reduce(&gpuReportedUsage, &sumGpu, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+   MPI_Reduce(&gpuMemUsageLoc, &maxGpu, 1, MPI_2INT, MPI_MAXLOC, 0, MPI_COMM_WORLD);
+   MPI_Reduce(&gpuMemUsageLoc, &minGpu, 1, MPI_2INT, MPI_MINLOC, 0, MPI_COMM_WORLD);
+
+   snprintf(reportstring,512, "(MEM) tstep %i t %.3g %-21s (GiB/rank; avg, min, max, sum): %-8.3g %-8.3g %-8.3g %-8.3g min rank %i max rank %i\n",
+         P::tstep, P::t, "Total reported GPU hardware usage",  (double)sumGpu/((double)nProcs*KiB), (double)minGpu.value/KiB, (double)maxGpu.value/KiB, (double)sumGpu/KiB, minGpu.rank, maxGpu.rank);
+   logFile << reportstring;
 
    #endif
+
+   logFile << writeVerbose;
+
+   MPI_Comm_free(&interComm);
+   MPI_Comm_free(&nodeComm);
 }