testpackage addition: restart read on single(or multiple) mpi rank

testpackage/run_tests.sh

@@ -82,12 +82,15 @@ do
     test -e test_prelude.sh && ./test_prelude.sh
 
     # Run the actual simulation
-    if [[ ${single_cell[$run]} ]]; then
-    	$small_run_command $bin --version  > VERSION.txt
-	$small_run_command $bin --run_config=${test_name[$run]}.cfg
+    if [[ ${ranks[$run]} ]]; then
+      if [[ ${ranks[$run]} == "half" ]]; then
+        ranks[$run]=$((SLURM_NTASKS/2))
+      fi
+    	$small_run_command -n ${ranks[$run]} $bin --version  > VERSION.txt
+	    $small_run_command -n ${ranks[$run]} $bin --run_config=${test_name[$run]}.cfg
     else
-	$run_command $bin --version  > VERSION.txt
-	$run_command $bin --run_config=${test_name[$run]}.cfg
+	    $run_command $bin --version  > VERSION.txt
+	    $run_command $bin --run_config=${test_name[$run]}.cfg
     fi
 
     # Run postprocessing script, if it exists

testpackage/small_test_carrington.sh

@@ -42,7 +42,7 @@ export OMP_NUM_THREADS=$t
 
 #command for running stuff
 run_command="srun --mpi=pmix_v3 -n $SLURM_NTASKS "
-small_run_command="srun --mpi=pmix_v3 -n 1"
+small_run_command="srun --mpi=pmix_v3"
 run_command_tools="mpirun -np 1 "
 
 umask 007

testpackage/small_test_carrington_github_ci.sh

@@ -61,7 +61,7 @@ export MALLOC_CONF="abort_conf:true"
 
 #command for running stuff
 run_command="srun --mpi=pmix_v3 -n $SLURM_NTASKS "
-small_run_command="srun --mpi=pmix_v3 -n 1"
+small_run_command="srun --mpi=pmix_v3 "
 run_command_tools="mpirun -np 1 "
 
 umask 007
@@ -144,8 +144,11 @@ for run in ${run_tests[*]}; do
    echo "running ${test_name[$run]} "
 
    # Run the actual simulation
-   if [[ ${single_cell[$run]} ]]; then
-      { $small_run_command $bin --run_config=${test_name[$run]}.cfg 2>&1 1>&3 3>&- | tee $GITHUB_WORKSPACE/stderr.txt; exit ${PIPESTATUS[0]}; } 3>&1 1>&2 | tee $GITHUB_WORKSPACE/stdout.txt
+   if [[ ${ranks[$run]} ]]; then
+      if [[ ${ranks[$run]} == "half" ]]; then
+        ranks[$run]=$((SLURM_NTASKS/2))
+      fi
+      { $small_run_command -n ${ranks[$run]} $bin --run_config=${test_name[$run]}.cfg 2>&1 1>&3 3>&- | tee $GITHUB_WORKSPACE/stderr.txt; exit ${PIPESTATUS[0]}; } 3>&1 1>&2 | tee $GITHUB_WORKSPACE/stdout.txt
    else
       { $run_command $bin --run_config=${test_name[$run]}.cfg 2>&1 1>&3 3>&- | tee $GITHUB_WORKSPACE/stderr.txt; exit ${PIPESTATUS[0]}; } 3>&1 1>&2 | tee $GITHUB_WORKSPACE/stdout.txt
    fi

testpackage/test_definitions_small.sh

@@ -30,7 +30,7 @@ comparison_vlsv[${index}]="fullf.0000000.vlsv fullf.0000001.vlsv fullf.0000002.v
 comparison_phiprof[${index}]="phiprof_0.txt"
 variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v proton"
 variable_components[${index}]="0 0 1 2"
-single_cell[${index}]=1
+ranks[${index}]=1
 ((index+=1))
 
 # 2  basic multipeak acceleration test (dynamic timestep)
@@ -39,7 +39,7 @@ comparison_vlsv[${index}]="fullf.0000001.vlsv"
 comparison_phiprof[${index}]="phiprof_0.txt"
 variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v proton"
 variable_components[${index}]="0 0 1 2"
-single_cell[${index}]=1
+ranks[${index}]=1
 ((index+=1))
 
 # 3  acceleration test w/ substepping
@@ -48,7 +48,7 @@ comparison_vlsv[${index}]="fullf.0000001.vlsv"
 comparison_phiprof[${index}]="phiprof_0.txt"
 variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v proton"
 variable_components[${index}]="0 0 1 2"
-single_cell[${index}]=1
+ranks[${index}]=1
 ((index+=1))
 
 # 4  Helium acceleration
@@ -57,7 +57,7 @@ comparison_vlsv[${index}]="fullf.0000001.vlsv"
 comparison_phiprof[${index}]="phiprof_0.txt"
 variable_names[${index}]="helium/vg_rho helium/vg_v helium/vg_v helium/vg_v"
 variable_components[${index}]="0 0 1 2"
-single_cell[${index}]=1
+ranks[${index}]=1
 ((index+=1))
 
 # 5  Gyration test with protons and antiprotons (multipop)
@@ -66,7 +66,7 @@ comparison_vlsv[${index}]="fullf.0000000.vlsv fullf.0000001.vlsv fullf.0000002.v
 comparison_phiprof[${index}]="phiprof_0.txt"
 variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v proton"
 variable_components[${index}]="0 0 1 2"
-single_cell[${index}]=1
+ranks[${index}]=1
 ((index+=1))
 
 #######
@@ -103,7 +103,7 @@ comparison_vlsv[${index}]="bulk.0000001.vlsv bulk.0000002.vlsv"
 comparison_phiprof[${index}]="phiprof_0.txt"
 variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e"
 variable_components[${index}]="0 0 1 2 0 1 2 0 1 2 0 0"
-single_cell[${index}]=1
+ranks[${index}]=1
 ((index+=1))
 
 #######
@@ -151,7 +151,7 @@ variable_components[${index}]="0 0 1 2 0 0 0 1 2 0 1 2"
 ((index+=1))
 
 #######
-# RESTARTING TESTS (15..18)
+# RESTARTING TESTS (15..22)
 #######
 
 # Restart tests. Writing and reading
@@ -171,35 +171,71 @@ variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b
 variable_components[${index}]="0 0 1 2 0 1 2 0 1 2"
 ((index+=1))
 
-# 17 Restart write with translation, acceleration, and fieldsolver
+# 17 Restart read and propagate with translation only single MPI rank
+test_name[${index}]="restart_read_single_rank"
+comparison_vlsv[${index}]="initial-grid.0000000.vlsv bulk.0000002.vlsv"
+comparison_phiprof[${index}]="phiprof_0.txt"
+variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e"
+variable_components[${index}]="0 0 1 2 0 1 2 0 1 2"
+ranks[${index}]=1 #This will use only 1 MPI rank
+((index+=1))
+
+# 18 Restart read and propagate with translation only half of max MPI ranks
+test_name[${index}]="restart_read_multi_rank"
+comparison_vlsv[${index}]="initial-grid.0000000.vlsv bulk.0000002.vlsv"
+comparison_phiprof[${index}]="phiprof_0.txt"
+variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e"
+variable_components[${index}]="0 0 1 2 0 1 2 0 1 2"
+ranks[${index}]="half"
+((index+=1))
+
+# 19 Restart write with translation, acceleration, and fieldsolver
 test_name[${index}]="restart_write_acc"
 comparison_vlsv[${index}]="bulk.0000001.vlsv"
 comparison_phiprof[${index}]="phiprof_0.txt"
 variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e"
 variable_components[${index}]="0 0 1 2 0 1 2 0 1 2"
 ((index+=1))
 
-# 18 Restart read and propagate with translation, acceleration, and fieldsolver
+# 20 Restart read and propagate with translation, acceleration, and fieldsolver
 test_name[${index}]="restart_read_acc"
 comparison_vlsv[${index}]="initial-grid.0000000.vlsv bulk.0000002.vlsv"
 comparison_phiprof[${index}]="phiprof_0.txt"
 variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e"
 variable_components[${index}]="0 0 1 2 0 1 2 0 1 2"
 ((index+=1))
 
+# 21 Restart read and propagate with translation, acceleration, and fieldsolver single MPI rank
+test_name[${index}]="restart_read_acc_single_rank"
+comparison_vlsv[${index}]="initial-grid.0000000.vlsv bulk.0000002.vlsv"
+comparison_phiprof[${index}]="phiprof_0.txt"
+variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e"
+variable_components[${index}]="0 0 1 2 0 1 2 0 1 2"
+ranks[${index}]=1 #This will use only 1 MPI rank
+((index+=1))
+
+# 22 Restart read and propagate with translation, acceleration, and fieldsolver half of max MPI ranks
+test_name[${index}]="restart_read_multi_rank"
+comparison_vlsv[${index}]="initial-grid.0000000.vlsv bulk.0000002.vlsv"
+comparison_phiprof[${index}]="phiprof_0.txt"
+variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e"
+variable_components[${index}]="0 0 1 2 0 1 2 0 1 2"
+ranks[${index}]="half"
+((index+=1))
+
 #######
-# FIELDSOLVER TESTS (19..20)
+# FIELDSOLVER TESTS (23..24)
 #######
 
-# 19 3D Field solver test
+# 23 3D Field solver test
 test_name[${index}]="test_fp_fsolver_only_3D"
 comparison_vlsv[${index}]="fullf.0000001.vlsv"
 comparison_phiprof[${index}]="phiprof_0.txt"
 variable_names[${index}]="fg_b fg_b fg_b fg_e fg_e fg_e"
 variable_components[${index}]="0 1 2 0 1 2"
 ((index+=1))
 
-# 20 3D Field solver test w/ subcycles
+# 24 3D Field solver test w/ subcycles
 test_name[${index}]="test_fp_substeps"
 comparison_vlsv[${index}]="fullf.0000001.vlsv"
 comparison_phiprof[${index}]="phiprof_0.txt"
@@ -208,18 +244,18 @@ variable_components[${index}]="0 1 2 0 1 2"
 ((index+=1))
 
 #######
-# GLOBAL 2D TESTS (21..22)
+# GLOBAL 2D TESTS (25..26)
 #######
 
-# 21 Very small ecliptic magnetosphere, no subcycling in ACC or FS
+# 25 Very small ecliptic magnetosphere, no subcycling in ACC or FS
 test_name[${index}]="Magnetosphere_small"
 comparison_vlsv[${index}]="bulk.0000001.vlsv bulk.0000002.vlsv"
 comparison_phiprof[${index}]="phiprof_0.txt"
 variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e proton"
 variable_components[${index}]="0 0 1 2 0 1 2 0 1 2"
 ((index+=1))
 
-# 22 Very small polar magnetosphere, with subcycling in ACC or FS
+# 26 Very small polar magnetosphere, with subcycling in ACC or FS
 test_name[${index}]="Magnetosphere_polar_small"
 comparison_vlsv[${index}]="bulk.0000001.vlsv bulk.0000002.vlsv"
 comparison_phiprof[${index}]="phiprof_0.txt"
@@ -228,18 +264,18 @@ variable_components[${index}]="0 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2"
 ((index+=1))
 
 #######
-# GLOBAL 3D TESTS (23..24)
+# GLOBAL 3D TESTS (27..28)
 #######
 
-# 23 Magnetosphere 3D, very small, 40 timesteps
+# 27 Magnetosphere 3D, very small, 40 timesteps
 test_name[${index}]="Magnetosphere_3D_small"
 comparison_vlsv[${index}]="bulk.0000001.vlsv bulk.0000002.vlsv"
 comparison_phiprof[${index}]="phiprof_0.txt"
 variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e"
 variable_components[${index}]="0 0 1 2 0 1 2 0 1 2"
 ((index+=1))
 
-# 24 Ionosphere 3D (not a very physical or successful test at the moment but verifies some things about IG grid and outputs all datareducers)
+# 28 Ionosphere 3D (not a very physical or successful test at the moment but verifies some things about IG grid and outputs all datareducers)
 test_name[${index}]="Ionosphere_small"
 comparison_vlsv[${index}]="bulk.0000001.vlsv bulk.0000002.vlsv"
 comparison_phiprof[${index}]="phiprof_0.txt"
@@ -248,16 +284,16 @@ variable_components[${index}]="0 0 1 2 0 1 2 0 1 2 0 0 0 0"
 ((index+=1))
 
 #######
-# SUBGRID TESTS (25)
+# SUBGRID TESTS (29)
 #######
 
-# 25  Pitch-angle diffusion test
+# 29  Pitch-angle diffusion test
 test_name[${index}]="subgrid_1_diffusion"
 comparison_vlsv[${index}]="bulk.0000001.vlsv"
 comparison_phiprof[${index}]="phiprof_0.txt"
 variable_names[${index}]="proton/vg_rho proton/vg_ptensor_diagonal proton/vg_ptensor_diagonal proton/vg_ptensor_diagonal proton/vg_ptensor_offdiagonal proton/vg_ptensor_offdiagonal proton/vg_ptensor_offdiagonal proton"
 variable_components[${index}]="0 0 1 2 0 1 2"
-single_cell[${index}]=1
+ranks[${index}]=1
 ((index+=1))
 
 # choose tests to run (default: all tests)

testpackage/tests/restart_read_acc_multi_rank/restart_read_acc_multi_rank.cfg

@@ -0,0 +1,112 @@
+project = Flowthrough
+propagate_field = 1
+propagate_vlasov_acceleration = 1
+propagate_vlasov_translation = 1
+dynamic_timestep = 1
+
+ParticlePopulations = proton
+
+[restart]
+#write_as_float = 1
+filename = restart.vlsv
+
+[io]
+write_initial_state = 1
+restart_walltime_interval = 1000 # setting to nonzero activates restart writing on exit
+
+system_write_t_interval = 10.0
+system_write_file_name = bulk
+system_write_distribution_stride = 1
+system_write_distribution_xline_stride = 0
+system_write_distribution_yline_stride = 0
+system_write_distribution_zline_stride = 0
+
+[variables]
+output = vg_rhom
+output = fg_e
+output = fg_b
+output = vg_pressure
+output = populations_vg_v
+output = populations_vg_rho
+output = vg_boundarytype
+output = vg_rank
+output = populations_vg_blocks
+diagnostic = populations_vg_blocks
+
+[gridbuilder]
+x_length = 20
+y_length = 20
+z_length = 1
+x_min = -1.3e8
+x_max = 1.3e8
+y_min = -1.3e8
+y_max = 1.3e8
+z_min = -6500000.0
+z_max = 6500000.0
+t_max = 20
+#dt = 2.0
+
+[proton_properties]
+mass = 1
+mass_units = PROTON
+charge = 1
+
+[proton_vspace]
+vx_min = -640000.0
+vx_max = +640000.0
+vy_min = -640000.0
+vy_max = +640000.0
+vz_min = -640000.0
+vz_max = +640000.0
+vx_length = 32
+vy_length = 32
+vz_length = 32
+
+[proton_sparse]
+minValue = 1.0e-15
+
+[boundaries]
+periodic_x = no
+periodic_y = no
+periodic_z = yes
+boundary = Outflow
+boundary = Maxwellian
+
+[outflow]
+precedence = 3
+
+[proton_outflow]
+reapplyFaceUponRestart = x+
+reapplyFaceUponRestart = y+
+vlasovScheme_face_x+ = Copy
+vlasovScheme_face_y+ = Copy
+face = x+
+face = y+
+
+[maxwellian]
+face = x-
+face = y-
+precedence = 2
+
+[proton_maxwellian]
+dynamic = 0
+file_x- = sw1.dat
+file_y- = sw1.dat
+
+[Flowthrough]
+#emptyBox = 1
+Bx = 1.0e-9
+By = 1.0e-9
+Bz = 1.0e-9
+densityModel = Maxwellian
+
+[proton_Flowthrough]
+# half of the inflow values
+T = 0.5e5
+rho  = 0.5e6
+VX0 = 4e5
+VY0 = 0
+VZ0 = 0
+
+[bailout]
+velocity_space_wall_block_margin = 0

testpackage/tests/restart_read_acc_multi_rank/sw1.dat

@@ -0,0 +1 @@
+0.0 1.0e6 1.0e5 +5.0e5 +2.5e5 0.0 0.0e-9 0.0 0.0

testpackage/tests/restart_read_acc_multi_rank/test_prelude.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+LAST_RESTART=$(ls -t ../restart_write_acc/restart*.vlsv | head -n 1)
+test -e $LAST_RESTART && ln -s $LAST_RESTART ./restart.vlsv