aspen

#!/usr/bin/env bash
# Author: Vishal Koparde, Ph.D.
# CCBR, NCI
# (c) 2021
#
# wrapper script to run the snakemake pipeline
# a) on an interactive node (runlocal) OR
# b) submit to the slurm load scheduler (run)
#
# DISCLAIMER: This wrapper only works on BIOWULF

# set -exo pipefail
# module purge
retries="2"

##########################################################################################
# functions
##########################################################################################

# get_git_commitid_tag() and printbanner() are provided by the sourced
# ccbr_pipeline_logging.sh library (see initial setup section below).

##########################################################################################
# initial setup
##########################################################################################

# ## setting PIPELINE_HOME
export PIPELINE_HOME=$(readlink -f $(dirname "$0"))

# set snakefile
SNAKEFILE="${PIPELINE_HOME}/workflow/Snakefile"

VERSIONFILE="${PIPELINE_HOME}/VERSION"

##########################################################################################
# Some more set up
##########################################################################################

PYTHONVERSION="python/3.10"
SNAKEMAKEVERSION="snakemake"
#SINGULARITYVERSION="singularity/3.7.4"
SINGULARITYVERSION="singularity"
export ASPENVERSION=$(head -n1 $VERSIONFILE|awk '{print $1}')

# ## source shared structured-logging / pipeline-state-marker library
# (staged in workflow/scripts/ pending migration to ccbr_tools; see
# plans_reviews/PLAN_shared_pipeline_logging_library.md Phase B)
export PIPELINE_NAME="ASPEN"
export PIPELINE_VERSION="${ASPENVERSION}"
# WORKDIR isn't parsed from argv until main() runs (it's the last line of this
# script), so at this point it's always unset. ccbr_pipeline_logging.sh
# requires SNAKEMAKE_LOG_PATH to be non-empty at source-time (its ":?" guard
# treats an empty string the same as unset), so we can't just leave this
# blank -- use an obviously-fake placeholder instead of a plausible-looking
# one like "/snakemake.log", so any accidental early use fails loudly (file
# not found) rather than silently writing to the filesystem root. main()
# refreshes this with the real value once WORKDIR is known.
export SNAKEMAKE_LOG_PATH="${WORKDIR:-__WORKDIR_NOT_YET_SET__}/snakemake.log"
if command -v ccbr_pipeline_logging.sh >/dev/null 2>&1; then
  source "$(command -v ccbr_pipeline_logging.sh)"
elif [[ -f "${PIPELINE_HOME}/workflow/scripts/ccbr_pipeline_logging.sh" ]]; then
  source "${PIPELINE_HOME}/workflow/scripts/ccbr_pipeline_logging.sh"
else
  echo "ERROR ccbr_pipeline_logging.sh not found (checked PATH and ${PIPELINE_HOME}/workflow/scripts/)" >&2
  exit 1
fi

# get github commit tag
GIT_COMMIT_TAG=$(get_git_commitid_tag $PIPELINE_HOME)

# set defaults
export GENOME="hg38"
SUPPORTED_GENOMES="hg19 hg38 mm10 mmul10 bosTau9 hs1 hs1_chrR"

# essential files
# these are relative to the workflows' base folder
# these are copied into the WORKDIR
ESSENTIAL_FILES="config/config.yaml config/samples.tsv resources/cluster.json resources/tools.yaml"
ESSENTIAL_FOLDERS="workflow/scripts"

SCRIPTNAME="$0"
SCRIPTDIRNAME=$(readlink -f $(dirname $0))
SCRIPTBASENAME=$(readlink -f $(basename $0))

# set extra singularity bindings comma separated
# /data/CCBR_Pipeliner required for fastq_screen_config.txt to work
EXTRA_SINGULARITY_BINDS="/lscratch,/data/CCBR_Pipeliner"

##########################################################################################
# USAGE
##########################################################################################

function usage() { cat << EOF

##########################################################################################

Welcome to
EOF
printbanner "ASPEN" "$ASPENVERSION"
cat << EOF

ATAC-Seq Analysis Pipeline

##########################################################################################

This pipeline was built by CCBR (https://bioinformatics.ccr.cancer.gov/ccbr)
Please contact Vishal Koparde for comments/questions (vishal.koparde@nih.gov)

##########################################################################################

Here is a list of genome supported by aspen:

  * hg19          [Human]
  * hg38          [Human]
  * mm10          [Mouse]
  * mmul10        [Macaca mulatta(Rhesus monkey) or rheMac10]
  * bosTau9       [Bos taurus(cattle)]
  * hs1           [Human T2T-CHM13]
  * hs1_chrR      [Human T2T-CHM13 + chrR rDNA unit]

aspen calls peaks using the following tools:

 * MACS2
 * Genrich        [RECOMMENDED FOR USE]

USAGE:
  bash ${SCRIPTNAME} -w/--workdir=<WORKDIR> -m/--runmode=<RUNMODE>

Required Arguments:
1.  WORKDIR     : [Type: String]: Absolute or relative path to the output folder with write permissions.

2.  RUNMODE     : [Type: String] Valid options:
    * init      : initialize workdir
    * dryrun    : dry run snakemake to generate DAG
    * run       : run with slurm
    * runlocal  : run without submitting to sbatch
    ADVANCED RUNMODES (use with caution!!)
    * unlock    : unlock WORKDIR if locked by snakemake NEVER UNLOCK WORKDIR WHERE PIPELINE IS CURRENTLY RUNNING!
    * reconfig  : recreate config file in WORKDIR (debugging option) EDITS TO config.yaml WILL BE LOST!
    * reset     : DELETE workdir dir and re-init it (debugging option) EDITS TO ALL FILES IN WORKDIR WILL BE LOST!
    * printbinds: print singularity binds (paths)
    * local     : same as runlocal

Optional Arguments:

--genome|-g     : genome eg. hg38
--manifest|-s   : absolute path to samples.tsv. This will be copied to output folder                    (--runmode=init only)
--help|-h       : print this help

Example commands:
  bash ${SCRIPTNAME} -w=/my/output/folder -m=init
  bash ${SCRIPTNAME} -w=/my/output/folder -m=dryrun
  bash ${SCRIPTNAME} -w=/my/output/folder -m=run

##########################################################################################

VersionInfo:
  python          : $PYTHONVERSION
  snakemake       : $SNAKEMAKEVERSION
  pipeline_home   : $PIPELINE_HOME
  git commit/tag  : $GIT_COMMIT_TAG
  aspen_version   : v${ASPENVERSION}

##########################################################################################

EOF
}

##########################################################################################
# ERR
##########################################################################################

function err() { log_error "$@"; exit 1 1>&2; }


##########################################################################################
# INIT
##########################################################################################

function init() {

# This function initializes the workdir by:
# 1. creating the working dir
# 2. copying essential files like config.yaml and samples.tsv into the workdir
# 3. setting up logs folder

printbanner "ASPEN" "$ASPENVERSION"

log_step "[init] Preparing workdir"
log_info "Workdir: $WORKDIR"

# create output folder
if [ -d $WORKDIR ];then err "Folder $WORKDIR already exists!"; fi
mkdir -p $WORKDIR

# copy essential files
# for f in $ESSENTIAL_FILES;do
f="${PIPELINE_HOME}/config/config.yaml"
log_info "Copying essential file: $f"
fbn=$(basename $f)
cat $f | envsubst '$PIPELINE_HOME $WORKDIR $GENOME $ASPENVERSION' > $WORKDIR/$fbn

for f in ${PIPELINE_HOME}/resources/cluster.json ${PIPELINE_HOME}/resources/tools.yaml
do
  log_info "Copying essential file: $f"
  fbn=$(basename $f)
  cp $f $WORKDIR/$fbn
done

if [[ "$MANIFEST_SUPPLIED" == "true" ]];then
  f=$MANIFEST
  log_info "Copying essential file: $f"
  fbn=$(basename $f)
  cp $f $WORKDIR/$fbn
fi

if [[ "$MANIFEST_SUPPLIED" == "false" ]];then
f=$MANIFEST
log_info "Copying essential file: $f"
fbn=$(basename $f)
cat $f | envsubst '$PIPELINE_HOME $WORKDIR $GENOME $ASPENVERSION' > $WORKDIR/$fbn
fi

# copy essential folders
for f in $ESSENTIAL_FOLDERS;do
  # rsync -az --progress ${PIPELINE_HOME}/$f $WORKDIR/
  cp -r ${PIPELINE_HOME}/$f ${WORKDIR}/
done

cd ${WORKDIR}

#create log folder
if [ ! -d $WORKDIR/logs ]; then mkdir -p $WORKDIR/logs; log_ok "Logs Dir: $WORKDIR/logs"; fi

log_ok "Initialization completed for $WORKDIR"
log_next "Edit $WORKDIR/config.yaml and $WORKDIR/samples.tsv"

}

##########################################################################################
# set random str
##########################################################################################

function _set_rand_str() {
  x=$(mktemp)
  rm -rf $x
  RAND_STR=$(echo $x|awk -F"." '{print $NF}')
}

##########################################################################################
# CHECK ESSENTIAL FILES
##########################################################################################

function check_essential_files() {

# Checks if files essential to start running the pipeline exist in the workdir

  if [ ! -d $WORKDIR ];then err "Folder $WORKDIR does not exist!"; fi
  for f in config.yaml samples.tsv cluster.json tools.yaml; do
    if [ ! -f $WORKDIR/$f ]; then err "Error: '${f}' file not found in workdir ... initialize first!";fi
  done

}

##########################################################################################
# set --reuun-triggers to "mtime" ... only for newer snakemake
##########################################################################################

function set_snakemake_rerun_triggers() {
  runcheck
  snakemakeVer=$(snakemake --version 2>/dev/null)
  verlte() {
[  "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ]
  }
  verlt() {
[ "$1" = "$2" ] && return 1 || verlte $1 $2
  }
  snakemakeOld=$(verlt $snakemakeVer 7.8 && echo "yes" || echo "no") # check if snakemake is older than 7.8
  if [[ "$snakemakeVer" == "" || "$snakemakeOld" == "no" ]];then
    RERUNTRIGGERS="--rerun-triggers mtime"
  else
    RERUNTRIGGERS=""
  fi
}
# Force Snakemake to Ignore Software Environment Changes ... like new masterdocker
RERUNTRIGGERS="--rerun-triggers mtime"
##########################################################################################
# RECONFIG ... recreate config.yaml and overwrite old version
##########################################################################################

function reconfig(){
# Rebuild config file and replace the config.yaml in the WORKDIR
# this is only for dev purposes when new key-value pairs are being added to the config file

  check_essential_files
  cat ${PIPELINE_HOME}/config/config.yaml |\
    envsubst '$PIPELINE_HOME $WORKDIR $GENOME $ASPENVERSION' \
    > $WORKDIR/config.yaml
  log_ok "$WORKDIR/config.yaml has been updated"

}

##########################################################################################
# SET SINGULARITY BINDS ... bind required singularity folders appropriately
##########################################################################################

function set_singularity_binds(){
# this functions tries find what folders to bind
# biowulf specific
  echo "$PIPELINE_HOME" > ${WORKDIR}/tmp1
  echo "$WORKDIR" >> ${WORKDIR}/tmp1
  grep -o '\/.*' <(cat ${WORKDIR}/config.yaml ${WORKDIR}/samples.tsv) | tr '\t' '\n' | grep -v ' \|\/\/' | sort | uniq >> ${WORKDIR}/tmp1
  grep "^/" ${WORKDIR}/tmp1 | grep /vf    | awk -F'/' -v OFS='/' '{print $1,$2,$3,$4,$5}' | sort | uniq > ${WORKDIR}/tmp2
  grep "^/" ${WORKDIR}/tmp1 | grep -v /vf | awk -F'/' -v OFS='/' '{print $1,$2,$3}'       | sort | uniq > ${WORKDIR}/tmp3
  while read a;do readlink -f $a;done < ${WORKDIR}/tmp3 > ${WORKDIR}/tmp4
  binds=$(cat ${WORKDIR}/tmp2 ${WORKDIR}/tmp3 ${WORKDIR}/tmp4 | sort | uniq | tr '\n' ',')
  rm -f ${WORKDIR}/tmp?
  binds=$(echo $binds | awk '{print substr($1,1,length($1)-1)}')
  SINGULARITY_BINDS="-B $EXTRA_SINGULARITY_BINDS,$binds"
  SINGULARITY_STR="--use-singularity --singularity-args \"${SINGULARITY_BINDS}\" --singularity-prefix \"${SING_CACHE_DIR}\" "
  log_info "Singularity bind string: $SINGULARITY_STR"
}


function set_cluster_arg(){
  CLUSTER_SBATCH_CMD="sbatch --cpus-per-task {cluster.threads} -p {cluster.partition} -t {cluster.time} --mem {cluster.mem} --job-name {cluster.name} --output {cluster.output} --error {cluster.error}"
  clustername=$(scontrol show config|grep ClusterName|awk '{print $NF}')
  if [[ "$clustername" == "biowulf" ]];then
    CLUSTER_SBATCH_CMD="${CLUSTER_SBATCH_CMD} --gres {resources.gres}"
  fi
}

# log_info/step/ok/warn/error/next/divider, print_versions, json_escape,
# write_pipeline_state_marker, and run_jobby_best_effort are provided by the
# sourced ccbr_pipeline_logging.sh library (see initial setup section above).

##########################################################################################
# SYNC GENOME FROM WORKDIR CONFIG ... display the genome actually configured
##########################################################################################

function sync_genome_from_workdir_config() {
# -g/--genome (and its "hg38" default) only matter for `init`/`reconfig`, where
# they're used to template a NEW config.yaml. For run/runlocal/dryrun/unlock on
# an EXISTING workdir, the workdir's config.yaml is authoritative -- Snakemake
# is invoked with --configfile $WORKDIR/config.yaml and never sees $GENOME.
# Without this, the "Genome:" line in the run summary could silently show the
# CLI/default genome instead of the one actually configured in the workdir,
# which is misleading (though the run itself is unaffected).

  local workdir_genome
  if [[ -f "${WORKDIR}/config.yaml" ]]; then
    workdir_genome=$(grep -E '^genome:' "${WORKDIR}/config.yaml" | head -n1 | sed -E 's/^genome:[[:space:]]*"?([^"[:space:]]+)"?.*/\1/')
    if [[ -n "${workdir_genome}" && "${workdir_genome}" != "${GENOME}" ]]; then
      log_warn "Genome '${GENOME}' (CLI/default) differs from '${workdir_genome}' configured in ${WORKDIR}/config.yaml; using the workdir's value. To change the genome for an existing workdir, edit ${WORKDIR}/config.yaml directly (-g/--genome only applies to init/reconfig)."
      GENOME="${workdir_genome}"
    fi
  fi
}

##########################################################################################
# RUNCHECK ... check essential files and load required packages
##########################################################################################

function runcheck(){
# Check "job-essential" files and load required modules

  check_essential_files
  sync_genome_from_workdir_config
  # MODULE_STR="module load $PYTHONVERSION $SNAKEMAKEVERSION singularity"
  MODULE_STR=$(
cat << END_HEREDOC
command -V python 2>/dev/null || module load $PYTHONVERSION || (>&2 echo "module $PYTHONVERSION could not be loaded"; exit 1)
command -V snakemake 2>/dev/null || module load $SNAKEMAKEVERSION || (>&2 echo "module $SNAKEMAKEVERSION could not be loaded"; exit 1)
command -V singularity 2>/dev/null || module load singularity || (>&2 echo "module singularity could not be loaded"; exit 1)
END_HEREDOC
  )
# If not on BIOWULF then change MODULE_STR such that python, snakemake and singularity are all in PATH

command -V python 2>/dev/null || module load $PYTHONVERSION || (>&2 echo "module $PYTHONVERSION could not be loaded"; exit 1)
command -V snakemake 2>/dev/null || module load $SNAKEMAKEVERSION || (>&2 echo "module $SNAKEMAKEVERSION could not be loaded"; exit 1)
command -V singularity 2>/dev/null || module load singularity || (>&2 echo "module singularity could not be loaded"; exit 1)

}

##########################################################################################
# DRYRUN ... also run automatically before actual run
##########################################################################################

function dryrun() {
# Dry-run
  log_step "[dryrun] Running preflight checks"
  runcheck
  if [ -f ${WORKDIR}/dryrun.log ]; then
    modtime=$(stat ${WORKDIR}/dryrun.log |grep Modify|awk '{print $2,$3}'|awk -F"." '{print $1}'|sed "s/ //g"|sed "s/-//g"|sed "s/://g")
    mv ${WORKDIR}/dryrun.log ${WORKDIR}/logs/dryrun.${modtime}.log
    if [ -f ${WORKDIR}/dryrun_git_commit.txt ];then
      mv ${WORKDIR}/dryrun_git_commit.txt ${WORKDIR}/logs/dryrun_git_commit.${modtime}.txt
    fi
  fi
  set -o pipefail
  run "--dry-run" 2>&1 | tee ${WORKDIR}/dryrun.log
  dryrun_rc=${PIPESTATUS[0]}
  set +o pipefail
  if [ "${dryrun_rc}" -ne 0 ];then
    log_error "Dry-run failed with exit code ${dryrun_rc}. See ${WORKDIR}/dryrun.log for details."
    exit "${dryrun_rc}"
  fi
  echo "Git Commit/Tag: $GIT_COMMIT_TAG" > ${WORKDIR}/dryrun_git_commit.txt
  log_ok "Dry-run completed."
  log_info "Full dry-run output captured in ${WORKDIR}/dryrun.log"
  if grep -qi "Nothing to be done" "${WORKDIR}/dryrun.log"; then
    log_info "Dry-run reports no pending jobs for this workdir; skipping NEXT prompt."
  else
    log_next "Submit run with: aspen --workdir=$WORKDIR --runmode=run"
  fi
}

##########################################################################################
# UNLOCK
##########################################################################################

function unlock() {
# Unlock the workdir if previous snakemake run ended abruptly

  runcheck
  run "--unlock"
}

##########################################################################################
# DAG
##########################################################################################

function dag() {
  runcheck
  snakemake -s $SNAKEFILE --configfile ${WORKDIR}/config.yaml --forceall --dag |dot -Teps > ${WORKDIR}/dag.eps
}

##########################################################################################
# RUNLOCAL ... run directly on local interactive node ... no submission to SLURM
##########################################################################################

function runlocal() {
# If the pipeline is fired up on an interactive node (with sinteractive), this function runs the pipeline

  runcheck
  set_singularity_binds
  set_cluster_arg
  if [ "$SLURM_JOB_ID" == "" ];then err "runlocal can only be done on an interactive node"; fi
  module load singularity
  log_step "[runlocal] Starting local interactive execution"
  log_divider
  log_info "Singularity/Apptainer Cache Directory: $SING_CACHE_DIR"
  log_divider
  print_versions
  run "--dry-run" && log_ok "Dry-run was successful. Starting local execution." && run "local"
}

##########################################################################################
# RUNSLURM ... submit head job to slurm which will spawn other jobs on SLURM
##########################################################################################

function runslurm() {
# Submit the execution of the pipeline to the biowulf job scheduler (slurm)

  runcheck
  if [[ ! -r "$CLUSTERSTATUSCMD" || ! -x "$CLUSTERSTATUSCMD" ]]; then
    err "Cluster status script '$CLUSTERSTATUSCMD' is missing or not executable."
  fi
  set_singularity_binds
  set_cluster_arg
  log_divider
  log_step "[run] Preparing SLURM submission"
  log_info "ASPEN Run Summary"
  log_info "Mode:      SLURM"
  log_info "Workdir:   $WORKDIR"
  log_info "Genome:    $GENOME"
  log_info "Log file:  ${WORKDIR}/snakemake.log"
  log_divider
  log_info "Singularity/Apptainer Cache Directory: $SING_CACHE_DIR"
  log_divider
  print_versions
  run "--dry-run" && log_ok "Dry-run was successful." && run "slurm"
}

##########################################################################################
# CREATE RUNINFO ... create runinfo.yaml in workdir
##########################################################################################

function create_runinfo {
  modtime=$1
  if [ "$modtime" == "" ];then
   modtime=$(stat ${WORKDIR}/runinfo.yaml 2>/dev/null|grep Modify|awk '{print $2,$3}'|awk -F"." '{print $1}'|sed "s/ //g"|sed "s/-//g"|sed "s/://g")
  fi
  if [ -f ${WORKDIR}/runinfo.yaml ];then
    mv ${WORKDIR}/runinfo.yaml ${WORKDIR}/logs/runinfo.${modtime}.yaml
  fi
  echo "Pipeline Dir: $PIPELINE_HOME" > ${WORKDIR}/runinfo.yaml
  echo "Git Commit/Tag: $GIT_COMMIT_TAG" >> ${WORKDIR}/runinfo.yaml
  userlogin=$(whoami)
  if [[ `which finger 2>/dev/null` ]];then
          username=$(finger $userlogin |grep ^Login | awk -F"Name: " '{print $2}');
  elif [[ `which lslogins 2>/dev/null` ]];then
          username=$(lslogins -u $userlogin | grep ^Geco | awk -F": " '{print $2}' | awk '{$1=$1;print}');
  else username="NOTFOUND";fi
  echo "Login: $userlogin" >> ${WORKDIR}/runinfo.yaml
  echo "Name: $username" >> ${WORKDIR}/runinfo.yaml
  g=$(groups)
  echo "Groups: $g" >> ${WORKDIR}/runinfo.yaml
  d=$(date)
  echo "Date/Time: $d" >> ${WORKDIR}/runinfo.yaml
}

##########################################################################################
# PRERUN CLEANUP ... get ready to run .. park old logs
##########################################################################################

function preruncleanup() {
# Cleanup function to rename/move files related to older runs to prevent overwriting them.

  log_step "[run] Preparing workdir for new execution"

  # check initialization
  check_essential_files

  cd $WORKDIR
  ## Archive previous run files
  if [ -f ${WORKDIR}/snakemake.log ];then
    modtime=$(stat ${WORKDIR}/snakemake.log |grep Modify|awk '{print $2,$3}'|awk -F"." '{print $1}'|sed "s/ //g"|sed "s/-//g"|sed "s/://g")
    mv ${WORKDIR}/snakemake.log ${WORKDIR}/logs/snakemake.${modtime}.log
    if [ -f ${WORKDIR}/snakemake.log.jobby ];then
      mv ${WORKDIR}/snakemake.log.jobby ${WORKDIR}/logs/snakemake.${modtime}.log.jobby
    fi
    if [ -f ${WORKDIR}/snakemake.log.jobby.short ];then
      mv ${WORKDIR}/snakemake.log.jobby.short ${WORKDIR}/logs/snakemake.${modtime}.log.jobby.short
    fi
    if [ -f ${WORKDIR}/runslurm_snakemake_report.html ];then
      mv ${WORKDIR}/runslurm_snakemake_report.html ${WORKDIR}/logs/runslurm_snakemake_report.${modtime}.html
    fi
    if [ -f ${WORKDIR}/run_git_commit.txt ];then
      mv ${WORKDIR}/run_git_commit.txt ${WORKDIR}/logs/run_git_commit.${modtime}.txt
    fi
  fi
  nslurmouts=$(find ${WORKDIR} -maxdepth 1 -name "slurm-*.out" |wc -l)
  if [ "$nslurmouts" != "0" ];then
    for f in $(ls ${WORKDIR}/slurm-*.out);do mv ${f} ${WORKDIR}/logs/;done
  fi

  # runinfo not needed as now handled by spooker from ccbr_tools v0.4+
  # create_runinfo $modtime

}


function run() {
# RUN function
# argument1 can be:
# 1. local or
# 2. dryrun or
# 3. unlock or
# 4. slurm

echo "Git Commit/Tag: $GIT_COMMIT_TAG" > ${WORKDIR}/run_git_commit.txt

##########################################################################################
# local run
##########################################################################################
  if [ "$1" == "local" ];then

  preruncleanup
  log_ok "[runlocal] Workdir ready"

  # --use-envmodules \
  _set_rand_str

  cat > ${HOME}/${RAND_STR} << EOF
#/bin/bash
set -exo pipefail

$MODULE_STR

$EXPORT_SING_CACHE_DIR_CMD

snakemake -s $SNAKEFILE \
--directory $WORKDIR \
--printshellcmds \
$SINGULARITY_STR \
--latency-wait 120 \
--configfile $CONFIGFILE \
--cores all \
--rerun-incomplete \
${RERUNTRIGGERS} \
--restart-times ${retries} \
--keep-going \
--stats ${WORKDIR}/snakemake.stats \
2>&1|tee ${WORKDIR}/snakemake.log

# if [ "$?" -eq "0" ];then
#   snakemake -s $SNAKEFILE \
#   --report ${WORKDIR}/runlocal_snakemake_report.html \
#   --directory $WORKDIR \
#   --configfile $CONFIGFILE
# fi

EOF

    if [[ "$EXPORT_SING_CACHE_DIR_CMD" != "" ]];then
      $EXPORT_SING_CACHE_DIR_CMD && \
      bash ${HOME}/${RAND_STR}
    else
      bash ${HOME}/${RAND_STR}
    fi

    rm -rf ${HOME}/${RAND_STR}

##########################################################################################
# slurm run
##########################################################################################

  elif [ "$1" == "slurm" ];then

  preruncleanup
  PARTITIONS="norm"
# find buy in partitions
  ACCOUNT_SPONSOR=$(sacctmgr -rn list user | awk '{print $2}')
  BUYINPARTITIONS=$(scontrol show partitions 2>/dev/null | grep -i $ACCOUNT_SPONSOR -B1 | grep '^PartitionName' | cut -d '=' -f2 | grep -iv 'gpu'| tr '\n' ',' | sed 's/.$//')
# remove "norm" partition
  BUYINPARTITIONS=$(echo $BUYINPARTITIONS | tr ',' '\n' | grep -v "norm" | tr '\n' ',' | sed 's/.$//')

if [ ! -z "$BUYINPARTITIONS" ];then
# as only 2 partitions are allow and 1 is norm .. so we randomly pick the first buyin partition
  BUYINPARTITIONS=$(echo $BUYINPARTITIONS | tr ',' '\n' | head -n1)
  PARTITIONS="norm,$BUYINPARTITIONS"
fi

  # Only create submit_script.sbatch if it doesn't already exist for this
  # workdir. Every script created from here on is generated by this same
  # up-to-date template (state markers, progress monitor, module purge all
  # included from the start), so there's no future scenario where an
  # existing script could be missing a feature the current template
  # provides -- nothing to detect or migrate. Reuse the existing script
  # as-is on subsequent runs. (Previously this re-derived the same
  # conclusion on every run via fragile grep-based feature checks that were
  # actually broken -- see CCBR/ASPEN#119.)
  if [[ ! -f "${WORKDIR}/submit_script.sbatch" ]]; then
  cat > ${WORKDIR}/submit_script.sbatch << EOF
#!/bin/bash
#SBATCH --job-name="aspen"
#SBATCH --mem=40g
#SBATCH --partition=$PARTITIONS
#SBATCH --time=96:00:00
#SBATCH --cpus-per-task=2

# Start from a clean module environment -- SLURM jobs otherwise inherit the
# submitting shell's full environment (modules already loaded, PATH,
# PYTHONPATH, etc.). A stale/different module version (e.g. an old
# ccbrpipeliner loaded interactively before submission) can silently shadow
# the tools this script explicitly loads below. See CCBR/ASPEN#144.
module purge >/dev/null 2>&1 || true

export PIPELINE_NAME="ASPEN"
export PIPELINE_VERSION="${ASPENVERSION}"
export WORKDIR="${WORKDIR}"
export PIPELINE_HOME="${PIPELINE_HOME}"
export SNAKEMAKE_LOG_PATH="${WORKDIR}/snakemake.log"
if command -v ccbr_pipeline_logging.sh >/dev/null 2>&1; then
  source "\$(command -v ccbr_pipeline_logging.sh)"
elif [[ -f "${WORKDIR}/scripts/ccbr_pipeline_logging.sh" ]]; then
  source "${WORKDIR}/scripts/ccbr_pipeline_logging.sh"
else
  module load ccbrpipeliner >/dev/null 2>&1 || true
  if command -v ccbr_pipeline_logging.sh >/dev/null 2>&1; then
    source "\$(command -v ccbr_pipeline_logging.sh)"
  else
    # Can't source the library (so write_pipeline_state_marker_job() doesn't
    # exist yet) -- write a plain-text pipeline.failed marker directly so a
    # user polling pipeline.* files (rather than squeue) finds out about this
    # failure instead of seeing a stale pipeline.running left over from the
    # head-node's earlier "sbatch_submitted" write. Mirrors the human-readable
    # format _pipeline_write_status_json() uses elsewhere without depending on
    # the library that just failed to load. See CCBR/ASPEN#119.
    {
      printf 'State    : failed\n'
      printf 'Reason   : ccbr_pipeline_logging_sh_not_found\n'
      printf 'Job ID   : %s\n' "\${SLURM_JOB_ID:-NA}"
      printf 'Updated  : %s\n' "\$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
    } > "${WORKDIR}/pipeline.failed"
    rm -f "${WORKDIR}/pipeline.running"
    echo "ERROR ccbr_pipeline_logging.sh not found (checked PATH, ${WORKDIR}/scripts/, and module ccbrpipeliner)" >&2
    exit 1
  fi
fi

trap 'write_pipeline_state_marker_job canceled "signal_SIGTERM" "\${SLURM_JOB_ID:-NA}" 130; exit 130' SIGTERM
trap 'write_pipeline_state_marker_job canceled "signal_SIGINT" "\${SLURM_JOB_ID:-NA}" 130; exit 130' SIGINT

$MODULE_STR

cd \$SLURM_SUBMIT_DIR

$EXPORT_SING_CACHE_DIR_CMD

_START_EPOCH=\$(date +%s)
_START_TS=\$(date -u +"%Y-%m-%dT%H:%M:%SZ")

_progress_monitor &
_monitor_pid=\$!

snakemake -s $SNAKEFILE --directory $WORKDIR $SINGULARITY_STR --printshellcmds --latency-wait 120 --configfile $CONFIGFILE --cluster-config $CLUSTERFILE --default-resources "gres='lscratch:256'" --cluster "$CLUSTER_SBATCH_CMD" --cluster-status "$CLUSTERSTATUSCMD" --cluster-cancel scancel -j 500 --rerun-incomplete ${RERUNTRIGGERS} --restart-times ${retries} --keep-going --stats ${WORKDIR}/snakemake.stats 2>&1 | tee ${WORKDIR}/snakemake.log
snakemake_rc="\${PIPESTATUS[0]}"
kill "\$_monitor_pid" 2>/dev/null || true
wait "\$_monitor_pid" 2>/dev/null || true

if ! command -v jobby >/dev/null 2>&1; then
  module load ccbrpipeliner >/dev/null 2>&1 || true
fi
if command -v jobby >/dev/null 2>&1; then
  jobby --tsv "${WORKDIR}/snakemake.log" | tee "${WORKDIR}/snakemake.log.jobby" >/dev/null 2>&1 || true
fi

if [ "\${snakemake_rc}" -eq "0" ]; then
  snakemake -s $SNAKEFILE --directory $WORKDIR --report ${WORKDIR}/runslurm_snakemake_report.html --configfile $CONFIGFILE
  report_rc="\$?"
  if [ "\${report_rc}" -eq "0" ]; then
    write_pipeline_state_marker_job completed "snakemake_and_report_succeeded" "\${SLURM_JOB_ID:-NA}" 0
  else
    write_pipeline_state_marker_job failed "report_generation_failed" "\${SLURM_JOB_ID:-NA}" "\${report_rc}"
    exit "\${report_rc}"
  fi
else
  write_pipeline_state_marker_job failed "snakemake_failed" "\${SLURM_JOB_ID:-NA}" "\${snakemake_rc}"
  exit "\${snakemake_rc}"
fi

EOF
  log_ok "Created ${WORKDIR}/submit_script.sbatch"
  fi

  write_pipeline_state_marker running "submission_started"

  if ! sbatch_output=$(sbatch --parsable ${WORKDIR}/submit_script.sbatch 2>&1); then
    write_pipeline_state_marker failed "sbatch_submission_failed"
    run_jobby_best_effort
    log_error "Failed to submit ${WORKDIR}/submit_script.sbatch"
    log_error "sbatch output: ${sbatch_output}"
    log_next "Check ${WORKDIR}/submit_script.sbatch and re-run: aspen --workdir=$WORKDIR --runmode=run"
    exit 1
  fi

  sbatch_job_id=$(echo "${sbatch_output}" | cut -d';' -f1 | tr -d '[:space:]')
  if [[ -z "${sbatch_job_id}" ]]; then
    sbatch_job_id="NA"
  fi
  write_pipeline_state_marker running "sbatch_submitted" "${sbatch_job_id}"

  log_divider
  log_ok "Job submitted successfully (SLURM job ID: ${sbatch_job_id})"
  log_next "Monitor:  squeue -u $USER"
  log_next "Progress: tail -f ${WORKDIR}/snakemake.log"
  log_next "Status:   ls -1 ${WORKDIR}/pipeline.*"
  log_next "Sidecar:  ${WORKDIR}/pipeline.status.json"
  log_divider


##########################################################################################
# unlock or dry-run
##########################################################################################

  else # for unlock and dryrun

  _set_rand_str
  set_cluster_arg

  cat > ${HOME}/${RAND_STR} << EOF
#/bin/bash
set -exo pipefail
$MODULE_STR

snakemake $1 -s $SNAKEFILE \
--directory $WORKDIR \
--printshellcmds \
--latency-wait 120 \
--configfile $CONFIGFILE \
--cluster-config $CLUSTERFILE \
--default-resources "gres='lscratch:256'" \
--cluster "$CLUSTER_SBATCH_CMD" \
-j 500 \
--rerun-incomplete \
$RERUNTRIGGERS \
--keep-going \
--reason \
--stats ${WORKDIR}/snakemake.stats

EOF

    if [[ "$EXPORT_SING_CACHE_DIR_CMD" != "" ]];then
      $EXPORT_SING_CACHE_DIR_CMD && \
      bash ${HOME}/${RAND_STR}
    else
      bash ${HOME}/${RAND_STR}
    fi

    rm -rf ${HOME}/${RAND_STR}

  fi

}

##########################################################################################
# RESET ... delete workdir and then initialize
##########################################################################################

function reset() {
# Delete the workdir and re-initialize it
  printbanner "ASPEN" "$ASPENVERSION"
  log_info "Working Dir: $WORKDIR"
  if [ ! -d $WORKDIR ];then err "Folder $WORKDIR does not exist!";fi
  log_warn "Deleting $WORKDIR"
  rm -rf $WORKDIR
  log_step "Re-initializing $WORKDIR"
  init
}

##########################################################################################
# Print singularity binds and exist
##########################################################################################

function printbinds(){
  printbanner "ASPEN" "$ASPENVERSION"
  set_singularity_binds
  echo $SINGULARITY_BINDS
}

##########################################################################################
# MAIN ... command line argument parsing
##########################################################################################

function main(){
# Main function which parses all arguments

  if [ $# -eq 0 ]; then usage; exit 1; fi

  for i in "$@"
  do
  case $i in
      -m=*|--runmode=*)
        RUNMODE="${i#*=}"
      ;;
      -w=*|--workdir=*)
        WORKDIR="${i#*=}"
      ;;
      -c=*|--singcache=*)
        SING_CACHE_DIR="${i#*=}"
      ;;
      -s=*|--manifest=*)
        MANIFEST="${i#*=}"
        if [ ! -f $MANIFEST ];then err "File $MANIFEST does NOT exist!";fi
      ;;
      -g=*|--genome=*)
        export GENOME="${i#*=}"
        found=0
        for g in $SUPPORTED_GENOMES;do
          if [[ "$GENOME" == "$g" ]];then
            found=1
            break
          fi
        done
        if [[ "$found" == "0" ]];then
          err "$GENOME is not supported by ASPEN; Supported genomes are: $SUPPORTED_GENOMES"
          exit 1
        fi
      ;;
      --version)
        printbanner "ASPEN" "${ASPENVERSION}" && exit 0;
      ;;
      -h|--help)
        usage && exit 0;;
      *)
        err "Unknown argument:     $i!"    # unknown option
      ;;
  esac
  done
  export WORKDIR=$(readlink -f "$WORKDIR")
  export SNAKEMAKE_LOG_PATH="${WORKDIR}/snakemake.log"
  MANIFEST_SUPPLIED="true"
  # if manifest is empty ... aka not supplied at cli
  if [[ -z $MANIFEST ]];then
    MANIFEST_SUPPLIED="false"
    MANIFEST=${PIPELINE_HOME}/config/samples.tsv
  fi
  log_info "Working Dir:      $WORKDIR"
  log_info "Samples Manifest: $MANIFEST"

  # required files
  CONFIGFILE="${WORKDIR}/config.yaml"
  CLUSTERFILE="${WORKDIR}/cluster.json"

  CLUSTERSTATUSCMD="${PIPELINE_HOME}/resources/cluster_status.sh"

  if [[ -z "$SING_CACHE_DIR" ]]; then
    if [[ -n "$SIFCACHE" && -d "$SIFCACHE" ]]; then
      SING_CACHE_DIR="$SIFCACHE"
      log_info "Using SIFCACHE directory for singularity cache: ${SING_CACHE_DIR}"
    else
      if [[ -d "/data/$USER" ]]; then
        SING_CACHE_DIR="/data/$USER/.singularity"
      else
        SING_CACHE_DIR="${WORKDIR}/.singularity"
      fi
      log_info "singularity cache dir (--singcache) is not set, using ${SING_CACHE_DIR}"
      mkdir -p "$SING_CACHE_DIR"
    fi
  fi
  EXPORT_SING_CACHE_DIR_CMD="export SINGULARITY_CACHEDIR=\"${SING_CACHE_DIR}\""
  case $RUNMODE in
    init)     init      && exit 0;;
    dag)      dag       && exit 0;;
    dryrun)   dryrun    && exit 0;;
    unlock)   unlock    && exit 0;;
    run)      runslurm  && exit 0;;
    runlocal) runlocal  && exit 0;;
    reset)    reset     && exit 0;;
    dry)      dryrun    && exit 0;;                      # hidden option
    local)    runlocal  && exit 0;;                  # hidden option
    reconfig) reconfig  && exit 0;;               # hidden option for debugging
    printbinds) printbinds && exit 0;;           # hidden option
    *) err "Unknown RUNMODE \"$RUNMODE\"";;
  esac
}

# call the main function

main "$@"