fix: multiqc config, tweak res

README.md

@@ -44,32 +44,39 @@ export GITHUB_TOKEN=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 
 #### Step 2: Test workflow
 
-5. Test your configuration by performing a dry-run via:
+5. Test your configuration:
+
+  5.1. Perform a dry-run:
 ```
 pixi run snakemake --snakefile <path to Snakfile> --configfile config/config.yaml --dry-run
 ```
 
-and confirm workflow analyses by checking the DAG:
+  5.2. Confirm workflow analyses by checking the DAG:
 ```
 pixi run snakemake --snakefile <path to Snakfile> --configfile config/config.yaml --dag | dot -Tsvg > dag.svg
 ```
 
-You can also create all needed `conda` environments beforehand:
+  5.3. Optionaly, create all needed `conda` environments beforehand:
 ```
 pixi run snakemake --snakefile <path to Snakfile> --configfile config/config.yaml --jobs 1 --conda-create-envs-only
 ```
 
 #### Step 3: Execute workflow
 6. Execute the workflow, using `$N` cores, either:
 
-  6.1. locally via:
+  6.1. locally:
 ```
 pixi run snakemake --snakefile <path to Snakfile> --configfile config/config.yaml --jobs $N --software-deployment-method conda
 ```
 
-  6.2. in a [SLURM](https://slurm.schedmd.com/overview.html) cluster environment via:
+  6.2. in a [SLURM](https://slurm.schedmd.com/overview.html) cluster environment:
 ```
 pixi run snakemake --snakefile <path to Snakfile> --configfile config/config.yaml --jobs $N --software-deployment-method conda --executor slurm
+```
+
+  6.3. Optionally, you can also use binaries from local modules, by adding `env-modules` to `--software-deployment-method`:
+```
+pixi run snakemake [...] --software-deployment-method env-modules conda [...]
 ```
 
 #### Step 4: Check results

pixi.toml

@@ -7,15 +7,18 @@ version = "1.1.0"
 [tasks]
 
 [dependencies]
-snakemake = ">=9.0.1,<10"
+snakemake = ">=9.0.1,<9.11.3"
 conda = ">=25.1.1,<26"
 python = "<=3.12.8"
+sqlalchemy = ">=2.0.43,<3"
+sqlalchemy-utils = ">=0.42.0,<1"
+psycopg2 = ">=2.9.10,<3"
 snakemake-executor-plugin-slurm = ">=1.1.0,<2"
 snakemake-storage-plugin-http = ">=0.3.0,<0.4"
 snakemake-storage-plugin-ftp = ">=0.1.3,<0.2"
 
 [environments]
-test = { features = ["test"] }
+test = { features = ["test"]}
 format = { features = ["format"], no-default-feature = true }
 
 [feature.test.dependencies]

scripts/launch_runs.py

@@ -72,53 +72,51 @@
 
 
 # Infer Pixi env and workflow paths
-if args.workflow.startswith("prod"):
+if args.workflow == "prod":
     pixi_env = workflow_path = "/projects/caeg/apps/aeDNA"
     extra_args += " --profile /projects/caeg/data/resources/profile_production"
-    if args.workflow == "prod-legacy":
-        workflow_path = "/projects/caeg/apps/aeDNA-legacy"
-    elif args.workflow == "prod-test":
-        pixi_env = workflow_path = "/projects/caeg/people/lnc113/workflows/aeDNA/aeDNA"
+elif args.workflow == "prod-legacy":
+    pixi_env = "/projects/caeg/apps/aeDNA"
+    workflow_path = "/projects/caeg/apps/aeDNA-legacy"
+elif args.workflow == "prod-test":
+    pixi_env = workflow_path = "/projects/caeg/people/lnc113/workflows/aeDNA/aeDNA"
 elif args.workflow == "caterpillar":
     pixi_env = workflow_path = "/projects/caeg/people/lnc113/workflows/caterpillar"
 
 
 # Infer hostname, HPC account and partition
-cmd = f"env --chdir={{id}} bash -c "
-if args.run == "local":
-    logging.info(f"Running jobs locally")
-    opts_hpc = ""
+import socket
+hostname = socket.gethostname()
+if hostname.startswith("dandy"):
+    hpc_snakemake_account = hpc_job_account = "prod"
+    hpc_job_partition = "compregular"
+    hpc_snakemake_partition = "compsnake"
+elif hostname.startswith("rubus"):
+    hpc_snakemake_account = hpc_job_account = "bench"
+    hpc_snakemake_partition = "epyc_long,xeon_fat_long"
+    hpc_job_partition = "epyc,epyc_noht,xeon_fat"
+else:
+    logging.error(f"Host {hostname} not supported yet!")
+    exit(-1)
+
+
+# Run Workflow
+if args.submit_snakemake:
+    cmd = f"sbatch --chdir {{id}} --job-name {{id}} --account {hpc_snakemake_account} --partition {hpc_snakemake_partition} --cpus-per-task 1 --mem 1G --time 5-00 --no-requeue --wrap="
+    logging.info(f"Workflows will be submitted to the {args.run} HPC, on account '{hpc_snakemake_account}' and partition '{hpc_snakemake_partition}'.")
 else:
-    import socket
-
-    opts_hpc = f"--executor {args.run}"
-    hostname = socket.gethostname()
-    if hostname.startswith("dandy"):
-        hpc_snakemake_account = hpc_job_account = "prod"
-        hpc_job_partition = "compregular"
-        hpc_snakemake_partition = "compsnake"
-    elif hostname.startswith("rubus"):
-        hpc_snakemake_account = hpc_job_account = "bench"
-        hpc_snakemake_partition = "epyc_long,xeon_fat_long"
-        hpc_job_partition = "epyc,epyc_noht,xeon_fat"
-    else:
-        logging.error(f"Host {hostname} not supported yet!")
-        exit(-1)
-
-
-    if args.submit_snakemake:
-        if args.run == "slurm":
-            cmd = f"sbatch --chdir {{id}} --job-name {{id}} --account {hpc_snakemake_account} --partition {hpc_snakemake_partition} --cpus-per-task 1 --mem 1G --time 5-00 --no-requeue --wrap="
-            opts_hpc += f" --default-resources slurm_account={hpc_job_account} slurm_partition={hpc_job_partition} tmpdir='{args.tmp_dir}' --slurm-status-attempts 1000"
-        else:
-            logging.error(f"HPC {args.run} not supported yet!")
-            exit(-1)
-
-        logging.info(f"Workflows will be submitted to the {args.run} HPC, on account '{hpc_snakemake_account}' and partition '{hpc_snakemake_partition}'.")
-    else:
-        logging.info(f"Workflows will be run locally on host {hostname}")
+    cmd = f"env --chdir={{id}} bash -c "
+    logging.info(f"Workflows will be run locally on host {hostname}")
+
 
+# Run jobs
+opts = "--jobs 300 --retries 1"
+if args.run == "local":
+    logging.info(f"Running jobs locally")
+    opts += f" --default-resources tmpdir='{args.tmp_dir}'"
+elif args.run == "slurm":
     logging.info(f"Jobs will be submitted to the {args.run} HPC, on account '{hpc_job_account}' and partition '{hpc_job_partition}'.")
+    opts += f" --executor {args.run} --default-resources slurm_account={hpc_job_account} slurm_partition={hpc_job_partition} tmpdir='{args.tmp_dir}'"
 
 
 # Read job list
@@ -140,6 +138,6 @@
 logging.info("Build command")
 
 for id in df.index:
-    print(f'{cmd.format(id=id)}"pixi run --manifest-path {pixi_env} snakemake --snakefile {workflow_path}/workflow/Snakefile --workflow-profile /projects/caeg/data/resources/profile {extra_args} {opts_hpc}"; sleep 0.5')
+    print(f'{cmd.format(id=id)}"pixi run --manifest-path {pixi_env} snakemake --snakefile {workflow_path}/workflow/Snakefile --workflow-profile /projects/caeg/data/resources/profile {extra_args} {opts}"; sleep 0.5')
 
 exit(0)

workflow/resources/multiqc_config.yaml

@@ -65,14 +65,11 @@ table_cond_formatting_colours:
 ## To remove a section of a module
 # https://multiqc.info/docs/#removing-modules-or-sections
 module_order:
-  - samtools
-
-  - picard
-
-  - biobambam2:
-      name: 'BamSorMaDup'
+  - samtools:
+      name: 'Samtools'
+      anchor: 'samtools'
       path_filters:
-        - '*.metrics.txt'
+        - '*/aligns/samtools_stats/*'
 
   - bowtie2:
       name: 'Bowtie2'
@@ -82,11 +79,18 @@ module_order:
 
   - fastqc:
       name: 'FastQC (prefilter)'
-      anchor: 'prefilter'
+      anchor: 'fastqc_prefilter'
       info: 'FastQC results after prefiltering.'
       path_filters:
         - '*/fastqc/prefilter/*'
 
+  - samtools:
+      name: 'Samtools (prefilter)'
+      anchor: 'samtools_prefilter'
+      info: 'Samtools Stats for prefiltering.'
+      path_filters:
+        - '*/prefilter_aligns/samtools_stats/*'
+
   - bowtie2:
       name: 'Bowtie2 (prefilter)'
       anchor: 'bowtie2_prefilter'

workflow/rules/prefilter.smk

@@ -21,8 +21,8 @@ rule prefilter_reads_taxonomy:
         )
     threads: 1
     resources:
-        mem=lambda w, input, attempt: f"{3* attempt} GiB",
-        runtime=lambda w, input, attempt: f"{(0.06* input.size_gb+2)* attempt} h",
+        mem=lambda w, input, attempt: f"{5* attempt} GiB",
+        runtime=lambda w, input, attempt: f"{(0.05* input.size_gb+2)* attempt} h",
     shell:
         "(extract_reads bytaxid -hts {input.bam} -names {input.names} -nodes {input.nodes} -acc2tax <(cat {input.acc2taxid}) {params.extra} -strict 1 -forcedump 1 -out - -type sam | awk '!/^@/' | cut -f 1 | uniq > {output.read_id}) 2> {log}"