Generate Genomeqc samplesheet

CHANGELOG.md

@@ -19,6 +19,10 @@ In addition, v2.0.0 contains these changes:
 
 Pull requests in reverse chronological order since v1.1.0
 
+[#215](https://github.com/nf-core/genomeassembler/pull/215)
+
+- Prepare samplesheet for downstream QC with [nf-core/genomeqc](https://nf-co.re/genomeqc)
+
 [#212](https://github.com/nf-core/genomeassembler/pull/212)
 
 - bumped medaka to 2.2.2
@@ -33,7 +37,7 @@ Pull requests in reverse chronological order since v1.1.0
 
 - Switched to nf-schema for input parsing
 
-- Added test with HiC dataset.
+- Added test with HiC dataset (kindly prepared by @OlivierCoen).
 
 [#198](https://github.com/nf-core/genomeassembler/pull/198)
 

conf/modules/assembly.config

@@ -26,7 +26,8 @@ process {
     withName: '.*BGZIP_FLYE.*' {
         publishDir = [
             path: { "${params.outdir}/${meta.id}/assembly/flye/" },
-            mode: params.publish_dir_mode
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.tokenize("/")[1]}
         ]
     }
     withName: HIFIASM {
@@ -49,35 +50,38 @@ process {
     }
     withName: GFA_2_FA {
         publishDir = [
-            path: { "${params.outdir}/${meta.id}/assembly/hifiasm/fasta" },
+            path: { "${params.outdir}/${meta.id}/assembly/hifiasm/" },
             mode: params.publish_dir_mode,
             saveAs: null
         ]
     }
     withName: GFA_2_FA_HIFI {
         publishDir = [
-            path: { "${params.outdir}/${meta.id}/assembly/hifiasm/fasta" },
+            path: { "${params.outdir}/${meta.id}/assembly/hifiasm/" },
             mode: params.publish_dir_mode,
             saveAs: null
         ]
     }
     withName: BGZIP_HIFI {
         publishDir = [
-            path: { "${params.outdir}/${meta.id}/assembly/hifiasm/fasta" },
-            mode: params.publish_dir_mode
+            path: { "${params.outdir}/${meta.id}/assembly/hifiasm/" },
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.tokenize("/")[1]}
         ]
     }
     withName: BGZIP_ONT {
         publishDir = [
-            path: { "${params.outdir}/${meta.id}/assembly/hifiasm_ont/fasta" },
-            mode: params.publish_dir_mode
+            path: { "${params.outdir}/${meta.id}/assembly/hifiasm_ont/" },
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.tokenize("/")[1]}
         ]
     }
 
-    withName: '.*ASSEMBLE:.*BZIP_RAGTAG.*' {
+    withName: '.*ASSEMBLE:.*BGZIP_RAGTAG.*' {
         publishDir = [
             path: { "${params.outdir}/${meta.id}/assembly/ragtag/" },
-            mode: params.publish_dir_mode
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.tokenize("/")[1]}
         ]
     }
 

conf/modules/polishing.config

@@ -13,7 +13,8 @@ process {
     withName: '.*MEDAKA:BGZIP.*' {
         publishDir = [
             path: { "${params.outdir}/${meta.id}/polish/medaka" },
-            mode: params.publish_dir_mode
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.tokenize("/")[1]}
         ]
     }
     withName: '.*MEDAKA:UNZIP.*' {
@@ -62,7 +63,8 @@ process {
     withName: '.*:POLISH_PILON:BGZIP:.*' {
         publishDir = [
             path: { "${params.outdir}/${meta.id}/polish/pilon" },
-            mode: params.publish_dir_mode
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.tokenize("/")[1]}
         ]
     }
 }

conf/modules/scaffolding.config

@@ -15,7 +15,8 @@ process {
     withName: '.*:RUN_RAGTAG:BGZIP.*' {
         publishDir = [
             path: { "${params.outdir}/${meta.id}/scaffold/ragtag/" },
-            mode: params.publish_dir_mode
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.tokenize("/")[1]}
         ]
     }
     withName: LINKS {
@@ -29,7 +30,8 @@ process {
     withName: '.*:RUN_LINKS:BGZIP.*' {
         publishDir = [
             path: { "${params.outdir}/${meta.id}/scaffold/links/" },
-            mode: params.publish_dir_mode
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.tokenize("/")[1]}
         ]
     }
     withName: LONGSTITCH {
@@ -42,7 +44,8 @@ process {
     withName: '.*:RUN_LONGSTITCH:BGZIP.*' {
         publishDir = [
             path: { "${params.outdir}/${meta.id}/scaffold/longstitch/" },
-            mode: params.publish_dir_mode
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.tokenize("/")[1]}
         ]
     }
     withName: BWAMEM2_INDEX {
@@ -94,7 +97,8 @@ process {
     withName: '.*:HIC:BGZIP.*' {
         publishDir = [
             path: { "${params.outdir}/${meta.id}/scaffold/hic/yahs/" },
-            mode: params.publish_dir_mode
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.tokenize("/")[1]}
         ]
     }
     withName: '.*HIC:SAMTOOLS_FAIDX.*' {

docs/usage.md

@@ -20,6 +20,9 @@ This pipeline can perform assembly, polishing, scaffolding using long-reads, HiC
 Since it is often difficult to know which tool, or assembly strategy will perform best on a dataset, `nf-core/genomeassembler` can also be used to compare outcomes of different approaches in one run.
 To compare different samples, a column named `group` is required, which should contain the same value for all samples that should be compared to each other.
 
+`nf-core/genomeassembler` performs genome qc and summarizes the results in a report.
+For more in-depth qc, `nf-core/genomeassembler` generates a samplesheet that can be used as inputs for [`nf-core/genomeqc`](https://nf-co.re/genomeqc) which can be found in `<resultsdir>/genomeqc_samplesheet/nf-core-genomeqc-in.csv`.
+
 ## Parameterization
 
 Parameters for this pipeline can either be supplied **globally**, e.g:

nf-test.config

@@ -1,22 +1,18 @@
 config {
     // location for all nf-test tests
     testsDir = "."
-    testsDir = "."
 
     // nf-test directory including temporary files for each test
     workDir = System.getenv("NFT_WORKDIR") ?: ".nf-test"
-    workDir = System.getenv("NFT_WORKDIR") ?: ".nf-test"
 
     // location of an optional nextflow.config file specific for executing tests
     configFile = "tests/nextflow.config"
-    configFile = "tests/nextflow.config"
 
     // ignore tests coming from the nf-core/modules repo
     ignore = ['modules/nf-core/**/tests/*', 'subworkflows/nf-core/**/tests/*']
 
     // run all test with defined profile(s) from the main nextflow.config
     profile = "test"
-    profile = "test"
 
     // list of filenames or patterns that should be trigger a full test run
     triggers = [
@@ -35,5 +31,6 @@ config {
     // load the necessary plugins
     plugins {
         load "nft-utils@0.0.3"
+        load "nft-csv@0.1.0"
     }
 }

subworkflows/local/scaffold/main.nf

@@ -60,93 +60,94 @@ workflow SCAFFOLD {
     ragtag_out = RUN_RAGTAG.out.ch_main
 
     // Deal with cases that are single scaffold
+    // !!! The individual sswfs return map-channels, this means that it -> it.meta needs to be used, otherwise there is a nesting
     ch_main = links_out
         .filter {it -> !it.meta.scaffold_longstitch && !it.meta.scaffold_ragtag }
-        .map { meta -> [ meta: meta - meta.subMap("links_scaffold") + [ scaffolds: [ links: meta.scaffolds_links ] ]  ]}
+        .map { it -> [meta: it.meta - it.meta.subMap("links_scaffold") + [ scaffolds: [ links: it.meta.scaffolds_links ] ] ]  }
         .mix(
             longstitch_out
                 .filter {it -> !it.meta.scaffold_links && !it.meta.scaffold_ragtag }
-                .map { meta -> [ meta: meta - meta.subMap("scaffolds_longstitch") + [ scaffolds: [ longstitch: meta.scaffolds_longstitch ] ]  ]}
+                .map { it -> [meta: it.meta - it.meta.subMap("scaffolds_longstitch") + [ scaffolds: [ longstitch: it.meta.scaffolds_longstitch ] ]]  }
         )
         .mix(
             ragtag_out
                 .filter {it -> !it.meta.scaffold_links && !it.meta.scaffold_longstitch }
-                .map { meta -> [ meta: meta - meta.subMap("scaffolds_ragtag") + [ scaffolds: [ ragtag: meta.scaffolds_ragtag ] ]  ]}
+                .map { it -> [meta: it.meta - it.meta.subMap("scaffolds_ragtag") + [ scaffolds: [ ragtag: it.meta.scaffolds_ragtag ] ] ]  }
         )
         .mix(
             hic_out
-                .map { meta -> [ meta: meta - meta.subMap("scaffolds_hic") + [ scaffolds: [ hic: meta.scaffolds_hic ] ]  ]}
+                .map { it -> [meta: it.meta - it.meta.subMap("scaffolds_hic") + [ scaffolds: [ hic: it.meta.scaffolds_hic ] ] ]}
 
             )
         // mix in those that are double scaffolded: , links-ragtag, longstitch-ragtag
         // links-longstitch
         .mix(
             links_out
                 .filter {it -> it.meta.scaffold_longstitch && !it.meta.scaffold_ragtag }
-                .map {meta -> [meta.id, meta]}
+                .map {it -> [it.meta.id, it.meta]}
                 // Join without filtering, inner-join
                 .join(
                     longstitch_out
-                        .map {meta -> [meta.id, meta]}
+                        .map {it -> [it.meta.id, it.meta]}
                 )
                 .map {
-                    _id, meta_links, meta_longstitch -> [
-                        meta: meta_links -
+                    _id, meta_links, meta_longstitch ->
+                        meta_links -
                          meta_links.subMap("scaffolds_links") +
-                         [scaffolds: [links: meta_links.scaffolds_links, longstitch: meta_longstitch.scaffolds_longstitch]] ]
+                         [scaffolds: [links: meta_links.scaffolds_links, longstitch: meta_longstitch.scaffolds_longstitch]]
                 }
         )
         //links-ragtag
         .mix(
             links_out
                 .filter {it -> !it.meta.scaffold_longstitch && it.meta.scaffold_ragtag }
-                .map {meta -> [meta.id, meta]}
+                .map {it -> [it.meta.id, it.meta]}
                 // Join without filtering, inner-join
                 .join(
                     ragtag_out
-                        .map {meta -> [meta.id, meta]}
+                        .map {it -> [it.meta.id, it.meta]}
                 )
                 .map {
-                    _id, meta_links, meta_ragtag -> [
-                        meta: meta_links -
+                    _id, meta_links, meta_ragtag ->
+                        meta_links -
                          meta_links.subMap("scaffolds_links") +
-                         [scaffolds: [links: meta_links.scaffolds_links, ragtag: meta_ragtag.scaffolds_ragtag]] ]
+                         [ scaffolds: [ links: meta_links.scaffolds_links, ragtag: meta_ragtag.scaffolds_ragtag ] ]
                 }
         )
         //longstitch-ragtag
         .mix(
             longstitch_out
                 .filter {it -> !it.meta.scaffold_links && it.meta.scaffold_ragtag }
-                .map {meta -> [meta.id, meta]}
+                .map {it -> [it.meta.id, it.meta]}
                 // Join without filtering, inner-join
                 .join(
                     ragtag_out
-                        .map {meta -> [meta.id, meta]}
+                        .map {it -> [it.meta.id, it.meta]}
                 )
                 .map {
-                    _id, meta_longstitch, meta_ragtag -> [
-                        meta: meta_longstitch -
+                    _id, meta_longstitch, meta_ragtag ->
+                        meta_longstitch -
                          meta_longstitch.subMap("scaffolds_longstitch") +
-                         [scaffolds: [longstitch: meta_longstitch.scaffolds_longstitch, ragtag: meta_ragtag.scaffolds_ragtag]] ]
+                         [scaffolds: [ longstitch: meta_longstitch.scaffolds_longstitch, ragtag: meta_ragtag.scaffolds_ragtag ] ]
                 }
         )
         // mix in triple-scaffolded
         .mix(
             links_out
                 .filter {it -> it.meta.scaffold_longstitch && it.meta.scaffold_ragtag }
-                .map {meta -> [meta.id, meta]}
+                .map {it -> [it.meta.id, it.meta]}
                 // Join without filtering, inner-join
                 .join(
                     longstitch_out
-                        .map {meta -> [meta.id, meta]}
+                        .map {it -> [it.meta.id, it.meta]}
                 )
                 .join(
                     ragtag_out
-                        .map {meta -> [meta.id, meta]}
+                        .map {it -> [it.meta.id, it.meta]}
                 )
                 .map {
-                    _id, meta_links, meta_longstitch, meta_ragtag -> [
-                        meta: meta_links -
+                    _id, meta_links, meta_longstitch, meta_ragtag ->
+                         meta_links -
                          meta_links.subMap("scaffolds_links") +
                             [
                                 scaffolds: [
@@ -155,11 +156,9 @@ workflow SCAFFOLD {
                                 ragtag: meta_ragtag.scaffolds_ragtag
                                 ]
                             ]
-                             ]
                 }
         )
 
-
     links_busco = RUN_LINKS.out.busco_out
     links_quast = RUN_LINKS.out.quast_out
     links_merqury = RUN_LINKS.out.merqury_report_files

subworkflows/local/scaffold/ragtag/main.nf

@@ -67,7 +67,7 @@ workflow RUN_RAGTAG {
         meryl_kmers)
 
     emit:
-    ch_main
+    ch_main                 = ch_main_scaffolded
     quast_out               = QC.out.quast_out
     busco_out               = QC.out.busco_out
     merqury_report_files    = QC.out.merqury_report_files

subworkflows/local/utils_nfcore_genomeassembler_pipeline/main.nf

@@ -132,14 +132,16 @@ workflow PIPELINE_INITIALISATION {
             def merqury         =   it.merqury && !it.shortread_F ? false : it.merqury
             def group           =   it.group ?: null
             def use_short_reads =   it.shortread_F && !params.use_short_reads ? true : it.use_short_reads
+            def lift_annotations=   it.use_ref && it.ref_gff ? true : false
             it + [
                     group: group,
                     assembler_ont: assembler_ont,
                     assembler_hifi: assembler_hifi,
                     polish: polish,
                     merqury: merqury,
                     use_short_reads: use_short_reads,
-                    paired: it.shortread_F && it.shortread_R ? true : false
+                    paired: it.shortread_F && it.shortread_R ? true : false,
+                    lift_annotations: lift_annotations
                 ]
 
         }

tests/.nftignore

@@ -17,3 +17,4 @@ test_flye_hifiasm/scaffold/ragtag/test_flye_hifiasm_ragtag.stats
 **/assembly.cut250.tigmint.*
 **/*.assembly.fa
 **/*MarkDuplicates.metrics.txt
+**/nf-core-genomeqc-in.csv

tests/default.nf.test

@@ -4,6 +4,7 @@ nextflow_pipeline {
     script "../main.nf"
     tag "pipeline"
 
+
     test("-profile test") {
 
         when {
@@ -18,6 +19,9 @@ nextflow_pipeline {
             // stable_content: All files in ${params.outdir}/ with stable content
             def stable_content = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
             assert workflow.success
+            def genomeqc_infile = path("$outputDir/genomeqc_samplesheet/nf-core-genomeqc-in.csv").csv
+            assert genomeqc_infile.rowCount == 15
+            assert genomeqc_infile.columnCount == 2
             assertAll(
                 { assert snapshot(
                     // Number of successful tasks
@@ -46,6 +50,9 @@ nextflow_pipeline {
             // stable_content: All files in ${params.outdir}/ with stable content
             def stable_content = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
             assert workflow.success
+            def genomeqc_infile = path("$outputDir/genomeqc_samplesheet/nf-core-genomeqc-in.csv").csv()
+            assert genomeqc_infile.rowCount == 15
+            assert genomeqc_infile.columnCount == 2
             assertAll(
                 { assert snapshot(
                     // Number of successful tasks
@@ -74,6 +81,9 @@ nextflow_pipeline {
             // stable_content: All files in ${params.outdir}/ with stable content
             def stable_content = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
             assert workflow.success
+            def genomeqc_infile = path("$outputDir/genomeqc_samplesheet/nf-core-genomeqc-in.csv").csv()
+            assert genomeqc_infile.rowCount == 15
+            assert genomeqc_infile.columnCount == 2
             assertAll(
                 { assert snapshot(
                     // Number of successful tasks
@@ -101,6 +111,9 @@ nextflow_pipeline {
             def stable_path = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}'])
             // stable_content: All files in ${params.outdir}/ with stable content
             def stable_content = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
+            def genomeqc_infile = path("$outputDir/genomeqc_samplesheet/nf-core-genomeqc-in.csv").csv()
+            assert genomeqc_infile.rowCount == 15
+            assert genomeqc_infile.columnCount == 2
             assert workflow.success
             assertAll(
                 { assert snapshot(