From c0fed088eaa53ade4f84186dc8ff4a18322ad50e Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Mon, 3 Aug 2026 09:32:17 -0500 Subject: [PATCH] docs: Move process directives into separate pages Signed-off-by: Ben Sherman --- .docusaurus_site/sidebars.js | 59 +- docs/aws.mdx | 8 +- docs/cache-and-resume.mdx | 12 +- docs/conda.mdx | 2 +- docs/config.mdx | 2 +- docs/container.mdx | 2 +- docs/executor.mdx | 2 +- docs/executor/aws-batch.mdx | 22 +- docs/executor/azure-batch.mdx | 18 +- docs/executor/bridge.mdx | 10 +- docs/executor/flux.mdx | 8 +- docs/executor/google-batch.mdx | 18 +- docs/executor/htcondor.mdx | 10 +- docs/executor/hyperqueue.mdx | 10 +- docs/executor/kubernetes.mdx | 14 +- docs/executor/local.mdx | 12 +- docs/executor/lsf.mdx | 10 +- docs/executor/moab.mdx | 10 +- docs/executor/nqsii.mdx | 10 +- docs/executor/oar.mdx | 10 +- docs/executor/pbs.mdx | 10 +- docs/executor/pbspro.mdx | 10 +- docs/executor/seqera.mdx | 18 +- docs/executor/sge.mdx | 12 +- docs/executor/slurm.mdx | 10 +- docs/executor/tcs.mdx | 4 +- docs/google.mdx | 18 +- docs/kubernetes.mdx | 4 +- docs/migrations/24-04.mdx | 4 +- docs/migrations/26-04.mdx | 2 +- docs/notifications.mdx | 1 - docs/plugins/developing-plugins.mdx | 2 +- docs/process.mdx | 32 +- docs/reference/config/k8s.mdx | 2 +- docs/reference/config/workflow.mdx | 2 +- docs/reference/env-vars.mdx | 6 +- docs/reference/operator.mdx | 2 +- docs/reference/process.mdx | 1845 +---------------- docs/reference/process/directives.mdx | 113 + .../process/directives/accelerator.mdx | 50 + .../process/directives/after-script.mdx | 10 + docs/reference/process/directives/arch.mdx | 37 + docs/reference/process/directives/array.mdx | 71 + .../process/directives/before-script.mdx | 23 + docs/reference/process/directives/cache.mdx | 43 + .../process/directives/cluster-options.mdx | 44 + docs/reference/process/directives/conda.mdx | 27 + .../process/directives/container-options.mdx | 31 + .../process/directives/container.mdx | 28 + docs/reference/process/directives/cpus.mdx | 25 + docs/reference/process/directives/debug.mdx | 29 + docs/reference/process/directives/disk.mdx | 39 + .../process/directives/error-strategy.mdx | 66 + .../reference/process/directives/executor.mdx | 29 + docs/reference/process/directives/ext.mdx | 42 + docs/reference/process/directives/fair.mdx | 40 + docs/reference/process/directives/hints.mdx | 35 + docs/reference/process/directives/label.mdx | 32 + .../process/directives/machine-type.mdx | 26 + .../process/directives/max-errors.mdx | 26 + .../process/directives/max-forks.mdx | 23 + .../process/directives/max-retries.mdx | 26 + .../process/directives/max-submit-await.mdx | 26 + docs/reference/process/directives/memory.mdx | 34 + docs/reference/process/directives/module.mdx | 32 + docs/reference/process/directives/penv.mdx | 25 + docs/reference/process/directives/pod.mdx | 222 ++ .../process/directives/publish-dir.mdx | 117 ++ docs/reference/process/directives/queue.mdx | 32 + .../process/directives/resource-labels.mdx | 57 + .../process/directives/resource-limits.mdx | 46 + docs/reference/process/directives/scratch.mdx | 62 + docs/reference/process/directives/secret.mdx | 32 + docs/reference/process/directives/shell.mdx | 25 + docs/reference/process/directives/spack.mdx | 27 + .../process/directives/stage-in-mode.mdx | 26 + .../process/directives/stage-out-mode.mdx | 36 + .../process/directives/store-dir.mdx | 40 + docs/reference/process/directives/tag.mdx | 50 + docs/reference/process/directives/time.mdx | 39 + .../process/inputs-outputs-typed.mdx | 88 + docs/reference/process/inputs-outputs.mdx | 165 ++ docs/reference/process/task-properties.mdx | 73 + docs/reference/semantics.mdx | 2 +- docs/reference/stdlib-namespaces/workflow.mdx | 2 +- docs/reference/syntax.mdx | 2 +- docs/secrets.mdx | 2 +- docs/spack.mdx | 4 +- docs/tutorials/static-types.mdx | 2 +- docs/tutorials/workflow-outputs.mdx | 2 +- docs/wave.mdx | 2 +- docs/workflow.mdx | 2 +- 92 files changed, 2414 insertions(+), 2008 deletions(-) create mode 100644 docs/reference/process/directives.mdx create mode 100644 docs/reference/process/directives/accelerator.mdx create mode 100644 docs/reference/process/directives/after-script.mdx create mode 100644 docs/reference/process/directives/arch.mdx create mode 100644 docs/reference/process/directives/array.mdx create mode 100644 docs/reference/process/directives/before-script.mdx create mode 100644 docs/reference/process/directives/cache.mdx create mode 100644 docs/reference/process/directives/cluster-options.mdx create mode 100644 docs/reference/process/directives/conda.mdx create mode 100644 docs/reference/process/directives/container-options.mdx create mode 100644 docs/reference/process/directives/container.mdx create mode 100644 docs/reference/process/directives/cpus.mdx create mode 100644 docs/reference/process/directives/debug.mdx create mode 100644 docs/reference/process/directives/disk.mdx create mode 100644 docs/reference/process/directives/error-strategy.mdx create mode 100644 docs/reference/process/directives/executor.mdx create mode 100644 docs/reference/process/directives/ext.mdx create mode 100644 docs/reference/process/directives/fair.mdx create mode 100644 docs/reference/process/directives/hints.mdx create mode 100644 docs/reference/process/directives/label.mdx create mode 100644 docs/reference/process/directives/machine-type.mdx create mode 100644 docs/reference/process/directives/max-errors.mdx create mode 100644 docs/reference/process/directives/max-forks.mdx create mode 100644 docs/reference/process/directives/max-retries.mdx create mode 100644 docs/reference/process/directives/max-submit-await.mdx create mode 100644 docs/reference/process/directives/memory.mdx create mode 100644 docs/reference/process/directives/module.mdx create mode 100644 docs/reference/process/directives/penv.mdx create mode 100644 docs/reference/process/directives/pod.mdx create mode 100644 docs/reference/process/directives/publish-dir.mdx create mode 100644 docs/reference/process/directives/queue.mdx create mode 100644 docs/reference/process/directives/resource-labels.mdx create mode 100644 docs/reference/process/directives/resource-limits.mdx create mode 100644 docs/reference/process/directives/scratch.mdx create mode 100644 docs/reference/process/directives/secret.mdx create mode 100644 docs/reference/process/directives/shell.mdx create mode 100644 docs/reference/process/directives/spack.mdx create mode 100644 docs/reference/process/directives/stage-in-mode.mdx create mode 100644 docs/reference/process/directives/stage-out-mode.mdx create mode 100644 docs/reference/process/directives/store-dir.mdx create mode 100644 docs/reference/process/directives/tag.mdx create mode 100644 docs/reference/process/directives/time.mdx create mode 100644 docs/reference/process/inputs-outputs-typed.mdx create mode 100644 docs/reference/process/inputs-outputs.mdx create mode 100644 docs/reference/process/task-properties.mdx diff --git a/.docusaurus_site/sidebars.js b/.docusaurus_site/sidebars.js index 18a8d7b830..2caf2566be 100644 --- a/.docusaurus_site/sidebars.js +++ b/.docusaurus_site/sidebars.js @@ -191,7 +191,64 @@ module.exports = { "reference/stdlib-groovy" ] }, - "reference/process", + { + type: "category", + label: "Process", + link: { type: "doc", id: "reference/process" }, + items: [ + "reference/process/task-properties", + "reference/process/inputs-outputs-typed", + "reference/process/inputs-outputs", + { + type: "category", + label: "Directives", + link: { type: "doc", id: "reference/process/directives" }, + items: [ + "reference/process/directives/accelerator", + "reference/process/directives/after-script", + "reference/process/directives/arch", + "reference/process/directives/array", + "reference/process/directives/before-script", + "reference/process/directives/cache", + "reference/process/directives/cluster-options", + "reference/process/directives/conda", + "reference/process/directives/container", + "reference/process/directives/container-options", + "reference/process/directives/cpus", + "reference/process/directives/debug", + "reference/process/directives/disk", + "reference/process/directives/error-strategy", + "reference/process/directives/executor", + "reference/process/directives/ext", + "reference/process/directives/fair", + "reference/process/directives/hints", + "reference/process/directives/label", + "reference/process/directives/machine-type", + "reference/process/directives/max-errors", + "reference/process/directives/max-forks", + "reference/process/directives/max-retries", + "reference/process/directives/max-submit-await", + "reference/process/directives/memory", + "reference/process/directives/module", + "reference/process/directives/penv", + "reference/process/directives/pod", + "reference/process/directives/publish-dir", + "reference/process/directives/queue", + "reference/process/directives/resource-labels", + "reference/process/directives/resource-limits", + "reference/process/directives/scratch", + "reference/process/directives/secret", + "reference/process/directives/shell", + "reference/process/directives/spack", + "reference/process/directives/stage-in-mode", + "reference/process/directives/stage-out-mode", + "reference/process/directives/store-dir", + "reference/process/directives/tag", + "reference/process/directives/time" + ] + } + ] + }, "reference/operator" ] }, diff --git a/docs/aws.mdx b/docs/aws.mdx index 4355c69583..9f8e9cd50d 100644 --- a/docs/aws.mdx +++ b/docs/aws.mdx @@ -538,8 +538,8 @@ See [aws][config-aws] to learn more about advanced configuration options. [config-aws]: ./reference/config/aws [config-process-selectors]: ./config#process-selectors [fusion-page]: ./fusion -[process-container]: ./reference/process#container -[process-containerOptions]: ./reference/process#containeroptions -[process-label]: ./reference/process#label -[process-queue]: ./reference/process#queue +[process-container]: ./reference/process/directives/container +[process-containerOptions]: ./reference/process/directives/container-options +[process-label]: ./reference/process/directives/label +[process-queue]: ./reference/process/directives/queue [wave-page]: ./wave diff --git a/docs/cache-and-resume.mdx b/docs/cache-and-resume.mdx index 09a96b7229..779e158585 100644 --- a/docs/cache-and-resume.mdx +++ b/docs/cache-and-resume.mdx @@ -241,16 +241,16 @@ Nextflow now has a built-in way to compare two task runs. See the [Data lineage] [cli-clean]: ./reference/cli/clean [cli-log]: ./reference/cli/log [operator-join]: ./reference/operator#join -[process-arch]: ./reference/process#arch -[process-cache]: ./reference/process#cache -[process-conda]: ./reference/process#conda -[process-ext]: ./reference/process#ext +[process-arch]: ./reference/process/directives/arch +[process-cache]: ./reference/process/directives/cache +[process-conda]: ./reference/process/directives/conda +[process-ext]: ./reference/process/directives/ext [process-input]: ./process#inputs [operator-merge]: ./reference/operator#merge -[process-module]: ./reference/process#module +[process-module]: ./reference/process/directives/module [process-out-eval]: ./process#eval-output-eval [process-script]: ./process#script -[process-spack]: ./reference/process#spack +[process-spack]: ./reference/process/directives/spack [process-stub]: ./process#stub [stdlib-namespaces-workflow]: ./reference/stdlib-namespaces/workflow [strict-syntax-page]: ./strict-syntax diff --git a/docs/conda.mdx b/docs/conda.mdx index c9977b45af..ac6f81f245 100644 --- a/docs/conda.mdx +++ b/docs/conda.mdx @@ -221,4 +221,4 @@ Conda advanced configuration settings are described in the [conda][config-conda] [config-conda]: ./reference/config/conda [config-profiles]: ./config#config-profiles -[process-conda]: ./reference/process#conda +[process-conda]: ./reference/process/directives/conda diff --git a/docs/config.mdx b/docs/config.mdx index 3bfa1cb15a..e138028c31 100644 --- a/docs/config.mdx +++ b/docs/config.mdx @@ -347,7 +347,7 @@ This approach is useful for handling workflow events without modifying the pipel [config-options]: ./reference/config [nxf-env-vars]: ./reference/env-vars#nextflow-settings [process-reference]: ./reference/process -[process-label]: ./reference/process#label +[process-label]: ./reference/process/directives/label [secrets-page]: ./secrets [strict-syntax-page]: ./strict-syntax [syntax-expressions]: ./reference/syntax#expressions diff --git a/docs/container.mdx b/docs/container.mdx index 5f300795f2..c9adb26d7a 100644 --- a/docs/container.mdx +++ b/docs/container.mdx @@ -91,4 +91,4 @@ Nextflow manages the file system mounts for a container to provide task inputs. [container-singularity]: ./container/singularity [container-smolvm]: ./container/smolvm [execution-report-tasks]: ./reports#tasks -[process-container]: ./reference/process#container +[process-container]: ./reference/process/directives/container diff --git a/docs/executor.mdx b/docs/executor.mdx index f8da34b748..4e4ab570f6 100644 --- a/docs/executor.mdx +++ b/docs/executor.mdx @@ -64,4 +64,4 @@ Use [`process.executor`][process-executor] to set the executor, e.g. `process.ex [executor-sge]: ./executor/sge [executor-slurm]: ./executor/slurm [executor-tcs]: ./executor/tcs -[process-executor]: ./reference/process#executor +[process-executor]: ./reference/process/directives/executor diff --git a/docs/executor/aws-batch.mdx b/docs/executor/aws-batch.mdx index 676b5f81cd..4dd296a600 100644 --- a/docs/executor/aws-batch.mdx +++ b/docs/executor/aws-batch.mdx @@ -33,14 +33,14 @@ The following [hints][process-hints] are supported: See [AWS Batch][aws-batch] for more information. [aws-batch]: ../aws#aws-batch -[process-accelerator]: ../reference/process#accelerator -[process-arch]: ../reference/process#arch -[process-container]: ../reference/process#container -[process-containeroptions]: ../reference/process#containeroptions -[process-cpus]: ../reference/process#cpus -[process-disk]: ../reference/process#disk -[process-hints]: ../reference/process#hints -[process-memory]: ../reference/process#memory -[process-queue]: ../reference/process#queue -[process-resourcelabels]: ../reference/process#resourcelabels -[process-time]: ../reference/process#time +[process-accelerator]: ../reference/process/directives/accelerator +[process-arch]: ../reference/process/directives/arch +[process-container]: ../reference/process/directives/container +[process-containeroptions]: ../reference/process/directives/container-options +[process-cpus]: ../reference/process/directives/cpus +[process-disk]: ../reference/process/directives/disk +[process-hints]: ../reference/process/directives/hints +[process-memory]: ../reference/process/directives/memory +[process-queue]: ../reference/process/directives/queue +[process-resourcelabels]: ../reference/process/directives/resource-labels +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/azure-batch.mdx b/docs/executor/azure-batch.mdx index fbebe5f538..8a2c3e3955 100644 --- a/docs/executor/azure-batch.mdx +++ b/docs/executor/azure-batch.mdx @@ -24,12 +24,12 @@ Use the following process directives to control resource requests and other job See [Azure Batch][azure-batch] for more information. [azure-batch]: ../azure#azure-batch -[process-container]: ../reference/process#container -[process-containeroptions]: ../reference/process#containeroptions -[process-cpus]: ../reference/process#cpus -[process-disk]: ../reference/process#disk -[process-machinetype]: ../reference/process#machinetype -[process-memory]: ../reference/process#memory -[process-queue]: ../reference/process#queue -[process-resourcelabels]: ../reference/process#resourcelabels -[process-time]: ../reference/process#time +[process-container]: ../reference/process/directives/container +[process-containeroptions]: ../reference/process/directives/container-options +[process-cpus]: ../reference/process/directives/cpus +[process-disk]: ../reference/process/directives/disk +[process-machinetype]: ../reference/process/directives/machine-type +[process-memory]: ../reference/process/directives/memory +[process-queue]: ../reference/process/directives/queue +[process-resourcelabels]: ../reference/process/directives/resource-labels +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/bridge.mdx b/docs/executor/bridge.mdx index 16ee888d66..6d0470044e 100644 --- a/docs/executor/bridge.mdx +++ b/docs/executor/bridge.mdx @@ -19,8 +19,8 @@ Use the following process directives to control resource requests and other job - [queue][process-queue] - [time][process-time] -[process-clusteroptions]: ../reference/process#clusteroptions -[process-cpus]: ../reference/process#cpus -[process-memory]: ../reference/process#memory -[process-queue]: ../reference/process#queue -[process-time]: ../reference/process#time +[process-clusteroptions]: ../reference/process/directives/cluster-options +[process-cpus]: ../reference/process/directives/cpus +[process-memory]: ../reference/process/directives/memory +[process-queue]: ../reference/process/directives/queue +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/flux.mdx b/docs/executor/flux.mdx index 1fa740cad7..9a6568cfdd 100644 --- a/docs/executor/flux.mdx +++ b/docs/executor/flux.mdx @@ -22,7 +22,7 @@ By default, Flux sends all output to the `.command.log` file. To send this outpu Flux does not support the `memory` directive. ::: -[process-clusteroptions]: ../reference/process#clusteroptions -[process-cpus]: ../reference/process#cpus -[process-queue]: ../reference/process#queue -[process-time]: ../reference/process#time +[process-clusteroptions]: ../reference/process/directives/cluster-options +[process-cpus]: ../reference/process/directives/cpus +[process-queue]: ../reference/process/directives/queue +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/google-batch.mdx b/docs/executor/google-batch.mdx index b5e0b18658..6b74876fd5 100644 --- a/docs/executor/google-batch.mdx +++ b/docs/executor/google-batch.mdx @@ -24,12 +24,12 @@ Use the following process directives to control resource requests and other job See [Cloud Batch][google-batch] for further configuration details. [google-batch]: ../google#cloud-batch -[process-accelerator]: ../reference/process#accelerator -[process-container]: ../reference/process#container -[process-containeroptions]: ../reference/process#containeroptions -[process-cpus]: ../reference/process#cpus -[process-disk]: ../reference/process#disk -[process-machinetype]: ../reference/process#machinetype -[process-memory]: ../reference/process#memory -[process-resourcelabels]: ../reference/process#resourcelabels -[process-time]: ../reference/process#time +[process-accelerator]: ../reference/process/directives/accelerator +[process-container]: ../reference/process/directives/container +[process-containeroptions]: ../reference/process/directives/container-options +[process-cpus]: ../reference/process/directives/cpus +[process-disk]: ../reference/process/directives/disk +[process-machinetype]: ../reference/process/directives/machine-type +[process-memory]: ../reference/process/directives/memory +[process-resourcelabels]: ../reference/process/directives/resource-labels +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/htcondor.mdx b/docs/executor/htcondor.mdx index 03c1348a97..5b146b26ad 100644 --- a/docs/executor/htcondor.mdx +++ b/docs/executor/htcondor.mdx @@ -24,8 +24,8 @@ Use the following process directives to control resource requests and other job - [memory][process-memory] - [time][process-time] -[process-clusteroptions]: ../reference/process#clusteroptions -[process-cpus]: ../reference/process#cpus -[process-disk]: ../reference/process#disk -[process-memory]: ../reference/process#memory -[process-time]: ../reference/process#time +[process-clusteroptions]: ../reference/process/directives/cluster-options +[process-cpus]: ../reference/process/directives/cpus +[process-disk]: ../reference/process/directives/disk +[process-memory]: ../reference/process/directives/memory +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/hyperqueue.mdx b/docs/executor/hyperqueue.mdx index db7824ae6e..0268952821 100644 --- a/docs/executor/hyperqueue.mdx +++ b/docs/executor/hyperqueue.mdx @@ -25,8 +25,8 @@ Use the following process directives to control resource requests and other job - [memory][process-memory] - [time][process-time] -[process-accelerator]: ../reference/process#accelerator -[process-clusteroptions]: ../reference/process#clusteroptions -[process-cpus]: ../reference/process#cpus -[process-memory]: ../reference/process#memory -[process-time]: ../reference/process#time +[process-accelerator]: ../reference/process/directives/accelerator +[process-clusteroptions]: ../reference/process/directives/cluster-options +[process-cpus]: ../reference/process/directives/cpus +[process-memory]: ../reference/process/directives/memory +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/kubernetes.mdx b/docs/executor/kubernetes.mdx index c0ae0a78aa..4e8ad073a0 100644 --- a/docs/executor/kubernetes.mdx +++ b/docs/executor/kubernetes.mdx @@ -21,10 +21,10 @@ See [Kubernetes][k8s-page] for how to set up a Kubernetes cluster to run Nextflo [config-k8s]: ../reference/config/k8s [k8s-page]: ../kubernetes -[process-accelerator]: ../reference/process#accelerator -[process-cpus]: ../reference/process#cpus -[process-disk]: ../reference/process#disk -[process-memory]: ../reference/process#memory -[process-pod]: ../reference/process#pod -[process-resourcelabels]: ../reference/process#resourcelabels -[process-time]: ../reference/process#time +[process-accelerator]: ../reference/process/directives/accelerator +[process-cpus]: ../reference/process/directives/cpus +[process-disk]: ../reference/process/directives/disk +[process-memory]: ../reference/process/directives/memory +[process-pod]: ../reference/process/directives/pod +[process-resourcelabels]: ../reference/process/directives/resource-labels +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/local.mdx b/docs/executor/local.mdx index a8bdec937b..ae314bc847 100644 --- a/docs/executor/local.mdx +++ b/docs/executor/local.mdx @@ -49,9 +49,9 @@ To run tasks in containers, propagate the device environment variable into the c [container-docker]: ../container/docker [container-podman]: ../container/podman -[process-accelerator]: ../reference/process#accelerator -[process-container]: ../reference/process#container -[process-containeroptions]: ../reference/process#containeroptions -[process-cpus]: ../reference/process#cpus -[process-memory]: ../reference/process#memory -[process-time]: ../reference/process#time +[process-accelerator]: ../reference/process/directives/accelerator +[process-container]: ../reference/process/directives/container +[process-containeroptions]: ../reference/process/directives/container-options +[process-cpus]: ../reference/process/directives/cpus +[process-memory]: ../reference/process/directives/memory +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/lsf.mdx b/docs/executor/lsf.mdx index 9e41eed80c..57dc633377 100644 --- a/docs/executor/lsf.mdx +++ b/docs/executor/lsf.mdx @@ -19,8 +19,8 @@ Use the following process directives to control resource requests and other job LSF supports both *per-core* and *per-job* memory limits. Nextflow assumes the *per-core* mode and divides the requested [memory][process-memory] by the number of requested [cpus][process-cpus]. When LSF runs in *per-job* mode, set the `executor.perJobMemLimit` config option. See the [Platform LSF documentation](https://www.ibm.com/support/knowledgecenter/SSETD4_9.1.3/lsf_config_ref/lsf.conf.lsb_job_memlimit.5.dita) for more information. -[process-clusteroptions]: ../reference/process#clusteroptions -[process-cpus]: ../reference/process#cpus -[process-memory]: ../reference/process#memory -[process-queue]: ../reference/process#queue -[process-time]: ../reference/process#time +[process-clusteroptions]: ../reference/process/directives/cluster-options +[process-cpus]: ../reference/process/directives/cpus +[process-memory]: ../reference/process/directives/memory +[process-queue]: ../reference/process/directives/queue +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/moab.mdx b/docs/executor/moab.mdx index 7954a80ada..743af5d066 100644 --- a/docs/executor/moab.mdx +++ b/docs/executor/moab.mdx @@ -20,8 +20,8 @@ Use the following process directives to control resource requests and other job - [queue][process-queue] - [time][process-time] -[process-clusteroptions]: ../reference/process#clusteroptions -[process-cpus]: ../reference/process#cpus -[process-memory]: ../reference/process#memory -[process-queue]: ../reference/process#queue -[process-time]: ../reference/process#time +[process-clusteroptions]: ../reference/process/directives/cluster-options +[process-cpus]: ../reference/process/directives/cpus +[process-memory]: ../reference/process/directives/memory +[process-queue]: ../reference/process/directives/queue +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/nqsii.mdx b/docs/executor/nqsii.mdx index 25eb0b3c67..48a18010b5 100644 --- a/docs/executor/nqsii.mdx +++ b/docs/executor/nqsii.mdx @@ -17,8 +17,8 @@ Use the following process directives to control resource requests and other job - [queue][process-queue] - [time][process-time] -[process-clusteroptions]: ../reference/process#clusteroptions -[process-cpus]: ../reference/process#cpus -[process-memory]: ../reference/process#memory -[process-queue]: ../reference/process#queue -[process-time]: ../reference/process#time +[process-clusteroptions]: ../reference/process/directives/cluster-options +[process-cpus]: ../reference/process/directives/cpus +[process-memory]: ../reference/process/directives/memory +[process-queue]: ../reference/process/directives/queue +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/oar.mdx b/docs/executor/oar.mdx index d11bb6fd53..d7e0020286 100644 --- a/docs/executor/oar.mdx +++ b/docs/executor/oar.mdx @@ -33,8 +33,8 @@ clusterOptions = [ '-t besteffort', '--project myproject' ] See [clusterOptions][process-clusteroptions] for details. -[process-clusteroptions]: ../reference/process#clusteroptions -[process-cpus]: ../reference/process#cpus -[process-memory]: ../reference/process#memory -[process-queue]: ../reference/process#queue -[process-time]: ../reference/process#time +[process-clusteroptions]: ../reference/process/directives/cluster-options +[process-cpus]: ../reference/process/directives/cpus +[process-memory]: ../reference/process/directives/memory +[process-queue]: ../reference/process/directives/queue +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/pbs.mdx b/docs/executor/pbs.mdx index 407571c4db..03de623295 100644 --- a/docs/executor/pbs.mdx +++ b/docs/executor/pbs.mdx @@ -17,8 +17,8 @@ Use the following process directives to control resource requests and other job - [queue][process-queue] - [time][process-time] -[process-clusteroptions]: ../reference/process#clusteroptions -[process-cpus]: ../reference/process#cpus -[process-memory]: ../reference/process#memory -[process-queue]: ../reference/process#queue -[process-time]: ../reference/process#time +[process-clusteroptions]: ../reference/process/directives/cluster-options +[process-cpus]: ../reference/process/directives/cpus +[process-memory]: ../reference/process/directives/memory +[process-queue]: ../reference/process/directives/queue +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/pbspro.mdx b/docs/executor/pbspro.mdx index bee2a6d1b8..b0f33cd4d1 100644 --- a/docs/executor/pbspro.mdx +++ b/docs/executor/pbspro.mdx @@ -17,8 +17,8 @@ Use the following process directives to control resource requests and other job - [queue][process-queue] - [time][process-time] -[process-clusteroptions]: ../reference/process#clusteroptions -[process-cpus]: ../reference/process#cpus -[process-memory]: ../reference/process#memory -[process-queue]: ../reference/process#queue -[process-time]: ../reference/process#time +[process-clusteroptions]: ../reference/process/directives/cluster-options +[process-cpus]: ../reference/process/directives/cpus +[process-memory]: ../reference/process/directives/memory +[process-queue]: ../reference/process/directives/queue +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/seqera.mdx b/docs/executor/seqera.mdx index 8fb279af2e..42a8e3bbaa 100644 --- a/docs/executor/seqera.mdx +++ b/docs/executor/seqera.mdx @@ -119,12 +119,12 @@ The EBS-specific options (`diskType`, `diskIops`, `diskThroughputMiBps`, `diskEn ::: [config-seqera]: ../reference/config/seqera -[process-arch]: ../reference/process#arch -[process-container]: ../reference/process#container -[process-containeroptions]: ../reference/process#containeroptions -[process-cpus]: ../reference/process#cpus -[process-disk]: ../reference/process#disk -[process-hints]: ../reference/process#hints -[process-memory]: ../reference/process#memory -[process-secret]: ../reference/process#secret -[process-time]: ../reference/process#time +[process-arch]: ../reference/process/directives/arch +[process-container]: ../reference/process/directives/container +[process-containeroptions]: ../reference/process/directives/container-options +[process-cpus]: ../reference/process/directives/cpus +[process-disk]: ../reference/process/directives/disk +[process-hints]: ../reference/process/directives/hints +[process-memory]: ../reference/process/directives/memory +[process-secret]: ../reference/process/directives/secret +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/sge.mdx b/docs/executor/sge.mdx index e32e140190..c92ec6701c 100644 --- a/docs/executor/sge.mdx +++ b/docs/executor/sge.mdx @@ -18,9 +18,9 @@ Use the following process directives to control resource requests and other job - [queue][process-queue] - [time][process-time] -[process-clusteroptions]: ../reference/process#clusteroptions -[process-cpus]: ../reference/process#cpus -[process-memory]: ../reference/process#memory -[process-penv]: ../reference/process#penv -[process-queue]: ../reference/process#queue -[process-time]: ../reference/process#time +[process-clusteroptions]: ../reference/process/directives/cluster-options +[process-cpus]: ../reference/process/directives/cpus +[process-memory]: ../reference/process/directives/memory +[process-penv]: ../reference/process/directives/penv +[process-queue]: ../reference/process/directives/queue +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/slurm.mdx b/docs/executor/slurm.mdx index 5918dc922f..4447e52d8e 100644 --- a/docs/executor/slurm.mdx +++ b/docs/executor/slurm.mdx @@ -31,8 +31,8 @@ Some SLURM clusters require memory allocations to be specified with `--mem-per-c Since SLURM 24, `squeue` supports an `--only-job-state` option that ignores the partition (`-p`) or user (`-u`) filters. To enable this behavior, set `executor.$slurm.onlyJobState = true` in your Nextflow configuration. Enabling `SchedulerParameters=enable_job_state_cache` improves Nextflow performance and reduces load on the SLURM controller. See [`enable_job_state_cache`](https://slurm.schedmd.com/slurm.conf.html#OPT_enable_job_state_cache) and [`--only-job-state`](https://slurm.schedmd.com/squeue.html#OPT_only-job-state) for more information. -[process-clusteroptions]: ../reference/process#clusteroptions -[process-cpus]: ../reference/process#cpus -[process-memory]: ../reference/process#memory -[process-queue]: ../reference/process#queue -[process-time]: ../reference/process#time +[process-clusteroptions]: ../reference/process/directives/cluster-options +[process-cpus]: ../reference/process/directives/cpus +[process-memory]: ../reference/process/directives/memory +[process-queue]: ../reference/process/directives/queue +[process-time]: ../reference/process/directives/time diff --git a/docs/executor/tcs.mdx b/docs/executor/tcs.mdx index 5c35a57c80..73596d215e 100644 --- a/docs/executor/tcs.mdx +++ b/docs/executor/tcs.mdx @@ -26,5 +26,5 @@ process { } ``` -[process-clusteroptions]: ../reference/process#clusteroptions -[process-time]: ../reference/process#time +[process-clusteroptions]: ../reference/process/directives/cluster-options +[process-time]: ../reference/process/directives/time diff --git a/docs/google.mdx b/docs/google.mdx index 2c5be639b2..1c385cc302 100644 --- a/docs/google.mdx +++ b/docs/google.mdx @@ -284,12 +284,12 @@ Nextflow will automatically manage the transfer of input and output files betwee [config-process-selectors]: ./config#process-selectors [fusion-page]: ./fusion [google-batch-executor]: ./executor/google-batch -[process-accelerator]: ./reference/process#accelerator -[process-container]: ./reference/process#container -[process-containeroptions]: ./reference/process#containeroptions -[process-cpus]: ./reference/process#cpus -[process-disk]: ./reference/process#disk -[process-executor]: ./reference/process#executor -[process-machinetype]: ./reference/process#machinetype -[process-memory]: ./reference/process#memory -[process-time]: ./reference/process#time +[process-accelerator]: ./reference/process/directives/accelerator +[process-container]: ./reference/process/directives/container +[process-containeroptions]: ./reference/process/directives/container-options +[process-cpus]: ./reference/process/directives/cpus +[process-disk]: ./reference/process/directives/disk +[process-executor]: ./reference/process/directives/executor +[process-machinetype]: ./reference/process/directives/machine-type +[process-memory]: ./reference/process/directives/memory +[process-time]: ./reference/process/directives/time diff --git a/docs/kubernetes.mdx b/docs/kubernetes.mdx index 3c06a71e88..137ea19bde 100644 --- a/docs/kubernetes.mdx +++ b/docs/kubernetes.mdx @@ -133,6 +133,6 @@ See [k8s][config-k8s] and [Kubernetes][k8s-executor] to learn more about advance [config-k8s]: ./reference/config/k8s [fusion-page]: ./fusion [k8s-executor]: ./executor/kubernetes -[process-container]: ./reference/process#container -[process-pod]: ./reference/process#pod +[process-container]: ./reference/process/directives/container +[process-pod]: ./reference/process/directives/pod [wave-page]: ./wave \ No newline at end of file diff --git a/docs/migrations/24-04.mdx b/docs/migrations/24-04.mdx index 57eae48e5b..d247fc59cd 100644 --- a/docs/migrations/24-04.mdx +++ b/docs/migrations/24-04.mdx @@ -144,8 +144,8 @@ The `nf-ga4gh` plugin has since been moved into its own repository, [nextflow-io [config-workflow]: ../reference/config/workflow [dynamic-task-resources]: ../process#dynamic-task-resources [google-batch-process]: ../google#process-definition -[process-array]: ../reference/process#array +[process-array]: ../reference/process/directives/array [process-out-eval]: ../process#eval-output-eval -[process-resourcelimits]: ../reference/process#resourcelimits +[process-resourcelimits]: ../reference/process/directives/resource-limits [secrets-pipeline-script]: ../secrets#pipeline-script [workflow-output-def]: ../workflow#outputs diff --git a/docs/migrations/26-04.mdx b/docs/migrations/26-04.mdx index 15d5799c8e..be7599e81b 100644 --- a/docs/migrations/26-04.mdx +++ b/docs/migrations/26-04.mdx @@ -436,7 +436,7 @@ A Platform access token with appropriate permissions is required to download non [migrating-static-types-operators]: ../tutorials/static-types-operators [module-registry]: ../modules [operator-page]: ../reference/operator -[process-hints]: ../reference/process#hints +[process-hints]: ../reference/process/directives/hints [process-typed-page]: ../process-typed [script-records]: ../script#records [static-typing-first-preview]: ./25-10#static-typing-preview diff --git a/docs/notifications.mdx b/docs/notifications.mdx index fcb77b73f2..385bfe9b6f 100644 --- a/docs/notifications.mdx +++ b/docs/notifications.mdx @@ -218,5 +218,4 @@ See [Completion handler](#completion-handler) to learn more about the workflow n [config-mail]: ./reference/config/mail [config-notification]: ./reference/config/notification -[process-error-strategy]: ./reference/process#errorstrategy [stdlib-namespaces-sendmail]: ./reference/stdlib-namespaces/global#sendmail-options- diff --git a/docs/plugins/developing-plugins.mdx b/docs/plugins/developing-plugins.mdx index a3e0c448d6..74955e3ffe 100644 --- a/docs/plugins/developing-plugins.mdx +++ b/docs/plugins/developing-plugins.mdx @@ -535,5 +535,5 @@ This variable is useful for testing a plugin release before publishing it to the [gradle-plugin-publish]: ../guides/gradle-plugin#publishing-a-plugin [gradle-plugin-test]: ../guides/gradle-plugin#testing-a-plugin [process-directives]: ../process#directives -[process-ext]: ../reference/process#ext +[process-ext]: ../reference/process/directives/ext [remote-files]: ../working-with-files#remote-files diff --git a/docs/process.mdx b/docs/process.mdx index c51f079aa0..f2e7ac437f 100644 --- a/docs/process.mdx +++ b/docs/process.mdx @@ -1327,27 +1327,27 @@ process hello { [operator-flatten]: ./reference/operator#flatten [operator-cross]: ./reference/operator#cross [operator-merge]: ./reference/operator#merge -[process-conda]: ./reference/process#conda -[process-container]: ./reference/process#container -[process-cpus]: ./reference/process#cpus -[process-error-strategy]: ./reference/process#errorstrategy -[process-executor]: ./reference/process#executor +[process-conda]: ./reference/process/directives/conda +[process-container]: ./reference/process/directives/container +[process-cpus]: ./reference/process/directives/cpus +[process-error-strategy]: ./reference/process/directives/error-strategy +[process-executor]: ./reference/process/directives/executor [process-input-path]: ./process#input-files-path -[process-label]: ./reference/process#label -[process-maxforks]: ./reference/process#maxforks -[process-maxretries]: ./reference/process#maxretries -[process-memory]: ./reference/process#memory +[process-label]: ./reference/process/directives/label +[process-maxforks]: ./reference/process/directives/max-forks +[process-maxretries]: ./reference/process/directives/max-retries +[process-memory]: ./reference/process/directives/memory [process-out-singleton]: ./process#singleton-outputs [process-out-tuple]: ./process#output-tuples-tuple -[process-queue]: ./reference/process#queue -[process-tag]: ./reference/process#tag -[process-time]: ./reference/process#time -[process-secret]: ./reference/process#secret +[process-queue]: ./reference/process/directives/queue +[process-tag]: ./reference/process/directives/tag +[process-time]: ./reference/process/directives/time +[process-secret]: ./reference/process/directives/secret [process-script]: ./process#script [process-multiple-inputs]: ./process#multiple-inputs -[process-reference-directives]: ./reference/process#directives -[process-reference-inputs]: ./reference/process#inputs -[process-reference-outputs]: ./reference/process#outputs +[process-reference-directives]: ./reference/process/directives +[process-reference-inputs]: ./reference/process/inputs-outputs#inputs +[process-reference-outputs]: ./reference/process/inputs-outputs#outputs [process-template]: ./process#template [process-typed-page]: ./process-typed [script-closure]: ./script#closures diff --git a/docs/reference/config/k8s.mdx b/docs/reference/config/k8s.mdx index 9be460fafd..a99b9599ed 100644 --- a/docs/reference/config/k8s.mdx +++ b/docs/reference/config/k8s.mdx @@ -143,4 +143,4 @@ Max attempts when retrying failed API requests (default: `4`). Max delay when retrying failed API requests (default: `90s`). -[process-pod]: ../process#pod +[process-pod]: ../process/directives/pod diff --git a/docs/reference/config/workflow.mdx b/docs/reference/config/workflow.mdx index 32f1d87402..0b2ec996e5 100644 --- a/docs/reference/config/workflow.mdx +++ b/docs/reference/config/workflow.mdx @@ -121,5 +121,5 @@ For example: workflow.output.tags = [FOO: 'hello', BAR: 'world'] ``` -[process-error-strategy]: ../process#errorstrategy +[process-error-strategy]: ../process/directives/error-strategy [workflow-handlers]: ../../notifications#workflow-handlers diff --git a/docs/reference/env-vars.mdx b/docs/reference/env-vars.mdx index a8cdc2d883..614742225e 100644 --- a/docs/reference/env-vars.mdx +++ b/docs/reference/env-vars.mdx @@ -399,7 +399,7 @@ Defines one or more host names that should not use the proxy server. Separate mu Forces the terminal width of ANSI-formatted log output. Overrides automatic terminal width detection and uses the specified width for line wrapping when set to a positive integer. Values that are not positive integers are ignored, in which case Nextflow falls back to `COLUMNS` and then to automatic detection. -[process-conda]: ./process#conda -[process-publishdir]: ./process#publishdir -[process-spack]: ./process#spack +[process-conda]: ./process/directives/conda +[process-publishdir]: ./process/directives/publish-dir +[process-spack]: ./process/directives/spack [strict-syntax-page]: ../strict-syntax diff --git a/docs/reference/operator.mdx b/docs/reference/operator.mdx index 28341fa4a5..6f92cbc93d 100644 --- a/docs/reference/operator.mdx +++ b/docs/reference/operator.mdx @@ -1524,7 +1524,7 @@ Associate the channel with a tag. For example: [operator-empty]: ./stdlib-namespaces/channel#empty [operator-typed]: ./stdlib-types/channel [preparing-static-types]: ../tutorials/static-types#preparing-for-static-typing -[process-cache]: ./process#cache +[process-cache]: ./process/directives/cache [script-closure]: ../script#closures [script-regexp]: ../script#regular-expressions [static-types-operators]: ../tutorials/static-types-operators diff --git a/docs/reference/process.mdx b/docs/reference/process.mdx index bf9ed3a95a..5a8fb24c41 100644 --- a/docs/reference/process.mdx +++ b/docs/reference/process.mdx @@ -5,1836 +5,17 @@ description: Reference for all task properties, input and output qualifiers, and # Process reference -This page lists the task properties, input/output methods, and directives available in [process][process-page] definitions. - -## Task properties - -The following task properties are defined in the process body: - -##### `task.attempt` - -The current task attempt. - -##### `task.exitStatus` - -*Available only in `script:` and `shell:` blocks* - -The exit code returned by the task script. - -The exit code is only available after the task has been executed (e.g., the [errorStrategy](#errorstrategy) directive). - -##### `task.hash` - -*Available only in `exec:` blocks* - -The task hash. - -##### `task.index` - -The process-level task index. - -##### `task.name` - -*Available only in `exec:` blocks* - -The task name. - -##### `task.previousException` - - - -The exception reported by the previous task attempt. - -Since the exception is available after a failed task attempt, it can only be accessed when retrying a failed task execution, i.e., when `task.attempt` is greater than 1. - -##### `task.previousTrace` - - - -The trace record associated with the previous task attempt. - -Since the trace record is available after a failed task attempt, it can only be accessed when retrying a failed task execution, i.e., when `task.attempt` is greater than 1. See [Trace file][trace-report] for a list of available fields. - -:::note -The trace fields `%cpu` and `%mem` can be accessed as `pcpu` and `pmem`, respectively. -::: - -##### `task.process` - -The name of the process that spawned the task. - -##### `task.workDir` - -*Available only in `exec:` blocks* - -The unique directory path for the task. - -:::note -[Directive values](#directives) for a task can be accessed via `task.`. See [Task directive values][task-directive-values] for more information. -::: - -## Inputs and outputs (typed) - - - -:::note -Typed processes require the `nextflow.enable.types` feature flag to be enabled in every script that uses them. -::: - -### Stage directives - -The following directives can be used in the `stage:` section of a typed process: - -##### `env( name: String, String value )` - -Declares an environment variable with the specified name and value in the task environment. - -##### `stageAs( value: Path, filePattern: String )` - -Stages a file into the task directory under the given alias. - -##### `stageAs( value: Iterable, filePattern: String )` - -Stages a collection of files into the task directory under the given alias. - -##### `stdin( value: String )` - -Stages the given value as the standard input (i.e., `stdin`) to the task script. - -### Outputs - -The following functions are available in the `output:` and `topic:` sections of a typed process: - -##### `env( name: String ) -> String` - -Returns the value of an environment variable from the task environment. - -##### `eval( command: String ) -> String` - -Returns the standard output of the specified command, which is executed in the task environment after the task script completes. - -##### `file( pattern: String, [options] ) -> Path` - -Returns a file from the task environment that matches the specified pattern. - -Available options: - -###### `followLinks: Boolean` - -When `true`, target files are returned in place of any matching symlink (default: `true`). - -###### `glob: Boolean` - -When `true`, the file name is interpreted as a glob pattern (default: `true`). - -###### `hidden: Boolean` - -When `true`, hidden files are included in the matching output files (default: `false`). - -###### `includeInputs: Boolean` - -When `true` and the file name is a glob pattern, any input files matching the pattern are also included in the output (default: `false`). - -###### `maxDepth: Integer` - -Maximum number of directory levels to visit (default: no limit). - -###### `optional: Boolean` - -When `true`, the task will not fail if the given file is missing (default: `false`). - -###### `type: String` - -Type of paths returned, either `file`, `dir` or `any` (default: `any`, or `file` if the given file name contains a double star (`**`)). - -##### `files( pattern: String, [options] ) -> Set` - -Returns files from the task environment that match the given pattern. - -Supports the same options as `file()` (except for `optional`). - -##### `stdout() -> String` - -Returns the standard output of the task script. - -## Inputs and outputs (legacy) - -### Inputs - -##### `val( identifier )` - -Declare a variable input. The received value can be any type, and it will be made available to the process body (i.e. `script`, `shell`, `exec`) as a variable given by `identifier`. - -##### `file( identifier | stageName )` - - -Use `path` instead. - - -Declare a file input. The received value can be any type, and it will be staged into the task directory. If the received value is not a file or collection of files, it is implicitly converted to a string and written to a file. - -The argument can be an identifier or string. If an identifier, the received value will be made available to the process body as a variable. If a string, the received value will be staged into the task directory under the given alias. - -##### `path( identifier | stageName )` - -Declare a file input. The received value should be a file or collection of files and will be staged into the task directory. - -:::tip -See [Multiple input files][process-multiple-input-files] for more information about accepting collections of files. -::: - -The argument can be an identifier or string. If an identifier, the received value will be made available to the process body as a variable. If a string, the received value will be staged into the task directory under the given alias. - -Available options: - -###### `arity` - - - -Specify the number of expected files. Can be a number, e.g. `'1'`, or a range, e.g. `'1..*'`. If a task receives an invalid number of files for this `path` input, it will fail. - -###### `name` - -Specify how the file should be named in the task work directory. Can be a name or a pattern. - -###### `stageAs` - -Alias of `name`. - -##### `env( name )` - -Declare an environment variable input. The received value should be a string, and it will be exported to the task environment as an environment variable given by `name`. - -##### `stdin` - -Declare a `stdin` input. The received value should be a string, and it will be provided as the standard input (i.e. `stdin`) to the task script. It should be declared only once for a process. - -##### `tuple( arg1, arg2, ... )` - -Declare a tuple input. Each argument should be an input declaration such as `val`, `path`, `env`, or `stdin`. - -The received value should be a tuple with the same number of elements as the `tuple` declaration, and each received element should be compatible with the corresponding `tuple` argument. Each tuple element is treated the same way as if it were a standalone input. - -### Outputs - -##### `val( value )` - -Declare a variable output. The argument can be any value, and it can reference any output variables defined in the process body (i.e. variables declared without the `def` keyword). - -##### `file( pattern )` - - -Use `path` instead. - - -Declare a file output. It receives the output files from the task environment that match the given pattern. - -Multiple patterns can be specified using the colon separator (`:`). The union of all files matched by each pattern will be collected. - -##### `path( pattern, [options] )` - -Declare a file output. It receives the output files from the task environment that match the given pattern. - -Available options: - -###### `arity` - - - -Specify the number of expected files. Can be a number or a range. If a task produces an invalid number of files for this `path` output, it will fail. - -If the arity is `1`, a single file will be emitted. Otherwise, a list will always be emitted, even if only one file is produced. - -:::warning -If the arity is not specified, a single file or list will be emitted based on whether a single file or multiple files are produced at runtime, resulting potentially in an output channel with a mixture of files and file collections. -::: - -###### `followLinks` - -When `true`, target files are returned in place of any matching symlink (default: `true`). - -###### `glob` - -When `true`, the specified name is interpreted as a glob pattern (default: `true`). - -###### `hidden` - -When `true`, hidden files are included in the matching output files (default: `false`). - -###### `includeInputs` - -When `true` and the output path is a glob pattern, any input files matching the pattern are also included in the output (default: `false`). - -###### `maxDepth` - -Maximum number of directory levels to visit (default: no limit). - -###### `type` - -Type of paths returned, either `file`, `dir` or `any` (default: `any`, or `file` if the specified file name pattern contains a double star (`**`)). - -##### `env( name )` - -Declare an environment variable output. It receives the value of the environment variable (given by `name`) from the task environment. - - -Prior to this version, if the environment variable contained multiple lines of output, the output would be compressed to a single line by converting newlines to spaces. - - -##### `stdout` - -Declare a `stdout` output. It receives the standard output of the task script. - -##### `eval( command )` - - - -Declare an `eval` output. It receives the standard output of the given command, which is executed in the task environment after the task script. - -If the command fails, the task will also fail. - -##### `tuple( arg1, arg2, ... )` - -Declare a tuple output. Each argument should be an output declaration such as `val`, `path`, `env`, `stdin`, or `eval`. Each tuple element is treated the same way as if it were a standalone output. - -### Generic options - -The following options are available for all process outputs: - -##### `emit: ` - -Defines the name of the output channel. - -##### `optional: true | false` - -When `true`, the task will not fail if the specified output is missing (default: `false`). - -##### `topic: ` - - - -Send the output to a [topic channel][channel-topic] with the given name. - -## Directives - -### `accelerator` - -The `accelerator` directive defines the number of hardware accelerators (e.g. GPUs) required by each task execution. For example: - -```nextflow -process hello { - accelerator 4, type: 'nvidia-tesla-k80' - - script: - """ - your_gpu_enabled --command --line - """ -} -``` - -The above example requests 4 GPUs of type `nvidia-tesla-k80` for each task. - -:::note -This directive is only used by certain executors. Refer to the [Executors][executor-page] page to see which executors support this directive. -::: - -:::note -Additional options may be required to fully enable the use of accelerators. When using containers with GPUs, you must pass the GPU drivers through to the container. For Docker, this requires the option `--gpus all` in the `docker run` command. For Apptainer/Singularity, this requires the option `--nv`. The specific implementation details depend on the accelerator and container type being used. -::: - -The following options are available: - -##### `request: Integer` - -The number of requested accelerators. - -Specifying this directive with a number (e.g., `accelerator 4`) is equivalent to the `request` option (e.g., `accelerator request: 4`). - -##### `type: String` - -The accelerator type. - -The meaning of this option depends on the target execution platform. See the platform-specific documentation for more information about the available accelerators: - - - [Google Cloud](https://cloud.google.com/compute/docs/gpus/) - - [Kubernetes](https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/#clusters-containing-different-types-of-gpus) - -This option is not supported for AWS Batch. You can control the accelerator type indirectly through the allowed instance types in your Compute Environment. See the [AWS Batch FAQs](https://aws.amazon.com/batch/faqs/?#GPU_Scheduling_) for more information. - -### `afterScript` - -The `afterScript` directive executes a custom (Bash) snippet immediately *after* the main process has run. This may be useful to clean up your staging area. - -When combined with the [container](#container) directive, the `afterScript` is executed outside the specified container. In other words, the `afterScript` is always executed in the host environment. - -### `arch` - -The `arch` directive defines the CPU architecture to build the software in use by the process' task. For example: - -```nextflow -process blast { - spack 'blast-plus@2.13.0' - arch 'linux/x86_64', target: 'cascadelake' - - script: - """ - blastp -query input_sequence -num_threads ${task.cpus} - """ -} -``` - -The example above declares that the CPU generic architecture is `linux/x86_64` (X86 64 bit), and more specifically that the microarchitecture is `cascadelake` (a specific generation of Intel CPUs). - -This directive is currently used by the following Nextflow functionalities: - -- by the [spack](#spack) directive, to build microarchitecture-optimized applications; -- by the [Wave][wave-page] service, to build containers for one of the generic families of CPU architectures (see below); -- by the `spack` strategy within [Wave][wave-page], to optimize the container builds for specific CPU microarchitectures. - -Allowed values for the `arch` directive are as follows, grouped by equivalent family (choices available for the sake of compatibility): -- X86 64 bit: `linux/x86_64`, `x86_64`, `linux/amd64`, `amd64` -- ARM 64 bit: `linux/aarch64`, `aarch64`, `linux/arm64`, `arm64`, `linux/arm64/v8` -- ARM 64 bit, older generation: `linux/arm64/v7` - -Examples of values for the architecture `target` option are `cascadelake`, `icelake`, `zen2` and `zen3`. See the [Spack documentation](https://spack.readthedocs.io/en/latest/basic_usage.html#support-for-specific-microarchitectures) for the full and up-to-date list of meaningful targets. - -### `array` - - - -The `array` directive submits tasks as *job arrays* for executors that support it. - -A job array is a collection of jobs with the same resource requirements and the same script (parameterized by an index). Job arrays incur significantly less scheduling overhead compared to individual jobs, and as a result they are preferred by HPC schedulers where possible. - -The directive should be specified with a given array size, along with an executor that supports job arrays. For example: - -```nextflow -process hello { - executor 'slurm' - array 100 - - script: - """ - your_command --here - """ -} -``` - -Nextflow currently supports job arrays for the following executors: - -- [AWS Batch][awsbatch-executor] -- [Google Cloud Batch][google-batch-executor] -- [LSF][lsf-executor] -- [PBS][pbs-executor] -- [PBS Pro][pbspro-executor] -- [SGE][sge-executor] -- [SLURM][slurm-executor] - -A process using job arrays collects tasks and submits each batch as a job array when it is ready. Any "leftover" tasks are submitted as a partial job array. - -Once a job array is submitted, each "child" task is executed as an independent job. Any tasks that fail (and can be retried) are retried without interfering with the tasks that succeeded. Retried tasks are submitted individually rather than through a job array, in order to allow for the use of [dynamic resources][dynamic-task-resources]. - -The following directives must be uniform across all tasks in a process that uses job arrays, because these directives are specified once for the entire job array: - -- [accelerator](#accelerator) -- [clusterOptions](#clusteroptions) -- [cpus](#cpus) -- [disk](#disk) -- [machineType](#machinetype) -- [memory](#memory) -- [queue](#queue) -- [resourceLabels](#resourcelabels) -- [resourceLimits](#resourcelimits) -- [time](#time) - -For cloud-based executors like AWS Batch, or when using Fusion with any executor, the following additional directives must be uniform: - -- [container](#container) -- [containerOptions](#containeroptions) - -When using Wave, the following additional directives must be uniform: - -- [conda](#conda) - -### `beforeScript` - -The `beforeScript` directive executes a custom (Bash) snippet *before* the main process script is run. This may be useful to initialize the underlying cluster environment or for other custom initialization. - -For example: - -```nextflow -process hello { - beforeScript 'source /cluster/bin/setup' - - script: - """ - echo 'hello' - """ -} -``` - -When the process is containerized (using the [container](#container) directive), the `beforeScript` is executed in the container only if the executor is *container-native* (e.g. cloud batch executors, Kubernetes). Otherwise, the `beforeScript` is executed outside the container. - -### `cache` - -The `cache` directive controls whether and how task executions are cached. - -By default, cached task executions are re-used when the pipeline is launched with the [resume][getstarted-resume] option. The `cache` directive can be used to disable caching for a specific process: - -```nextflow -process hello { - cache false - - // ... -} -``` - -See [Cache and resume][cache-resume-page] for more information. - -The following options are available: - -##### `false` - -Disable caching. - -##### `true` (default) - -Enable caching. Input file metadata (name, size, last updated timestamp) are included in the cache keys. - -##### `'deep'` - -Enable caching. Input file content is included in the cache keys. - -##### `'lenient'` - -Enable caching. Minimal input file metadata (name and size only) are included in the cache keys. - -This strategy provides a workaround for incorrect caching invalidation observed on shared file systems due to inconsistent file timestamps. - -### `clusterOptions` - -The `clusterOptions` directive specifies additional submission options for grid executors. You can use it to specify options for your cluster that are not supported directly by other process directives. - -The cluster options can be a string: - -```nextflow -process hello { - clusterOptions '-x 1 -y 2' - - // ... -} -``` - - -Prior to this version, grid executors that require each option to be on a separate line in the job script would attempt to split multiple options using a variety of different conventions. Multiple options can now be specified more clearly using a string list as shown below. - - -The cluster options can also be a string list: - -```nextflow -process hello { - clusterOptions '-x 1', '-y 2', '--flag' - - // ... -} -``` - -Grid executors that require one option per line will write each option to a separate line, while grid executors that allow multiple options per line will write all options to a single line, the same as with a string. This form is useful to control how the options are split across lines when it is required by the scheduler. - -:::note -This directive is only used by grid executors. Refer to the [Executors][executor-page] page to see which executors support this directive. -::: - -:::warning -While you can use the `clusterOptions` directive to specify options that are supported as process directives (`queue`, `memory`, `time`, etc), you should not use both at the same time, as it will cause undefined behavior. Most HPC schedulers will either fail or simply ignore one or the other. -::: - -### `conda` - -The `conda` directive defines the set of [Conda](https://conda.io) packages required by each task. For example: - -```nextflow -process hello { - conda 'bwa=0.7.15' - - script: - """ - your_command --here - """ -} -``` - -Nextflow automatically creates an environment for each unique set of Conda packages. - -The name of the desired channel for a specific package can be specified using the standard Conda notation, e.g. `bioconda::bwa=0.7.15`. Multiple packages can be specified separating them with a blank space, e.g. `bwa=0.7.15 fastqc=0.11.5`. - -The `conda` directive can also accept a Conda environment file path or the path of an existing Conda environment. See [Conda][conda-page] for more information. - -### `container` - -The `container` directive defines the container required by each task. For example: - -```nextflow -process hello_docker { - container 'busybox:latest' - - script: - """ - your_command --here - """ -} -``` - -The corresponding container runtime (e.g. Docker, Singularity) should be running on the compute nodes where tasks are executed. See [Containers][container-page] for the container runtimes supported by Nextflow. - -:::note -This directive is ignored by [native processes][process-native] (i.e. `exec` processes). -::: - -### `containerOptions` - -The `containerOptions` directive specifies additional container options for the underlying container runtime (e.g. Docker, Singularity). For example: - -```nextflow -process hello_docker { - container 'busybox:latest' - containerOptions '--volume /data/db:/db' - - output: - path 'output.txt' - - script: - """ - your_command --data /db > output.txt - """ -} -``` - -The above example provides a custom volume mount for a specific process. - -:::warning -This directive is not supported by the [Kubernetes][k8s-executor] executor. -::: - -### `cpus` - -The `cpus` directive defines the number of CPUs required by each task execution. For example: - -```nextflow -process blast { - cpus 8 - - script: - """ - blastp -query input_sequence -num_threads ${task.cpus} - """ -} -``` - -This directive is required for tasks that execute multi-process or multi-threaded commands/tools and it is meant to reserve enough CPUs when a pipeline task is executed through a cluster resource manager. - -See also: [disk](#disk), [memory](#memory), [time](#time), [queue](#queue), [dynamic task resources][dynamic-task-resources] - -### `debug` - -The `debug` directive prints the standard output of each task to the pipeline standard output. - -For example: - -```nextflow -process hello { - debug true - - script: - """ - echo Hello - """ -} -``` - -Prints: - -``` -Hello -``` - -Removing the `debug` directive or setting it to `false` in the above example will cause `Hello` to not be printed. - -### `disk` - -The `disk` directive defines the amount of disk storage required by each task execution. For example: - -```nextflow -process hello { - disk 2.GB - - script: - """ - your_command --here - """ -} -``` - -The following suffixes can be used to specify disk values: - -- `B`: Bytes -- `KB`: Kilobytes -- `MB`: Megabytes -- `GB`: Gigabytes -- `TB`: Terabytes - -See [MemoryUnit][stdlib-types-memoryunit] for more information. - -:::note -The `disk` directive is only used by certain executors. Refer to the [Executors][executor-page] page to see which executors support this directive. -::: - -See also: [cpus](#cpus), [memory](#memory), [time](#time), [queue](#queue), [dynamic task resources][dynamic-task-resources] - -### `errorStrategy` - -The `errorStrategy` directive defines how to handle task failures. - -A task failure occurs when the executed script returns a non-zero exit code. By default, the pipeline run is aborted. - -The following error strategies are available: - -##### `'terminate'` (default) - -When a task fails, terminate the pipeline immediately and report an error. Pending and running jobs are killed. - -##### `'finish'` - -When a task fails, wait for submitted and running tasks to finish and then terminate the pipeline, reporting an error. - -##### `'ignore'` - -When a task fails, ignore it and continue the pipeline execution. If the `workflow.failOnIgnore` config option is set to `true`, the pipeline will report an error (i.e. return a non-zero exit code) upon completion. Otherwise, the pipeline will complete successfully. - -See the [workflow][stdlib-namespaces-workflow] namespace for more information. - -##### `'retry'` - -When a task fails, retry it. - -When setting the `errorStrategy` directive to `ignore` the process doesn't stop on an error condition, it just reports a message notifying you of the error event. - -For example: - -```nextflow -process hello { - errorStrategy 'ignore' - - // ... -} -``` - -In this case, the workflow will complete successfully and return an exit status of 0. However, if you set `workflow.failOnIgnore = true` in your Nextflow configuration, the workflow will return a non-zero exit status and report the failed tasks as an error. - -The `retry` error strategy retries failed tasks. For example: - -```nextflow -process hello { - errorStrategy 'retry' - - // ... -} -``` - -The number of times a failing process is re-executed is defined by the [maxRetries](#maxretries) and [maxErrors](#maxerrors) directives. - -:::tip -More complex strategies depending on the task exit status or other parametric values can be defined using a dynamic `errorStrategy`. See [Dynamic directives][dynamic-directives] for details. -::: - -See also: [maxErrors](#maxerrors), [maxRetries](#maxretries), [dynamic task resources][dynamic-task-resources] - -### `executor` - -The `executor` directive defines the underlying system where tasks are executed. For example: - -```nextflow -process hello { - executor 'slurm' - - // ... -} -``` - -Commonly used executors include: - -- `awsbatch`: [AWS Batch](https://aws.amazon.com/batch/) -- `azurebatch`: [Azure Batch](https://azure.microsoft.com/en-us/services/batch/) -- `google-batch`: [Google Cloud Batch](https://cloud.google.com/batch) -- `local`: local machine -- `lsf`: [Platform LSF](http://en.wikipedia.org/wiki/Platform_LSF) job scheduler -- `slurm`: [SLURM](https://en.wikipedia.org/wiki/Slurm_Workload_Manager) workload manager - -Each executor supports additional configuration options under the `executor` config scope. See [Executors][executor-page] for more information. - -### `ext` - -The `ext` is a generic directive for user-defined properties. For example: - -```nextflow -process star { - container "biocontainers/star:${task.ext.version}" - - input: - path genome - tuple val(sampleId), path(reads) - - script: - """ - STAR --genomeDir $genome --readFilesIn $reads ${task.ext.args ?: ''} - """ -} -``` - -In the above example, the process container version is controlled by `ext.version`, and the script supports additional command line arguments through `ext.args`. - -The `ext` directive can be set in the process definition: - -```nextflow -process hello { - ext version: '2.5.3', args: '--alpha --beta' - - // ... -} -``` - -Or in the Nextflow configuration: - -```groovy -process.ext.version = '2.5.3' -process.ext.args = '--alpha --beta' -``` - -### `fair` - - - -The `fair` directive, when enabled, guarantees that process outputs will be emitted in the order in which they were received. For example: - -```nextflow -process hello { - fair true - - input: - val x - - output: - tuple val(task.index), val(x) - - script: - """ - sleep \$((RANDOM % 3)) - """ -} - -workflow { - channel.of('A','B','C','D') | hello | view -} -``` - -The above example produces: - -``` -[1, A] -[2, B] -[3, C] -[4, D] -``` - -### `hints` - -The `hints` directive specifies executor-specific hints as key-value pairs. Each executor uses the hints it recognizes and ignores the rest. Hint values can be any raw value (i.e., numbers, strings, booleans, lists, and maps). - -Unprefixed keys are available to **every** executor. Any executor that recognizes the key consumes it. Prefixing a key with an executor name (e.g., `awsbatch/...`) restricts the hint to that executor only. For example: - -```nextflow -process hello { - hints consumableResources: ['my-license': 1] - - script: - """ - your_command --here - """ -} -``` - -To restrict a hint to a single executor, prefix the key with the executor name: - -```nextflow -hints 'awsbatch/consumableResources': ['my-license': 1] -``` - -When the same hint is provided both unprefixed and with a matching executor prefix, the prefixed form takes precedence for that executor. - -Calling `hints` multiple times in a process definition accumulates entries, with later calls overwriting entries for the same key. Setting `hints` via configuration (e.g., in `nextflow.config`) replaces the entire map. - -See [Executors][executor-page] to see which hints are recognized by each executor. - -### `label` - -The `label` directive attaches a custom label to the process. For example: - -```nextflow -process hello { - label 'big_mem' - - script: - """ - your_command --here - """ -} -``` - -A label may contain alphanumeric characters or `_`. It must start and end with an alphabetic character. - -The same label can be applied to multiple processes. Multiple labels can be applied to the same process by using the `label` directive multiple times. - -Process labels are used to apply shared process configuration via `withLabel` selectors. They are not recorded in execution logs, trace reports, or lineage metadata. See [Process selectors][config-process-selectors] for more information. - -:::note -To tag individual task executions for logging and debugging, use [tag](#tag). To tag cloud computing resources for cost tracking, use [resourceLabels](#resourcelabels). To attach metadata labels to output files for lineage tracking, use the `label` [output directive][workflow-output-def] in the `output` block. -::: - -### `machineType` - -The `machineType` can be used to specify a predefined Google Compute Platform [machine type](https://cloud.google.com/compute/docs/machine-types) when running using the [Google Batch][google-batch-executor], or when using auto-pools with [Azure Batch][azurebatch-executor]. - -For example: - -```nextflow -process hello { - machineType 'n1-highmem-8' - - script: - """ - your_command --here - """ -} -``` - -See also: [cpus](#cpus), [memory](#memory) - -### `maxErrors` - -The `maxErrors` directive defines the maximum number of task failures allowed for a process when using the `retry` error strategy. For example: - -```nextflow -process hello { - errorStrategy 'retry' - maxErrors 5 - - script: - """ - echo 'do this as that .. ' - """ -} -``` - -In the above example, the run will fail if the `hello` process accrues more than 5 failures across all of its task executions. - -By default, there is no limit. However, the run can still fail if an individual task exceeds the number of retries allowed by the `maxRetries` directive. - -See also: [errorStrategy](#errorstrategy), [maxRetries](#maxretries) - -### `maxForks` - -The `maxForks` directive defines the maximum number of concurrent task executions for a process. For example: - -```nextflow -process hello { - maxForks 1 - - script: - """ - your_command --here - """ -} -``` - -The above example forces the `hello` process to execute tasks sequentially. - -By default, there is no limit. However, the number of concurrent tasks can still be limited globally by the number of CPUs (for local tasks) and the `executor.queueSize` config option. - -### `maxRetries` - -The `maxRetries` directive defines the maximum number of times a task can be retried when using the `retry` error strategy. For example: - -```nextflow -process hello { - errorStrategy 'retry' - maxRetries 3 - - script: - """ - echo 'do this as that .. ' - """ -} -``` - -In the above example, the run will fail if any task executed by `hello` fails more than three times. - -By default, only one retry per task is allowed. However, the run can still fail if the total number of failures for the process exceeds the number allowed by the `maxErrors` directive. - -See also: [errorStrategy](#errorstrategy), [maxErrors](#maxerrors) - -### `maxSubmitAwait` - -The `maxSubmitAwait` directive defines how long a task can remain in submission queue without being executed. Tasks that exceed this duration in the queue will fail. - -It can be used with the `retry` error strategy to re-submit tasks to a different queue or with different resource requirements. For example: - -```nextflow -process hello { - errorStrategy 'retry' - maxSubmitAwait 10.m - maxRetries 3 - queue "${task.submitAttempt==1 ? 'spot-compute' : 'on-demand-compute'}" - - script: - """ - your_command --here - """ -} -``` - -In the above example, each task is submitted to the `spot-compute` queue on the first attempt (`task.submitAttempt==1`). If a task remains in the queue for more than 10 minutes, it fails and is re-submitted to the `on-demand-compute` queue. - -### `memory` - -The `memory` directive defines how much memory is required by each task execution. For example: - -```nextflow -process hello { - memory 2.GB - - script: - """ - your_command --here - """ -} -``` - -The following suffixes can be used to specify memory values: - -- `B`: Bytes -- `KB`: Kilobytes -- `MB`: Megabytes -- `GB`: Gigabytes -- `TB`: Terabytes - -See [MemoryUnit][stdlib-types-memoryunit] for more information. - -See also: [cpus](#cpus), [disk](#disk), [time](#time), [queue](#queue), [dynamic task resources][dynamic-task-resources] - -### `module` - -The `module` directive defines the set of [Environment Modules](http://modules.sourceforge.net/) required by each task, if supported by your compute environment. For example: - -```nextflow -process blast { - module 'ncbi-blast/2.2.27' - - script: - """ - blastp -query - """ -} -``` - -Multiple modules can be specified using the `:` separator: - -```nextflow -process blast { - module 'ncbi-blast/2.2.27:t_coffee/10.0:clustalw/2.1' - - script: - """ - blastp -query - """ -} -``` - -### `penv` - -The `penv` directive defines the parallel environment to use when submitting tasks to the [SGE][sge-executor] resource manager. For example: - -```nextflow -process blast { - cpus 4 - penv 'smp' - executor 'sge' - - script: - """ - blastp -query input_sequence -num_threads ${task.cpus} - """ -} -``` - -Refer to your cluster documentation or your system administrator to determine whether this feature is supported in your environment. - -### `pod` - -The `pod` directive defines pod-specific settings, such as environment variables, secrets, and config maps, when using the [Kubernetes][k8s-executor] executor. - -For example: - -```nextflow -process echo { - pod env: 'MESSAGE', value: 'hello world' - - script: - """ - echo $MESSAGE - """ -} -``` - -The above snippet defines an environment variable named `MESSAGE` whose value is `'hello world'`. - -Pod settings can be specified in Nextflow configuration: - -```groovy -// single setting -process.pod = [env: 'MESSAGE', value: 'hello world'] - -// multiple settings -process.pod = [ - [env: 'MESSAGE', value: 'hello world'], - [secret: 'my-secret/key1', mountPath: '/etc/file.txt'] -] -``` - -The following options are available: - -##### `affinity: ` - -Specifies the pod [affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) with the given configuration. - -##### `annotation: '', value: ''` - -*Can be specified multiple times* - -Defines a pod [annotation](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) with the given name and value. - -##### `automountServiceAccountToken: true | false` - -Specifies whether to [automount service account token](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#opt-out-of-api-credential-automounting) into the pod (default: `true`). - -##### `config: '/', mountPath: ''` - -*Can be specified multiple times* - -Mounts a [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) with name and optional key to the given path. If the key is omitted, the path is interpreted as a directory and all entries in the `ConfigMap` are exposed in that path. - -##### `csi: '', mountPath: ''` - -*Can be specified multiple times* - -Mounts a [CSI ephemeral volume](https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#csi-ephemeral-volumes) with the given configuration to the given path. - -##### `emptyDir: , mountPath: ''` - -*Can be specified multiple times* - -Mounts an [emptyDir](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir) with the given configuration to the given path. - -##### `env: '', config: '/'` - -*Can be specified multiple times* - -Defines an environment variable whose value is defined by the given [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) and key. - -##### `env: '', fieldPath: ''` - -*Can be specified multiple times* - -Defines an environment variable whose value is defined by the given [field path](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/#use-pod-fields-as-values-for-environment-variables) value. - -For example, the following pod option: - -```groovy -process.pod = [env: 'MY_NODE_NAME', fieldPath: 'spec.nodeName'] -``` - -Maps to the following pod spec: - -```yaml -env: - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName -``` - -##### `env: '', secret: '/'` - -*Can be specified multiple times* - -Defines an environment variable whose value is defined by the given [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) and key. - -##### `env: '', value: ''` - -*Can be specified multiple times* - -Defines an environment variable with the given name and value. - -##### `hostPath: '/host/absolute/path', mountPath: ''` - - - -*Can be specified multiple times* - -Allows creating [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) volume and access it with the specified `mountPath` in the pod. - -##### `imagePullPolicy: 'IfNotPresent' | 'Always' | 'Never'` - -Specifies the [image pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used by the pod to pull the container image. - -##### `imagePullSecret: ''` - -Specifies the [image pull secret](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) used to access a private container image registry. - -##### `label: '', value: ''` - -*Can be specified multiple times* - -Defines a pod [label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) with the given name and value. - -##### `nodeSelector: ` - -Specifies the [node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) with the given configuration. - -The configuration can be a map or a string: - -```groovy -// map -process.pod = [nodeSelector: [disktype: 'ssd', cpu: 'intel']] - -// string -process.pod = [nodeSelector: 'disktype=ssd,cpu=intel'] -``` - -##### `priorityClassName: ''` - -Specifies the [priority class name](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) for pods. - -##### `privileged: true | false` - -Specifies whether the pod should run as a *privileged* container (default: `false`). - -##### `runAsUser: ''` - -Specifies the user ID with which to run the container. Shortcut for the `securityContext` option. - -##### `runtimeClassName: ''` - -Specifies the [runtime class](https://kubernetes.io/docs/concepts/containers/runtime-class/). - -##### `schedulerName: ''` - -Specifies which [scheduler](https://kubernetes.io/docs/tasks/extend-kubernetes/configure-multiple-schedulers/#specify-schedulers-for-pods) is used to schedule the container. - -##### `secret: '/', mountPath: ''` - -*Can be specified multiple times* - -Mounts a [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) with name and optional key to the given path. If the key is omitted, the path is interpreted as a directory and all entries in the `Secret` are exposed in that path. - -##### `securityContext: ` - -Specifies the pod [security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) with the given configuration. - -##### `toleration: ` - -*Can be specified multiple times* - -Specifies the pod [toleration](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) with the given configuration. - -The configuration should be a map corresponding to a single toleration rule. For example, the following pod options: - -```groovy -process.pod = [ - [toleration: [key: 'key1', operator: 'Equal', value: 'value1', effect: 'NoSchedule']], - [toleration: [key: 'key1', operator: 'Exists', effect: 'NoSchedule']], -] -``` - -Maps to the following pod spec: - -```yaml -tolerations: - - key: "key1" - operator: "Equal" - value: "value1" - effect: "NoSchedule" - - key: "key1" - operator: "Exists" - effect: "NoSchedule" -``` - -##### `ttlSecondsAfterFinished` - - - -Specifies the [TTL mechanism](https://kubernetes.io/docs/concepts/workloads/controllers/job/#ttl-mechanism-for-finished-jobs) for finished jobs in seconds. Applies to both successful and failed jobs. - -##### `volumeClaim: '', mountPath: '' [, subPath: '', readOnly: true | false]` - -*Can be specified multiple times* - -Mounts a [Persistent volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) with the given name to the given path. - -The `subPath` option can be used to mount a sub-directory of the volume instead of its root. - -The `readOnly` option can be used to mount the volume as read-only (default: `false`) - -### `publishDir` - -:::note -[Workflow outputs][workflow-output-def] can be used instead of `publishDir`. See [Migrating to workflow outputs][migrating-workflow-outputs] to learn how to migrate existing code. -::: - -The `publishDir` directive publishes matching process output files to a target directory. For example: - -```nextflow -process hello { - publishDir '/data/chunks' - - output: - path 'chunk_*' - - script: - """ - printf 'Hola' | split -b 1 - chunk_ - """ -} -``` - -The above example publishes the `chunk_*` output files into the `/data/chunks` directory. - -Only files that match the declaration in the `output` block are published, not all the outputs of the process. - -The `publishDir` directive can be specified more than once in order to publish output files to different target directories based on different rules. - -By default, files are published via *symbolic link* from the task directory to the target directory. Use the `mode` option to control this behavior: - -```nextflow -process hello { - publishDir '/data/chunks', mode: 'copy', overwrite: false - - output: - path 'chunk_*' - - script: - """ - printf 'Hola' | split -b 1 - chunk_ - """ -} -``` - -:::warning -Output files are published *asynchronously* after the task execution, so they may not be immediately available in the publish directory during the pipeline run. Downstream processes should access output files through the declared process outputs, not the publish directory. -::: - -Available options: - -##### `contentType` - -*Experimental: currently only supported for S3.* - -Allow specifying the media content type of the published file a.k.a. [MIME type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_Types). If set to `true`, the content type is inferred from the file extension (default: `false`). - -##### `enabled` - -Enable or disable the publish rule depending on the boolean value specified (default: `true`). - -##### `failOnError` - - -The default value was changed from `false` to `true` - - -When `true` abort the execution if some file can't be published to the specified target directory or bucket for any cause (default: `true`) - -##### `mode` - -The file publishing method. Can be one of the following values: - -- `'copy'`: Copies the output files into the publish directory. -- `'copyNoFollow'`: Copies the output files into the publish directory without following symlinks ie. copies the links themselves. -- `'link'`: Creates a hard link in the publish directory for each output file. -- `'move'`: Moves the output files into the publish directory. **Note**: this is only supposed to be used for a *terminal* process i.e. a process whose output is not consumed by any other downstream process. -- `'rellink'`: Creates a relative symbolic link in the publish directory for each output file. -- `'symlink'`: Creates an absolute symbolic link in the publish directory for each output file (default). - -##### `overwrite` - -When `true` any existing file in the target directory will be overridden (default: `true` during normal pipeline execution and `false` when pipeline execution is `resumed`). - -##### `path` - -Specifies the directory where files need to be published. **Note**: the syntax `publishDir '/some/dir'` is a shortcut for `publishDir path: '/some/dir'`. - -##### `pattern` - -Specifies a [glob][glob] file pattern that selects which files to publish from the overall set of output files. - -##### `saveAs` - -A closure which, given the name of the file being published, returns the actual file name or a full path where the file is required to be stored. This can be used to rename or change the destination directory of the published files dynamically by using a custom strategy. Return the value `null` from the closure to *not* publish a file. This is useful when the process has multiple output files, but you want to publish only some of them. - -##### `storageClass` - - - -*Experimental: currently only supported for S3.* - -Allow specifying the storage class to be used for the published file. - -##### `tags` - -*Experimental: currently only supported for S3.* - -Allow the association of arbitrary tags with the published file e.g. `tags: [MESSAGE: 'Hello world']`. - -### `queue` - -The `queue` directive defines the queue to which tasks should be submitted, for executors that support queues. For example: - -```nextflow -process hello { - queue 'long' - executor 'slurm' - - script: - """ - your_command --here - """ -} -``` - -Some executors can accept multiple queue names as a comma-separated string: - -```nextflow -queue 'short,long,cn-el6' -``` - -However, this is not generally supported by cloud executors such as AWS Batch, Azure Batch, and Google Batch. - -See [Executors][executor-page] to see which executors support this directive. - -### `resourceLabels` - -The `resourceLabels` directive attaches custom name-value pairs to task executions, for executors that support it. For example: - -```nextflow -process hello { - resourceLabels region: 'some-region', user: 'some-username' - - script: - """ - your_command --here - """ -} -``` - -Resource labels are attached to underlying resources such as cloud VMs, and are intended for operational purposes such as cost tracking. They are not recorded in lineage metadata. - -When `resourceLabels` is specified multiple times in the config, only the last setting is used. Additionally, when `resourceLabels` is specified both in the config and the process definition, only the process definition is used. - -As a best practice, define all resource labels in a single config setting: - -```groovy -process { - resourceLabels = [ region: 'some-region', user: 'some-username' ] -} -``` - -Use process selectors (`withName:` or `withLabel:`) to override resource labels for a specific process. - -Resource labels are currently supported by the following executors: - -- [AWS Batch][awsbatch-executor] -- [Azure Batch][azurebatch-executor] -- [Google Cloud Batch][google-batch-executor] -- [Kubernetes][k8s-executor] -- [Seqera executor][seqera-executor] - -:::note -The limits and the syntax of the corresponding executor should be taken into consideration when using resource labels. -::: - - -Resource labels in Azure are added to auto-pools, rather than jobs, in order to facilitate cost analysis. A new pool will be created for each new set of resource labels. Therefore, it is recommended to also set `azure.batch.deletePoolsOnCompletion = true` when using process-specific resource labels. - - -See also: [label](#label) (for shared process configuration), [tag](#tag) (for per-task identification) - -### `resourceLimits` - - - -The `resourceLimits` directive defines environment-specific limits for task resource requests. - -Resource limits can be specified in a process: - -```nextflow -process hello { - resourceLimits cpus: 24, memory: 768.GB, time: 72.h - - script: - """ - your_command --here - """ -} -``` - -Or in the Nextflow configuration: - -```nextflow -process.resourceLimits = [ - cpus: 24, - memory: 768.GB, - time: 72.h -] -``` - -Resource limits can be defined for the following directives: - -- [cpus](#cpus) -- [disk](#disk) -- [memory](#memory) -- [time](#time) - -When a task resource request exceeds the corresponding limit, the task resources are automatically reduced to comply with these limits before the job is submitted. - -Resource limits are a useful way to prevent tasks with [dynamic resources][dynamic-task-resources] from requesting more resources than can be provided by an executor (e.g. a task requests 32 cores but the largest node in the cluster has 24). - -### `scratch` - -The `scratch` directive executes each task in a temporary directory that is local to the compute node. - -This is useful when executing tasks on an executor with a shared filesystem, because it decreases the network overhead of reading and writing files. Only the files declared as process outputs are copied to the pipeline work directory. - -For example: - -```nextflow -process hello { - scratch true - - output: - path 'data_out' - - script: - """ - your_command --here - """ -} -``` - -It can also be specified in the Nextflow configuration: - -```groovy -process.scratch = true -``` - -By default, the `scratch` directive uses the `$TMPDIR` environment variable in the underlying node as the base scratch directory. If `$TMPDIR` is not defined, then it creates a scratch directory using the `mktemp` command. - -Each task creates a subdirectory within the base scratch directory and automatically deletes it upon completion. - -:::note -Cloud-based executors enable `scratch` by default since the pipeline work directory resides in object storage. -::: - -The following values are supported: - -##### `false` - -Do not use a scratch directory. - -##### `true` - -Create a scratch directory in the directory defined by the `$TMPDIR` environment variable, or `$(mktemp /tmp)` if `$TMPDIR` is not set. - -##### `'$YOUR_VAR'` - -Create a scratch directory in the directory defined by the given environment variable, or `$(mktemp /tmp)` if that variable is not set. The value must use single quotes, otherwise the environment variable will be evaluated in the pipeline script context. - -##### `'/my/tmp/path'` - -Create a scratch directory in the specified directory. - -##### `'ram-disk'` - -Create a scratch directory in the RAM disk `/dev/shm/`. - -### `secret` - -The `secret` directive allows a process to access secrets. - -For example: - -```nextflow -process hello_secret { - secret 'MY_ACCESS_KEY' - secret 'MY_SECRET_KEY' - - script: - """ - your_command --access \$MY_ACCESS_KEY --secret \$MY_SECRET_KEY - """ -} -``` - -Each secret is provided to the task as an environment variable. - -See [Secrets][secrets-page] for more information. - -:::note -Secrets can only be used with the local or grid executors (e.g., Slurm or Grid Engine). Secrets can be used with AWS Batch and Google Batch when launched from Seqera Platform. -::: - -### `shell` - -The `shell` directive defines a custom shell command for process scripts. By default, script blocks are executed with `/bin/bash -ue`. - -```nextflow -process hello { - shell '/bin/bash', '-euo', 'pipefail' - - script: - """ - your_command --here - """ -} -``` - -It can also be specified in the Nextflow configuration: - -```groovy -process.shell = ['/bin/bash', '-euo', 'pipefail'] -``` - -### `spack` - -The `spack` directive defines the set of [Spack](https://spack.io) packages required by each task. For example: - -```nextflow -process hello { - spack 'bwa@0.7.15' - - script: - """ - your_command --here - """ -} -``` - -Nextflow automatically creates a Spack environment for each unique set of packages. - -Multiple packages can be specified separating them with a blank space, e.g. `bwa@0.7.15 fastqc@0.11.5`. - -The `spack` directive also accepts a Spack environment file path or the path of an existing Spack environment. See [Spack][spack-page] for more information. - -### `stageInMode` - -The `stageInMode` directive defines how input files are staged into the task work directory. - -The following modes are supported: - -##### `'copy'` - -Input files are staged in the task work directory by creating a copy. - -##### `'link'` - -Input files are staged in the task work directory by creating a hard link for each of them. - -##### `'rellink'` - -Input files are staged in the task work directory by creating a symbolic link with a relative path for each of them. - -##### `'symlink'` - -Input files are staged in the task work directory by creating a symbolic link with an absolute path for each of them (default). - -### `stageOutMode` - -The `stageOutMode` directive defines how output files are staged out from the scratch directory to the task work directory. - -The following modes are supported: - -##### `'copy'` - -Output files are copied from the scratch directory to the work directory. - -##### `'fcp'` - - - -Output files are copied from the scratch directory to the work directory by using the [fcp](https://github.com/Svetlitski/fcp) utility (note: it must be available in the task environment). - -##### `'move'` - -Output files are moved from the scratch directory to the work directory. - -##### `'rclone'` - - - -Output files are copied from the scratch directory to the work directory by using the [rclone](https://rclone.org) utility (note: it must be available in the task environment). - -##### `'rsync'` - -Output files are copied from the scratch directory to the work directory by using the `rsync` utility. - -See also: [scratch](#scratch) - -### `storeDir` - -The `storeDir` directive stores task outputs in a permanent *store directory* instead of the work directory. - -On subsequent runs, each task is executed only if the declared output files do not exist in the store directory. When the files are present, the task is skipped and these files are used as the task outputs. - -The following example shows how to use the `storeDir` directive to create a directory containing a BLAST database for each species specified by an input parameter: - -```nextflow -process make_blast_db { - storeDir '/db/genomes' - - input: - path species - - output: - path "${dbName}.*" - - script: - dbName = species.baseName - """ - makeblastdb -dbtype nucl -in ${species} -out ${dbName} - """ -} -``` - -Caveats: - -- The `env`, `eval`, and `stdout` output qualifiers cannot be used with `storeDir` because they rely on helper files in the task directory. Use `path` outputs instead. - -- If a process uses `storeDir` and all of its outputs are optional, the process will always be skipped, even if the store directory is empty. Avoid this issue by specifying at least one required file output. - -- The `storeDir` directive is not a replacement for publishing outputs. Use the [publishDir](#publishdir) directive or [workflow outputs][workflow-output-def] instead. - -### `tag` - -The `tag` directive defines a custom identifier for each task execution. For example: - -```nextflow -process hello { - tag "$code" - - input: - val code - - script: - """ - echo $code - """ -} - -workflow { - ch_codes = channel.of('alpha', 'gamma', 'omega') - hello(ch_codes) -} -``` - -The above example logs each task with its corresponding tag: - -``` -[6e/28919b] Submitted process > hello (alpha) -[d2/1c6175] Submitted process > hello (gamma) -[1c/3ef220] Submitted process > hello (omega) -``` - -Tags are a useful way to track related tasks in a pipeline run. Tasks can be identified by tag in the [execution log][execution-log] and the [trace report][trace-report]. - -:::note -The name of a task in both [reports][tracing-page] and [lineage][data-lineage-page] is defined as ` ()`. -::: - -:::note -The `tag` directive is not related to the [label](#label) directive. Process labels are only used for shared process configuration, not for tracking. -::: - -### `time` - -The `time` directive defines the maximum runtime for each task. For example: - -```nextflow -process hello { - time 1.h - - script: - """ - your_command --here - """ -} -``` - -The following suffixes can be used to specify duration values: - -- `ms`: milliseconds -- `s`: seconds -- `m`: minutes -- `h`: hours -- `d`: days - -See [Duration][stdlib-types-duration] for more information. - -:::note -This directive is only used by certain executors. Refer to the [Executors][executor-page] page to see which executors support this directive. -::: - -See also: [cpus](#cpus), [disk](#disk), [memory](#memory), [queue](#queue), [dynamic task resources][dynamic-task-resources] - -[awsbatch-executor]: ../executor/aws-batch -[azurebatch-executor]: ../executor/azure-batch -[cache-resume-page]: ../cache-and-resume -[channel-topic]: ./stdlib-namespaces/channel#topic -[config-process-selectors]: ../config#process-selectors -[conda-page]: ../conda -[container-page]: ../container -[data-lineage-page]: ../tutorials/data-lineage -[dynamic-directives]: ../process#dynamic-directives -[dynamic-task-resources]: ../process#dynamic-task-resources -[execution-log]: ../reports#execution-log -[executor-page]: ../executor -[getstarted-resume]: ../your-first-script#modify-and-resume -[glob]: https://docs.oracle.com/javase/tutorial/essential/io/fileOps.html#glob -[google-batch-executor]: ../executor/google-batch -[k8s-executor]: ../executor/kubernetes -[lsf-executor]: ../executor/lsf -[migrating-workflow-outputs]: ../tutorials/workflow-outputs -[pbs-executor]: ../executor/pbs -[pbspro-executor]: ../executor/pbspro -[process-multiple-input-files]: ../process#multiple-input-files -[process-native]: ../process#native-execution +This section describes the task properties, input/output qualifiers, and directives available in [process][process-page] definitions. + +| Page | Description | +| ---- | ----------- | +| [Task properties][process-task-properties] | The properties of the current task, available in the process body. | +| [Inputs and outputs (typed)][process-inputs-outputs-typed] | The stage directives and output functions available in a typed process. | +| [Inputs and outputs (legacy)][process-inputs-outputs] | The input and output qualifiers available in a legacy process. | +| [Directives][process-directives] | The settings that control how each task is executed. | + +[process-directives]: ./process/directives +[process-inputs-outputs]: ./process/inputs-outputs +[process-inputs-outputs-typed]: ./process/inputs-outputs-typed [process-page]: ../process -[secrets-page]: ../secrets -[seqera-executor]: ../executor/seqera -[sge-executor]: ../executor/sge -[slurm-executor]: ../executor/slurm -[spack-page]: ../spack -[stdlib-namespaces-workflow]: ./stdlib-namespaces/workflow -[stdlib-types-duration]: ./stdlib-types/duration -[stdlib-types-memoryunit]: ./stdlib-types/memory-unit -[task-directive-values]: ../process#using-task-directive-values -[trace-report]: ../reports#trace-file -[tracing-page]: ../reports -[wave-page]: ../wave -[workflow-output-def]: ../workflow#outputs +[process-task-properties]: ./process/task-properties diff --git a/docs/reference/process/directives.mdx b/docs/reference/process/directives.mdx new file mode 100644 index 0000000000..49ca8a85df --- /dev/null +++ b/docs/reference/process/directives.mdx @@ -0,0 +1,113 @@ +--- +title: Directives +description: Reference for the directives that control the execution of process tasks. +--- + +# Directives + +Directives are optional settings that affect the execution of the current process. They are declared at the top of a process definition, before any input, output, or script section. + +By default, directives are evaluated when the process is defined. However, if the value is a dynamic string or closure, it will be evaluated separately for each task. See [Dynamic directives][dynamic-directives] for more information. + +Directives can also be specified in a configuration file using the `process` config scope, which is the recommended way to configure a pipeline for a particular compute environment. See [Process configuration][config-process-scope] for more information. + +Some directives are only supported by specific executors. See [Executors][executor-page] for more information about each executor. + +The following directives are available: + +| Directive | Description | +| --------- | ----------- | +| [`accelerator`][accelerator] | The number of hardware accelerators (e.g. GPUs) required by each task. | +| [`afterScript`][after-script] | A Bash snippet to execute after the task script. | +| [`arch`][arch] | The CPU architecture to build the software required by each task. | +| [`array`][array] | Submit tasks as job arrays, for executors that support it. | +| [`beforeScript`][before-script] | A Bash snippet to execute before the task script. | +| [`cache`][cache] | Whether and how task executions are cached. | +| [`clusterOptions`][cluster-options] | Additional submission options for grid executors. | +| [`conda`][conda] | The set of Conda packages required by each task. | +| [`container`][container] | The container image required by each task. | +| [`containerOptions`][container-options] | Additional options for the underlying container runtime. | +| [`cpus`][cpus] | The number of CPUs required by each task. | +| [`debug`][debug] | Print the standard output of each task to the pipeline standard output. | +| [`disk`][disk] | The amount of disk storage required by each task. | +| [`errorStrategy`][error-strategy] | How to handle task failures. | +| [`executor`][executor] | The underlying system where tasks are executed. | +| [`ext`][ext] | A generic directive for user-defined properties. | +| [`fair`][fair] | Emit process outputs in the order in which they were received. | +| [`hints`][hints] | Executor-specific hints, specified as key-value pairs. | +| [`label`][label] | A custom label, used to apply shared process configuration. | +| [`machineType`][machine-type] | A predefined machine type for cloud executors that support it. | +| [`maxErrors`][max-errors] | The maximum number of task failures allowed for a process. | +| [`maxForks`][max-forks] | The maximum number of concurrent task executions for a process. | +| [`maxRetries`][max-retries] | The maximum number of times a task can be retried. | +| [`maxSubmitAwait`][max-submit-await] | How long a task can remain in the submission queue before failing. | +| [`memory`][memory] | The amount of memory required by each task. | +| [`module`][module] | The set of Environment Modules required by each task. | +| [`penv`][penv] | The parallel environment to use when submitting tasks to SGE. | +| [`pod`][pod] | Pod-specific settings for the Kubernetes executor. | +| [`publishDir`][publish-dir] | Publish matching process output files to a target directory. | +| [`queue`][queue] | The queue to which tasks should be submitted. | +| [`resourceLabels`][resource-labels] | Custom name-value pairs to attach to task executions. | +| [`resourceLimits`][resource-limits] | Environment-specific limits for task resource requests. | +| [`scratch`][scratch] | Execute each task in a temporary directory local to the compute node. | +| [`secret`][secret] | The secrets required by each task. | +| [`shell`][shell] | A custom shell command for process scripts. | +| [`spack`][spack] | The set of Spack packages required by each task. | +| [`stageInMode`][stage-in-mode] | How input files are staged into the task directory. | +| [`stageOutMode`][stage-out-mode] | How output files are staged out of the scratch directory. | +| [`storeDir`][store-dir] | Store task outputs in a permanent store directory. | +| [`tag`][tag] | A custom identifier for each task execution. | +| [`time`][time] | The maximum runtime for each task. | + +## See also + +- [Dynamic directives][dynamic-directives]: How to define a directive value based on the task inputs. +- [Dynamic task resources][dynamic-task-resources]: How to define task resources that scale with each retry. +- [Task directive values][task-directive-values]: How to access directive values in a process body. + +[config-process-scope]: ../../config#process-configuration +[dynamic-directives]: ../../process#dynamic-directives +[executor-page]: ../../executor +[dynamic-task-resources]: ../../process#dynamic-task-resources +[task-directive-values]: ../../process#using-task-directive-values +[accelerator]: ./directives/accelerator +[after-script]: ./directives/after-script +[arch]: ./directives/arch +[array]: ./directives/array +[before-script]: ./directives/before-script +[cache]: ./directives/cache +[cluster-options]: ./directives/cluster-options +[conda]: ./directives/conda +[container]: ./directives/container +[container-options]: ./directives/container-options +[cpus]: ./directives/cpus +[debug]: ./directives/debug +[disk]: ./directives/disk +[error-strategy]: ./directives/error-strategy +[executor]: ./directives/executor +[ext]: ./directives/ext +[fair]: ./directives/fair +[hints]: ./directives/hints +[label]: ./directives/label +[machine-type]: ./directives/machine-type +[max-errors]: ./directives/max-errors +[max-forks]: ./directives/max-forks +[max-retries]: ./directives/max-retries +[max-submit-await]: ./directives/max-submit-await +[memory]: ./directives/memory +[module]: ./directives/module +[penv]: ./directives/penv +[pod]: ./directives/pod +[publish-dir]: ./directives/publish-dir +[queue]: ./directives/queue +[resource-labels]: ./directives/resource-labels +[resource-limits]: ./directives/resource-limits +[scratch]: ./directives/scratch +[secret]: ./directives/secret +[shell]: ./directives/shell +[spack]: ./directives/spack +[stage-in-mode]: ./directives/stage-in-mode +[stage-out-mode]: ./directives/stage-out-mode +[store-dir]: ./directives/store-dir +[tag]: ./directives/tag +[time]: ./directives/time diff --git a/docs/reference/process/directives/accelerator.mdx b/docs/reference/process/directives/accelerator.mdx new file mode 100644 index 0000000000..d06f7811bd --- /dev/null +++ b/docs/reference/process/directives/accelerator.mdx @@ -0,0 +1,50 @@ +--- +title: accelerator +description: Reference for the `accelerator` process directive. +--- + +# `accelerator` + +The `accelerator` directive defines the number of hardware accelerators (e.g. GPUs) required by each task execution. For example: + +```nextflow +process hello { + accelerator 4, type: 'nvidia-tesla-k80' + + script: + """ + your_gpu_enabled --command --line + """ +} +``` + +The above example requests 4 GPUs of type `nvidia-tesla-k80` for each task. + +:::note +This directive is only used by certain executors. Refer to the [Executors][executor-page] page to see which executors support this directive. +::: + +:::note +Additional options may be required to fully enable the use of accelerators. When using containers with GPUs, you must pass the GPU drivers through to the container. For Docker, this requires the option `--gpus all` in the `docker run` command. For Apptainer/Singularity, this requires the option `--nv`. The specific implementation details depend on the accelerator and container type being used. +::: + +The following options are available: + +##### `request: Integer` + +The number of requested accelerators. + +Specifying this directive with a number (e.g., `accelerator 4`) is equivalent to the `request` option (e.g., `accelerator request: 4`). + +##### `type: String` + +The accelerator type. + +The meaning of this option depends on the target execution platform. See the platform-specific documentation for more information about the available accelerators: + + - [Google Cloud](https://cloud.google.com/compute/docs/gpus/) + - [Kubernetes](https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/#clusters-containing-different-types-of-gpus) + +This option is not supported for AWS Batch. You can control the accelerator type indirectly through the allowed instance types in your Compute Environment. See the [AWS Batch FAQs](https://aws.amazon.com/batch/faqs/?#GPU_Scheduling_) for more information. + +[executor-page]: ../../../executor diff --git a/docs/reference/process/directives/after-script.mdx b/docs/reference/process/directives/after-script.mdx new file mode 100644 index 0000000000..47fed34869 --- /dev/null +++ b/docs/reference/process/directives/after-script.mdx @@ -0,0 +1,10 @@ +--- +title: afterScript +description: Reference for the `afterScript` process directive. +--- + +# `afterScript` + +The `afterScript` directive executes a custom (Bash) snippet immediately *after* the main process has run. This may be useful to clean up your staging area. + +When combined with the [container](./container) directive, the `afterScript` is executed outside the specified container. In other words, the `afterScript` is always executed in the host environment. diff --git a/docs/reference/process/directives/arch.mdx b/docs/reference/process/directives/arch.mdx new file mode 100644 index 0000000000..c9672a6ab0 --- /dev/null +++ b/docs/reference/process/directives/arch.mdx @@ -0,0 +1,37 @@ +--- +title: arch +description: Reference for the `arch` process directive. +--- + +# `arch` + +The `arch` directive defines the CPU architecture to build the software in use by the process' task. For example: + +```nextflow +process blast { + spack 'blast-plus@2.13.0' + arch 'linux/x86_64', target: 'cascadelake' + + script: + """ + blastp -query input_sequence -num_threads ${task.cpus} + """ +} +``` + +The example above declares that the CPU generic architecture is `linux/x86_64` (X86 64 bit), and more specifically that the microarchitecture is `cascadelake` (a specific generation of Intel CPUs). + +This directive is currently used by the following Nextflow functionalities: + +- by the [spack](./spack) directive, to build microarchitecture-optimized applications; +- by the [Wave][wave-page] service, to build containers for one of the generic families of CPU architectures (see below); +- by the `spack` strategy within [Wave][wave-page], to optimize the container builds for specific CPU microarchitectures. + +Allowed values for the `arch` directive are as follows, grouped by equivalent family (choices available for the sake of compatibility): +- X86 64 bit: `linux/x86_64`, `x86_64`, `linux/amd64`, `amd64` +- ARM 64 bit: `linux/aarch64`, `aarch64`, `linux/arm64`, `arm64`, `linux/arm64/v8` +- ARM 64 bit, older generation: `linux/arm64/v7` + +Examples of values for the architecture `target` option are `cascadelake`, `icelake`, `zen2` and `zen3`. See the [Spack documentation](https://spack.readthedocs.io/en/latest/basic_usage.html#support-for-specific-microarchitectures) for the full and up-to-date list of meaningful targets. + +[wave-page]: ../../../wave diff --git a/docs/reference/process/directives/array.mdx b/docs/reference/process/directives/array.mdx new file mode 100644 index 0000000000..3b238e61ae --- /dev/null +++ b/docs/reference/process/directives/array.mdx @@ -0,0 +1,71 @@ +--- +title: array +description: Reference for the `array` process directive. +--- + +# `array` + + + +The `array` directive submits tasks as *job arrays* for executors that support it. + +A job array is a collection of jobs with the same resource requirements and the same script (parameterized by an index). Job arrays incur significantly less scheduling overhead compared to individual jobs, and as a result they are preferred by HPC schedulers where possible. + +The directive should be specified with a given array size, along with an executor that supports job arrays. For example: + +```nextflow +process hello { + executor 'slurm' + array 100 + + script: + """ + your_command --here + """ +} +``` + +Nextflow currently supports job arrays for the following executors: + +- [AWS Batch][awsbatch-executor] +- [Google Cloud Batch][google-batch-executor] +- [LSF][lsf-executor] +- [PBS][pbs-executor] +- [PBS Pro][pbspro-executor] +- [SGE][sge-executor] +- [SLURM][slurm-executor] + +A process using job arrays collects tasks and submits each batch as a job array when it is ready. Any "leftover" tasks are submitted as a partial job array. + +Once a job array is submitted, each "child" task is executed as an independent job. Any tasks that fail (and can be retried) are retried without interfering with the tasks that succeeded. Retried tasks are submitted individually rather than through a job array, in order to allow for the use of [dynamic resources][dynamic-task-resources]. + +The following directives must be uniform across all tasks in a process that uses job arrays, because these directives are specified once for the entire job array: + +- [accelerator](./accelerator) +- [clusterOptions](./cluster-options) +- [cpus](./cpus) +- [disk](./disk) +- [machineType](./machine-type) +- [memory](./memory) +- [queue](./queue) +- [resourceLabels](./resource-labels) +- [resourceLimits](./resource-limits) +- [time](./time) + +For cloud-based executors like AWS Batch, or when using Fusion with any executor, the following additional directives must be uniform: + +- [container](./container) +- [containerOptions](./container-options) + +When using Wave, the following additional directives must be uniform: + +- [conda](./conda) + +[awsbatch-executor]: ../../../executor/aws-batch +[dynamic-task-resources]: ../../../process#dynamic-task-resources +[google-batch-executor]: ../../../executor/google-batch +[lsf-executor]: ../../../executor/lsf +[pbs-executor]: ../../../executor/pbs +[pbspro-executor]: ../../../executor/pbspro +[sge-executor]: ../../../executor/sge +[slurm-executor]: ../../../executor/slurm diff --git a/docs/reference/process/directives/before-script.mdx b/docs/reference/process/directives/before-script.mdx new file mode 100644 index 0000000000..16f36a81b0 --- /dev/null +++ b/docs/reference/process/directives/before-script.mdx @@ -0,0 +1,23 @@ +--- +title: beforeScript +description: Reference for the `beforeScript` process directive. +--- + +# `beforeScript` + +The `beforeScript` directive executes a custom (Bash) snippet *before* the main process script is run. This may be useful to initialize the underlying cluster environment or for other custom initialization. + +For example: + +```nextflow +process hello { + beforeScript 'source /cluster/bin/setup' + + script: + """ + echo 'hello' + """ +} +``` + +When the process is containerized (using the [container](./container) directive), the `beforeScript` is executed in the container only if the executor is *container-native* (e.g. cloud batch executors, Kubernetes). Otherwise, the `beforeScript` is executed outside the container. diff --git a/docs/reference/process/directives/cache.mdx b/docs/reference/process/directives/cache.mdx new file mode 100644 index 0000000000..50491c0629 --- /dev/null +++ b/docs/reference/process/directives/cache.mdx @@ -0,0 +1,43 @@ +--- +title: cache +description: Reference for the `cache` process directive. +--- + +# `cache` + +The `cache` directive controls whether and how task executions are cached. + +By default, cached task executions are re-used when the pipeline is launched with the [resume][getstarted-resume] option. The `cache` directive can be used to disable caching for a specific process: + +```nextflow +process hello { + cache false + + // ... +} +``` + +See [Cache and resume][cache-resume-page] for more information. + +The following options are available: + +##### `false` + +Disable caching. + +##### `true` (default) + +Enable caching. Input file metadata (name, size, last updated timestamp) are included in the cache keys. + +##### `'deep'` + +Enable caching. Input file content is included in the cache keys. + +##### `'lenient'` + +Enable caching. Minimal input file metadata (name and size only) are included in the cache keys. + +This strategy provides a workaround for incorrect caching invalidation observed on shared file systems due to inconsistent file timestamps. + +[cache-resume-page]: ../../../cache-and-resume +[getstarted-resume]: ../../../your-first-script#modify-and-resume diff --git a/docs/reference/process/directives/cluster-options.mdx b/docs/reference/process/directives/cluster-options.mdx new file mode 100644 index 0000000000..dc6ffcdf6e --- /dev/null +++ b/docs/reference/process/directives/cluster-options.mdx @@ -0,0 +1,44 @@ +--- +title: clusterOptions +description: Reference for the `clusterOptions` process directive. +--- + +# `clusterOptions` + +The `clusterOptions` directive specifies additional submission options for grid executors. You can use it to specify options for your cluster that are not supported directly by other process directives. + +The cluster options can be a string: + +```nextflow +process hello { + clusterOptions '-x 1 -y 2' + + // ... +} +``` + + +Prior to this version, grid executors that require each option to be on a separate line in the job script would attempt to split multiple options using a variety of different conventions. Multiple options can now be specified more clearly using a string list as shown below. + + +The cluster options can also be a string list: + +```nextflow +process hello { + clusterOptions '-x 1', '-y 2', '--flag' + + // ... +} +``` + +Grid executors that require one option per line will write each option to a separate line, while grid executors that allow multiple options per line will write all options to a single line, the same as with a string. This form is useful to control how the options are split across lines when it is required by the scheduler. + +:::note +This directive is only used by grid executors. Refer to the [Executors][executor-page] page to see which executors support this directive. +::: + +:::warning +While you can use the `clusterOptions` directive to specify options that are supported as process directives (`queue`, `memory`, `time`, etc), you should not use both at the same time, as it will cause undefined behavior. Most HPC schedulers will either fail or simply ignore one or the other. +::: + +[executor-page]: ../../../executor diff --git a/docs/reference/process/directives/conda.mdx b/docs/reference/process/directives/conda.mdx new file mode 100644 index 0000000000..561e390aa0 --- /dev/null +++ b/docs/reference/process/directives/conda.mdx @@ -0,0 +1,27 @@ +--- +title: conda +description: Reference for the `conda` process directive. +--- + +# `conda` + +The `conda` directive defines the set of [Conda](https://conda.io) packages required by each task. For example: + +```nextflow +process hello { + conda 'bwa=0.7.15' + + script: + """ + your_command --here + """ +} +``` + +Nextflow automatically creates an environment for each unique set of Conda packages. + +The name of the desired channel for a specific package can be specified using the standard Conda notation, e.g. `bioconda::bwa=0.7.15`. Multiple packages can be specified separating them with a blank space, e.g. `bwa=0.7.15 fastqc=0.11.5`. + +The `conda` directive can also accept a Conda environment file path or the path of an existing Conda environment. See [Conda][conda-page] for more information. + +[conda-page]: ../../../conda diff --git a/docs/reference/process/directives/container-options.mdx b/docs/reference/process/directives/container-options.mdx new file mode 100644 index 0000000000..cf73da8c90 --- /dev/null +++ b/docs/reference/process/directives/container-options.mdx @@ -0,0 +1,31 @@ +--- +title: containerOptions +description: Reference for the `containerOptions` process directive. +--- + +# `containerOptions` + +The `containerOptions` directive specifies additional container options for the underlying container runtime (e.g. Docker, Singularity). For example: + +```nextflow +process hello_docker { + container 'busybox:latest' + containerOptions '--volume /data/db:/db' + + output: + path 'output.txt' + + script: + """ + your_command --data /db > output.txt + """ +} +``` + +The above example provides a custom volume mount for a specific process. + +:::warning +This directive is not supported by the [Kubernetes][k8s-executor] executor. +::: + +[k8s-executor]: ../../../executor/kubernetes diff --git a/docs/reference/process/directives/container.mdx b/docs/reference/process/directives/container.mdx new file mode 100644 index 0000000000..f164cba10c --- /dev/null +++ b/docs/reference/process/directives/container.mdx @@ -0,0 +1,28 @@ +--- +title: container +description: Reference for the `container` process directive. +--- + +# `container` + +The `container` directive defines the container required by each task. For example: + +```nextflow +process hello_docker { + container 'busybox:latest' + + script: + """ + your_command --here + """ +} +``` + +The corresponding container runtime (e.g. Docker, Singularity) should be running on the compute nodes where tasks are executed. See [Containers][container-page] for the container runtimes supported by Nextflow. + +:::note +This directive is ignored by [native processes][process-native] (i.e. `exec` processes). +::: + +[container-page]: ../../../container +[process-native]: ../../../process#native-execution diff --git a/docs/reference/process/directives/cpus.mdx b/docs/reference/process/directives/cpus.mdx new file mode 100644 index 0000000000..f5e1157d63 --- /dev/null +++ b/docs/reference/process/directives/cpus.mdx @@ -0,0 +1,25 @@ +--- +title: cpus +description: Reference for the `cpus` process directive. +--- + +# `cpus` + +The `cpus` directive defines the number of CPUs required by each task execution. For example: + +```nextflow +process blast { + cpus 8 + + script: + """ + blastp -query input_sequence -num_threads ${task.cpus} + """ +} +``` + +This directive is required for tasks that execute multi-process or multi-threaded commands/tools and it is meant to reserve enough CPUs when a pipeline task is executed through a cluster resource manager. + +See also: [disk](./disk), [memory](./memory), [time](./time), [queue](./queue), [dynamic task resources][dynamic-task-resources] + +[dynamic-task-resources]: ../../../process#dynamic-task-resources diff --git a/docs/reference/process/directives/debug.mdx b/docs/reference/process/directives/debug.mdx new file mode 100644 index 0000000000..7da376c473 --- /dev/null +++ b/docs/reference/process/directives/debug.mdx @@ -0,0 +1,29 @@ +--- +title: debug +description: Reference for the `debug` process directive. +--- + +# `debug` + +The `debug` directive prints the standard output of each task to the pipeline standard output. + +For example: + +```nextflow +process hello { + debug true + + script: + """ + echo Hello + """ +} +``` + +Prints: + +``` +Hello +``` + +Removing the `debug` directive or setting it to `false` in the above example will cause `Hello` to not be printed. diff --git a/docs/reference/process/directives/disk.mdx b/docs/reference/process/directives/disk.mdx new file mode 100644 index 0000000000..603df3b1b1 --- /dev/null +++ b/docs/reference/process/directives/disk.mdx @@ -0,0 +1,39 @@ +--- +title: disk +description: Reference for the `disk` process directive. +--- + +# `disk` + +The `disk` directive defines the amount of disk storage required by each task execution. For example: + +```nextflow +process hello { + disk 2.GB + + script: + """ + your_command --here + """ +} +``` + +The following suffixes can be used to specify disk values: + +- `B`: Bytes +- `KB`: Kilobytes +- `MB`: Megabytes +- `GB`: Gigabytes +- `TB`: Terabytes + +See [MemoryUnit][stdlib-types-memoryunit] for more information. + +:::note +The `disk` directive is only used by certain executors. Refer to the [Executors][executor-page] page to see which executors support this directive. +::: + +See also: [cpus](./cpus), [memory](./memory), [time](./time), [queue](./queue), [dynamic task resources][dynamic-task-resources] + +[dynamic-task-resources]: ../../../process#dynamic-task-resources +[executor-page]: ../../../executor +[stdlib-types-memoryunit]: ../../stdlib-types/memory-unit diff --git a/docs/reference/process/directives/error-strategy.mdx b/docs/reference/process/directives/error-strategy.mdx new file mode 100644 index 0000000000..97482613d2 --- /dev/null +++ b/docs/reference/process/directives/error-strategy.mdx @@ -0,0 +1,66 @@ +--- +title: errorStrategy +description: Reference for the `errorStrategy` process directive. +--- + +# `errorStrategy` + +The `errorStrategy` directive defines how to handle task failures. + +A task failure occurs when the executed script returns a non-zero exit code. By default, the pipeline run is aborted. + +The following error strategies are available: + +##### `'terminate'` (default) + +When a task fails, terminate the pipeline immediately and report an error. Pending and running jobs are killed. + +##### `'finish'` + +When a task fails, wait for submitted and running tasks to finish and then terminate the pipeline, reporting an error. + +##### `'ignore'` + +When a task fails, ignore it and continue the pipeline execution. If the `workflow.failOnIgnore` config option is set to `true`, the pipeline will report an error (i.e. return a non-zero exit code) upon completion. Otherwise, the pipeline will complete successfully. + +See the [workflow][stdlib-namespaces-workflow] namespace for more information. + +##### `'retry'` + +When a task fails, retry it. + +When setting the `errorStrategy` directive to `ignore` the process doesn't stop on an error condition, it just reports a message notifying you of the error event. + +For example: + +```nextflow +process hello { + errorStrategy 'ignore' + + // ... +} +``` + +In this case, the workflow will complete successfully and return an exit status of 0. However, if you set `workflow.failOnIgnore = true` in your Nextflow configuration, the workflow will return a non-zero exit status and report the failed tasks as an error. + +The `retry` error strategy retries failed tasks. For example: + +```nextflow +process hello { + errorStrategy 'retry' + + // ... +} +``` + +The number of times a failing process is re-executed is defined by the [maxRetries](./max-retries) and [maxErrors](./max-errors) directives. + +:::tip +More complex strategies depending on the task exit status or other parametric values can be defined using a dynamic `errorStrategy`. See [Dynamic directives][dynamic-directives] for details. +::: + +See also: [maxErrors](./max-errors), [maxRetries](./max-retries), [dynamic task resources][dynamic-task-resources] + +[dynamic-directives]: ../../../process#dynamic-directives +[dynamic-task-resources]: ../../../process#dynamic-task-resources +[stdlib-namespaces-workflow]: ../../stdlib-namespaces/workflow diff --git a/docs/reference/process/directives/executor.mdx b/docs/reference/process/directives/executor.mdx new file mode 100644 index 0000000000..26ea0350f3 --- /dev/null +++ b/docs/reference/process/directives/executor.mdx @@ -0,0 +1,29 @@ +--- +title: executor +description: Reference for the `executor` process directive. +--- + +# `executor` + +The `executor` directive defines the underlying system where tasks are executed. For example: + +```nextflow +process hello { + executor 'slurm' + + // ... +} +``` + +Commonly used executors include: + +- `awsbatch`: [AWS Batch](https://aws.amazon.com/batch/) +- `azurebatch`: [Azure Batch](https://azure.microsoft.com/en-us/services/batch/) +- `google-batch`: [Google Cloud Batch](https://cloud.google.com/batch) +- `local`: local machine +- `lsf`: [Platform LSF](http://en.wikipedia.org/wiki/Platform_LSF) job scheduler +- `slurm`: [SLURM](https://en.wikipedia.org/wiki/Slurm_Workload_Manager) workload manager + +Each executor supports additional configuration options under the `executor` config scope. See [Executors][executor-page] for more information. + +[executor-page]: ../../../executor diff --git a/docs/reference/process/directives/ext.mdx b/docs/reference/process/directives/ext.mdx new file mode 100644 index 0000000000..8ac6e87925 --- /dev/null +++ b/docs/reference/process/directives/ext.mdx @@ -0,0 +1,42 @@ +--- +title: ext +description: Reference for the `ext` process directive. +--- + +# `ext` + +The `ext` is a generic directive for user-defined properties. For example: + +```nextflow +process star { + container "biocontainers/star:${task.ext.version}" + + input: + path genome + tuple val(sampleId), path(reads) + + script: + """ + STAR --genomeDir $genome --readFilesIn $reads ${task.ext.args ?: ''} + """ +} +``` + +In the above example, the process container version is controlled by `ext.version`, and the script supports additional command line arguments through `ext.args`. + +The `ext` directive can be set in the process definition: + +```nextflow +process hello { + ext version: '2.5.3', args: '--alpha --beta' + + // ... +} +``` + +Or in the Nextflow configuration: + +```groovy +process.ext.version = '2.5.3' +process.ext.args = '--alpha --beta' +``` diff --git a/docs/reference/process/directives/fair.mdx b/docs/reference/process/directives/fair.mdx new file mode 100644 index 0000000000..779e64d564 --- /dev/null +++ b/docs/reference/process/directives/fair.mdx @@ -0,0 +1,40 @@ +--- +title: fair +description: Reference for the `fair` process directive. +--- + +# `fair` + + + +The `fair` directive, when enabled, guarantees that process outputs will be emitted in the order in which they were received. For example: + +```nextflow +process hello { + fair true + + input: + val x + + output: + tuple val(task.index), val(x) + + script: + """ + sleep \$((RANDOM % 3)) + """ +} + +workflow { + channel.of('A','B','C','D') | hello | view +} +``` + +The above example produces: + +``` +[1, A] +[2, B] +[3, C] +[4, D] +``` diff --git a/docs/reference/process/directives/hints.mdx b/docs/reference/process/directives/hints.mdx new file mode 100644 index 0000000000..ec0d9366ef --- /dev/null +++ b/docs/reference/process/directives/hints.mdx @@ -0,0 +1,35 @@ +--- +title: hints +description: Reference for the `hints` process directive. +--- + +# `hints` + +The `hints` directive specifies executor-specific hints as key-value pairs. Each executor uses the hints it recognizes and ignores the rest. Hint values can be any raw value (i.e., numbers, strings, booleans, lists, and maps). + +Unprefixed keys are available to **every** executor. Any executor that recognizes the key consumes it. Prefixing a key with an executor name (e.g., `awsbatch/...`) restricts the hint to that executor only. For example: + +```nextflow +process hello { + hints consumableResources: ['my-license': 1] + + script: + """ + your_command --here + """ +} +``` + +To restrict a hint to a single executor, prefix the key with the executor name: + +```nextflow +hints 'awsbatch/consumableResources': ['my-license': 1] +``` + +When the same hint is provided both unprefixed and with a matching executor prefix, the prefixed form takes precedence for that executor. + +Calling `hints` multiple times in a process definition accumulates entries, with later calls overwriting entries for the same key. Setting `hints` via configuration (e.g., in `nextflow.config`) replaces the entire map. + +See [Executors][executor-page] to see which hints are recognized by each executor. + +[executor-page]: ../../../executor diff --git a/docs/reference/process/directives/label.mdx b/docs/reference/process/directives/label.mdx new file mode 100644 index 0000000000..e2393cc5f8 --- /dev/null +++ b/docs/reference/process/directives/label.mdx @@ -0,0 +1,32 @@ +--- +title: label +description: Reference for the `label` process directive. +--- + +# `label` + +The `label` directive attaches a custom label to the process. For example: + +```nextflow +process hello { + label 'big_mem' + + script: + """ + your_command --here + """ +} +``` + +A label may contain alphanumeric characters or `_`. It must start and end with an alphabetic character. + +The same label can be applied to multiple processes. Multiple labels can be applied to the same process by using the `label` directive multiple times. + +Process labels are used to apply shared process configuration via `withLabel` selectors. They are not recorded in execution logs, trace reports, or lineage metadata. See [Process selectors][config-process-selectors] for more information. + +:::note +To tag individual task executions for logging and debugging, use [tag](./tag). To tag cloud computing resources for cost tracking, use [resourceLabels](./resource-labels). To attach metadata labels to output files for lineage tracking, use the `label` [output directive][workflow-output-def] in the `output` block. +::: + +[config-process-selectors]: ../../../config#process-selectors +[workflow-output-def]: ../../../workflow#outputs diff --git a/docs/reference/process/directives/machine-type.mdx b/docs/reference/process/directives/machine-type.mdx new file mode 100644 index 0000000000..a04be58a91 --- /dev/null +++ b/docs/reference/process/directives/machine-type.mdx @@ -0,0 +1,26 @@ +--- +title: machineType +description: Reference for the `machineType` process directive. +--- + +# `machineType` + +The `machineType` can be used to specify a predefined Google Compute Platform [machine type](https://cloud.google.com/compute/docs/machine-types) when running using the [Google Batch][google-batch-executor], or when using auto-pools with [Azure Batch][azurebatch-executor]. + +For example: + +```nextflow +process hello { + machineType 'n1-highmem-8' + + script: + """ + your_command --here + """ +} +``` + +See also: [cpus](./cpus), [memory](./memory) + +[azurebatch-executor]: ../../../executor/azure-batch +[google-batch-executor]: ../../../executor/google-batch diff --git a/docs/reference/process/directives/max-errors.mdx b/docs/reference/process/directives/max-errors.mdx new file mode 100644 index 0000000000..aa7781c2be --- /dev/null +++ b/docs/reference/process/directives/max-errors.mdx @@ -0,0 +1,26 @@ +--- +title: maxErrors +description: Reference for the `maxErrors` process directive. +--- + +# `maxErrors` + +The `maxErrors` directive defines the maximum number of task failures allowed for a process when using the `retry` error strategy. For example: + +```nextflow +process hello { + errorStrategy 'retry' + maxErrors 5 + + script: + """ + echo 'do this as that .. ' + """ +} +``` + +In the above example, the run will fail if the `hello` process accrues more than 5 failures across all of its task executions. + +By default, there is no limit. However, the run can still fail if an individual task exceeds the number of retries allowed by the `maxRetries` directive. + +See also: [errorStrategy](./error-strategy), [maxRetries](./max-retries) diff --git a/docs/reference/process/directives/max-forks.mdx b/docs/reference/process/directives/max-forks.mdx new file mode 100644 index 0000000000..be870a6002 --- /dev/null +++ b/docs/reference/process/directives/max-forks.mdx @@ -0,0 +1,23 @@ +--- +title: maxForks +description: Reference for the `maxForks` process directive. +--- + +# `maxForks` + +The `maxForks` directive defines the maximum number of concurrent task executions for a process. For example: + +```nextflow +process hello { + maxForks 1 + + script: + """ + your_command --here + """ +} +``` + +The above example forces the `hello` process to execute tasks sequentially. + +By default, there is no limit. However, the number of concurrent tasks can still be limited globally by the number of CPUs (for local tasks) and the `executor.queueSize` config option. diff --git a/docs/reference/process/directives/max-retries.mdx b/docs/reference/process/directives/max-retries.mdx new file mode 100644 index 0000000000..b8ba91a5d4 --- /dev/null +++ b/docs/reference/process/directives/max-retries.mdx @@ -0,0 +1,26 @@ +--- +title: maxRetries +description: Reference for the `maxRetries` process directive. +--- + +# `maxRetries` + +The `maxRetries` directive defines the maximum number of times a task can be retried when using the `retry` error strategy. For example: + +```nextflow +process hello { + errorStrategy 'retry' + maxRetries 3 + + script: + """ + echo 'do this as that .. ' + """ +} +``` + +In the above example, the run will fail if any task executed by `hello` fails more than three times. + +By default, only one retry per task is allowed. However, the run can still fail if the total number of failures for the process exceeds the number allowed by the `maxErrors` directive. + +See also: [errorStrategy](./error-strategy), [maxErrors](./max-errors) diff --git a/docs/reference/process/directives/max-submit-await.mdx b/docs/reference/process/directives/max-submit-await.mdx new file mode 100644 index 0000000000..8d674eb560 --- /dev/null +++ b/docs/reference/process/directives/max-submit-await.mdx @@ -0,0 +1,26 @@ +--- +title: maxSubmitAwait +description: Reference for the `maxSubmitAwait` process directive. +--- + +# `maxSubmitAwait` + +The `maxSubmitAwait` directive defines how long a task can remain in submission queue without being executed. Tasks that exceed this duration in the queue will fail. + +It can be used with the `retry` error strategy to re-submit tasks to a different queue or with different resource requirements. For example: + +```nextflow +process hello { + errorStrategy 'retry' + maxSubmitAwait 10.m + maxRetries 3 + queue "${task.submitAttempt==1 ? 'spot-compute' : 'on-demand-compute'}" + + script: + """ + your_command --here + """ +} +``` + +In the above example, each task is submitted to the `spot-compute` queue on the first attempt (`task.submitAttempt==1`). If a task remains in the queue for more than 10 minutes, it fails and is re-submitted to the `on-demand-compute` queue. diff --git a/docs/reference/process/directives/memory.mdx b/docs/reference/process/directives/memory.mdx new file mode 100644 index 0000000000..7a2285188e --- /dev/null +++ b/docs/reference/process/directives/memory.mdx @@ -0,0 +1,34 @@ +--- +title: memory +description: Reference for the `memory` process directive. +--- + +# `memory` + +The `memory` directive defines how much memory is required by each task execution. For example: + +```nextflow +process hello { + memory 2.GB + + script: + """ + your_command --here + """ +} +``` + +The following suffixes can be used to specify memory values: + +- `B`: Bytes +- `KB`: Kilobytes +- `MB`: Megabytes +- `GB`: Gigabytes +- `TB`: Terabytes + +See [MemoryUnit][stdlib-types-memoryunit] for more information. + +See also: [cpus](./cpus), [disk](./disk), [time](./time), [queue](./queue), [dynamic task resources][dynamic-task-resources] + +[dynamic-task-resources]: ../../../process#dynamic-task-resources +[stdlib-types-memoryunit]: ../../stdlib-types/memory-unit diff --git a/docs/reference/process/directives/module.mdx b/docs/reference/process/directives/module.mdx new file mode 100644 index 0000000000..ba861b0062 --- /dev/null +++ b/docs/reference/process/directives/module.mdx @@ -0,0 +1,32 @@ +--- +title: module +description: Reference for the `module` process directive. +--- + +# `module` + +The `module` directive defines the set of [Environment Modules](http://modules.sourceforge.net/) required by each task, if supported by your compute environment. For example: + +```nextflow +process blast { + module 'ncbi-blast/2.2.27' + + script: + """ + blastp -query + """ +} +``` + +Multiple modules can be specified using the `:` separator: + +```nextflow +process blast { + module 'ncbi-blast/2.2.27:t_coffee/10.0:clustalw/2.1' + + script: + """ + blastp -query + """ +} +``` diff --git a/docs/reference/process/directives/penv.mdx b/docs/reference/process/directives/penv.mdx new file mode 100644 index 0000000000..0a17cc715f --- /dev/null +++ b/docs/reference/process/directives/penv.mdx @@ -0,0 +1,25 @@ +--- +title: penv +description: Reference for the `penv` process directive. +--- + +# `penv` + +The `penv` directive defines the parallel environment to use when submitting tasks to the [SGE][sge-executor] resource manager. For example: + +```nextflow +process blast { + cpus 4 + penv 'smp' + executor 'sge' + + script: + """ + blastp -query input_sequence -num_threads ${task.cpus} + """ +} +``` + +Refer to your cluster documentation or your system administrator to determine whether this feature is supported in your environment. + +[sge-executor]: ../../../executor/sge diff --git a/docs/reference/process/directives/pod.mdx b/docs/reference/process/directives/pod.mdx new file mode 100644 index 0000000000..16bf571b7e --- /dev/null +++ b/docs/reference/process/directives/pod.mdx @@ -0,0 +1,222 @@ +--- +title: pod +description: Reference for the `pod` process directive. +--- + +# `pod` + +The `pod` directive defines pod-specific settings, such as environment variables, secrets, and config maps, when using the [Kubernetes][k8s-executor] executor. + +For example: + +```nextflow +process echo { + pod env: 'MESSAGE', value: 'hello world' + + script: + """ + echo $MESSAGE + """ +} +``` + +The above snippet defines an environment variable named `MESSAGE` whose value is `'hello world'`. + +Pod settings can be specified in Nextflow configuration: + +```groovy +// single setting +process.pod = [env: 'MESSAGE', value: 'hello world'] + +// multiple settings +process.pod = [ + [env: 'MESSAGE', value: 'hello world'], + [secret: 'my-secret/key1', mountPath: '/etc/file.txt'] +] +``` + +The following options are available: + +##### `affinity: ` + +Specifies the pod [affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) with the given configuration. + +##### `annotation: '', value: ''` + +*Can be specified multiple times* + +Defines a pod [annotation](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) with the given name and value. + +##### `automountServiceAccountToken: true | false` + +Specifies whether to [automount service account token](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#opt-out-of-api-credential-automounting) into the pod (default: `true`). + +##### `config: '/', mountPath: ''` + +*Can be specified multiple times* + +Mounts a [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) with name and optional key to the given path. If the key is omitted, the path is interpreted as a directory and all entries in the `ConfigMap` are exposed in that path. + +##### `csi: '', mountPath: ''` + +*Can be specified multiple times* + +Mounts a [CSI ephemeral volume](https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#csi-ephemeral-volumes) with the given configuration to the given path. + +##### `emptyDir: , mountPath: ''` + +*Can be specified multiple times* + +Mounts an [emptyDir](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir) with the given configuration to the given path. + +##### `env: '', config: '/'` + +*Can be specified multiple times* + +Defines an environment variable whose value is defined by the given [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) and key. + +##### `env: '', fieldPath: ''` + +*Can be specified multiple times* + +Defines an environment variable whose value is defined by the given [field path](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/#use-pod-fields-as-values-for-environment-variables) value. + +For example, the following pod option: + +```groovy +process.pod = [env: 'MY_NODE_NAME', fieldPath: 'spec.nodeName'] +``` + +Maps to the following pod spec: + +```yaml +env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName +``` + +##### `env: '', secret: '/'` + +*Can be specified multiple times* + +Defines an environment variable whose value is defined by the given [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) and key. + +##### `env: '', value: ''` + +*Can be specified multiple times* + +Defines an environment variable with the given name and value. + +##### `hostPath: '/host/absolute/path', mountPath: ''` + + + +*Can be specified multiple times* + +Allows creating [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) volume and access it with the specified `mountPath` in the pod. + +##### `imagePullPolicy: 'IfNotPresent' | 'Always' | 'Never'` + +Specifies the [image pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used by the pod to pull the container image. + +##### `imagePullSecret: ''` + +Specifies the [image pull secret](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) used to access a private container image registry. + +##### `label: '', value: ''` + +*Can be specified multiple times* + +Defines a pod [label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) with the given name and value. + +##### `nodeSelector: ` + +Specifies the [node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) with the given configuration. + +The configuration can be a map or a string: + +```groovy +// map +process.pod = [nodeSelector: [disktype: 'ssd', cpu: 'intel']] + +// string +process.pod = [nodeSelector: 'disktype=ssd,cpu=intel'] +``` + +##### `priorityClassName: ''` + +Specifies the [priority class name](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) for pods. + +##### `privileged: true | false` + +Specifies whether the pod should run as a *privileged* container (default: `false`). + +##### `runAsUser: ''` + +Specifies the user ID with which to run the container. Shortcut for the `securityContext` option. + +##### `runtimeClassName: ''` + +Specifies the [runtime class](https://kubernetes.io/docs/concepts/containers/runtime-class/). + +##### `schedulerName: ''` + +Specifies which [scheduler](https://kubernetes.io/docs/tasks/extend-kubernetes/configure-multiple-schedulers/#specify-schedulers-for-pods) is used to schedule the container. + +##### `secret: '/', mountPath: ''` + +*Can be specified multiple times* + +Mounts a [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) with name and optional key to the given path. If the key is omitted, the path is interpreted as a directory and all entries in the `Secret` are exposed in that path. + +##### `securityContext: ` + +Specifies the pod [security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) with the given configuration. + +##### `toleration: ` + +*Can be specified multiple times* + +Specifies the pod [toleration](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) with the given configuration. + +The configuration should be a map corresponding to a single toleration rule. For example, the following pod options: + +```groovy +process.pod = [ + [toleration: [key: 'key1', operator: 'Equal', value: 'value1', effect: 'NoSchedule']], + [toleration: [key: 'key1', operator: 'Exists', effect: 'NoSchedule']], +] +``` + +Maps to the following pod spec: + +```yaml +tolerations: + - key: "key1" + operator: "Equal" + value: "value1" + effect: "NoSchedule" + - key: "key1" + operator: "Exists" + effect: "NoSchedule" +``` + +##### `ttlSecondsAfterFinished` + + + +Specifies the [TTL mechanism](https://kubernetes.io/docs/concepts/workloads/controllers/job/#ttl-mechanism-for-finished-jobs) for finished jobs in seconds. Applies to both successful and failed jobs. + +##### `volumeClaim: '', mountPath: '' [, subPath: '', readOnly: true | false]` + +*Can be specified multiple times* + +Mounts a [Persistent volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) with the given name to the given path. + +The `subPath` option can be used to mount a sub-directory of the volume instead of its root. + +The `readOnly` option can be used to mount the volume as read-only (default: `false`) + +[k8s-executor]: ../../../executor/kubernetes diff --git a/docs/reference/process/directives/publish-dir.mdx b/docs/reference/process/directives/publish-dir.mdx new file mode 100644 index 0000000000..09a3397bcf --- /dev/null +++ b/docs/reference/process/directives/publish-dir.mdx @@ -0,0 +1,117 @@ +--- +title: publishDir +description: Reference for the `publishDir` process directive. +--- + +# `publishDir` + +:::note +[Workflow outputs][workflow-output-def] can be used instead of `publishDir`. See [Migrating to workflow outputs][migrating-workflow-outputs] to learn how to migrate existing code. +::: + +The `publishDir` directive publishes matching process output files to a target directory. For example: + +```nextflow +process hello { + publishDir '/data/chunks' + + output: + path 'chunk_*' + + script: + """ + printf 'Hola' | split -b 1 - chunk_ + """ +} +``` + +The above example publishes the `chunk_*` output files into the `/data/chunks` directory. + +Only files that match the declaration in the `output` block are published, not all the outputs of the process. + +The `publishDir` directive can be specified more than once in order to publish output files to different target directories based on different rules. + +By default, files are published via *symbolic link* from the task directory to the target directory. Use the `mode` option to control this behavior: + +```nextflow +process hello { + publishDir '/data/chunks', mode: 'copy', overwrite: false + + output: + path 'chunk_*' + + script: + """ + printf 'Hola' | split -b 1 - chunk_ + """ +} +``` + +:::warning +Output files are published *asynchronously* after the task execution, so they may not be immediately available in the publish directory during the pipeline run. Downstream processes should access output files through the declared process outputs, not the publish directory. +::: + +Available options: + +##### `contentType` + +*Experimental: currently only supported for S3.* + +Allow specifying the media content type of the published file a.k.a. [MIME type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_Types). If set to `true`, the content type is inferred from the file extension (default: `false`). + +##### `enabled` + +Enable or disable the publish rule depending on the boolean value specified (default: `true`). + +##### `failOnError` + + +The default value was changed from `false` to `true` + + +When `true` abort the execution if some file can't be published to the specified target directory or bucket for any cause (default: `true`) + +##### `mode` + +The file publishing method. Can be one of the following values: + +- `'copy'`: Copies the output files into the publish directory. +- `'copyNoFollow'`: Copies the output files into the publish directory without following symlinks ie. copies the links themselves. +- `'link'`: Creates a hard link in the publish directory for each output file. +- `'move'`: Moves the output files into the publish directory. **Note**: this is only supposed to be used for a *terminal* process i.e. a process whose output is not consumed by any other downstream process. +- `'rellink'`: Creates a relative symbolic link in the publish directory for each output file. +- `'symlink'`: Creates an absolute symbolic link in the publish directory for each output file (default). + +##### `overwrite` + +When `true` any existing file in the target directory will be overridden (default: `true` during normal pipeline execution and `false` when pipeline execution is `resumed`). + +##### `path` + +Specifies the directory where files need to be published. **Note**: the syntax `publishDir '/some/dir'` is a shortcut for `publishDir path: '/some/dir'`. + +##### `pattern` + +Specifies a [glob][glob] file pattern that selects which files to publish from the overall set of output files. + +##### `saveAs` + +A closure which, given the name of the file being published, returns the actual file name or a full path where the file is required to be stored. This can be used to rename or change the destination directory of the published files dynamically by using a custom strategy. Return the value `null` from the closure to *not* publish a file. This is useful when the process has multiple output files, but you want to publish only some of them. + +##### `storageClass` + + + +*Experimental: currently only supported for S3.* + +Allow specifying the storage class to be used for the published file. + +##### `tags` + +*Experimental: currently only supported for S3.* + +Allow the association of arbitrary tags with the published file e.g. `tags: [MESSAGE: 'Hello world']`. + +[glob]: https://docs.oracle.com/javase/tutorial/essential/io/fileOps.html#glob +[migrating-workflow-outputs]: ../../../tutorials/workflow-outputs +[workflow-output-def]: ../../../workflow#outputs diff --git a/docs/reference/process/directives/queue.mdx b/docs/reference/process/directives/queue.mdx new file mode 100644 index 0000000000..a8ba5460bf --- /dev/null +++ b/docs/reference/process/directives/queue.mdx @@ -0,0 +1,32 @@ +--- +title: queue +description: Reference for the `queue` process directive. +--- + +# `queue` + +The `queue` directive defines the queue to which tasks should be submitted, for executors that support queues. For example: + +```nextflow +process hello { + queue 'long' + executor 'slurm' + + script: + """ + your_command --here + """ +} +``` + +Some executors can accept multiple queue names as a comma-separated string: + +```nextflow +queue 'short,long,cn-el6' +``` + +However, this is not generally supported by cloud executors such as AWS Batch, Azure Batch, and Google Batch. + +See [Executors][executor-page] to see which executors support this directive. + +[executor-page]: ../../../executor diff --git a/docs/reference/process/directives/resource-labels.mdx b/docs/reference/process/directives/resource-labels.mdx new file mode 100644 index 0000000000..1c32e5a3d4 --- /dev/null +++ b/docs/reference/process/directives/resource-labels.mdx @@ -0,0 +1,57 @@ +--- +title: resourceLabels +description: Reference for the `resourceLabels` process directive. +--- + +# `resourceLabels` + +The `resourceLabels` directive attaches custom name-value pairs to task executions, for executors that support it. For example: + +```nextflow +process hello { + resourceLabels region: 'some-region', user: 'some-username' + + script: + """ + your_command --here + """ +} +``` + +Resource labels are attached to underlying resources such as cloud VMs, and are intended for operational purposes such as cost tracking. They are not recorded in lineage metadata. + +When `resourceLabels` is specified multiple times in the config, only the last setting is used. Additionally, when `resourceLabels` is specified both in the config and the process definition, only the process definition is used. + +As a best practice, define all resource labels in a single config setting: + +```groovy +process { + resourceLabels = [ region: 'some-region', user: 'some-username' ] +} +``` + +Use process selectors (`withName:` or `withLabel:`) to override resource labels for a specific process. + +Resource labels are currently supported by the following executors: + +- [AWS Batch][awsbatch-executor] +- [Azure Batch][azurebatch-executor] +- [Google Cloud Batch][google-batch-executor] +- [Kubernetes][k8s-executor] +- [Seqera executor][seqera-executor] + +:::note +The limits and the syntax of the corresponding executor should be taken into consideration when using resource labels. +::: + + +Resource labels in Azure are added to auto-pools, rather than jobs, in order to facilitate cost analysis. A new pool will be created for each new set of resource labels. Therefore, it is recommended to also set `azure.batch.deletePoolsOnCompletion = true` when using process-specific resource labels. + + +See also: [label](./label) (for shared process configuration), [tag](./tag) (for per-task identification) + +[awsbatch-executor]: ../../../executor/aws-batch +[azurebatch-executor]: ../../../executor/azure-batch +[google-batch-executor]: ../../../executor/google-batch +[k8s-executor]: ../../../executor/kubernetes +[seqera-executor]: ../../../executor/seqera diff --git a/docs/reference/process/directives/resource-limits.mdx b/docs/reference/process/directives/resource-limits.mdx new file mode 100644 index 0000000000..be236996f0 --- /dev/null +++ b/docs/reference/process/directives/resource-limits.mdx @@ -0,0 +1,46 @@ +--- +title: resourceLimits +description: Reference for the `resourceLimits` process directive. +--- + +# `resourceLimits` + + + +The `resourceLimits` directive defines environment-specific limits for task resource requests. + +Resource limits can be specified in a process: + +```nextflow +process hello { + resourceLimits cpus: 24, memory: 768.GB, time: 72.h + + script: + """ + your_command --here + """ +} +``` + +Or in the Nextflow configuration: + +```nextflow +process.resourceLimits = [ + cpus: 24, + memory: 768.GB, + time: 72.h +] +``` + +Resource limits can be defined for the following directives: + +- [cpus](./cpus) +- [disk](./disk) +- [memory](./memory) +- [time](./time) + +When a task resource request exceeds the corresponding limit, the task resources are automatically reduced to comply with these limits before the job is submitted. + +Resource limits are a useful way to prevent tasks with [dynamic resources][dynamic-task-resources] from requesting more resources than can be provided by an executor (e.g. a task requests 32 cores but the largest node in the cluster has 24). + +[dynamic-task-resources]: ../../../process#dynamic-task-resources diff --git a/docs/reference/process/directives/scratch.mdx b/docs/reference/process/directives/scratch.mdx new file mode 100644 index 0000000000..34ebd37081 --- /dev/null +++ b/docs/reference/process/directives/scratch.mdx @@ -0,0 +1,62 @@ +--- +title: scratch +description: Reference for the `scratch` process directive. +--- + +# `scratch` + +The `scratch` directive executes each task in a temporary directory that is local to the compute node. + +This is useful when executing tasks on an executor with a shared filesystem, because it decreases the network overhead of reading and writing files. Only the files declared as process outputs are copied to the pipeline work directory. + +For example: + +```nextflow +process hello { + scratch true + + output: + path 'data_out' + + script: + """ + your_command --here + """ +} +``` + +It can also be specified in the Nextflow configuration: + +```groovy +process.scratch = true +``` + +By default, the `scratch` directive uses the `$TMPDIR` environment variable in the underlying node as the base scratch directory. If `$TMPDIR` is not defined, then it creates a scratch directory using the `mktemp` command. + +Each task creates a subdirectory within the base scratch directory and automatically deletes it upon completion. + +:::note +Cloud-based executors enable `scratch` by default since the pipeline work directory resides in object storage. +::: + +The following values are supported: + +##### `false` + +Do not use a scratch directory. + +##### `true` + +Create a scratch directory in the directory defined by the `$TMPDIR` environment variable, or `$(mktemp /tmp)` if `$TMPDIR` is not set. + +##### `'$YOUR_VAR'` + +Create a scratch directory in the directory defined by the given environment variable, or `$(mktemp /tmp)` if that variable is not set. The value must use single quotes, otherwise the environment variable will be evaluated in the pipeline script context. + +##### `'/my/tmp/path'` + +Create a scratch directory in the specified directory. + +##### `'ram-disk'` + +Create a scratch directory in the RAM disk `/dev/shm/`. diff --git a/docs/reference/process/directives/secret.mdx b/docs/reference/process/directives/secret.mdx new file mode 100644 index 0000000000..b085ed2025 --- /dev/null +++ b/docs/reference/process/directives/secret.mdx @@ -0,0 +1,32 @@ +--- +title: secret +description: Reference for the `secret` process directive. +--- + +# `secret` + +The `secret` directive allows a process to access secrets. + +For example: + +```nextflow +process hello_secret { + secret 'MY_ACCESS_KEY' + secret 'MY_SECRET_KEY' + + script: + """ + your_command --access \$MY_ACCESS_KEY --secret \$MY_SECRET_KEY + """ +} +``` + +Each secret is provided to the task as an environment variable. + +See [Secrets][secrets-page] for more information. + +:::note +Secrets can only be used with the local or grid executors (e.g., Slurm or Grid Engine). Secrets can be used with AWS Batch and Google Batch when launched from Seqera Platform. +::: + +[secrets-page]: ../../../secrets diff --git a/docs/reference/process/directives/shell.mdx b/docs/reference/process/directives/shell.mdx new file mode 100644 index 0000000000..72db612804 --- /dev/null +++ b/docs/reference/process/directives/shell.mdx @@ -0,0 +1,25 @@ +--- +title: shell +description: Reference for the `shell` process directive. +--- + +# `shell` + +The `shell` directive defines a custom shell command for process scripts. By default, script blocks are executed with `/bin/bash -ue`. + +```nextflow +process hello { + shell '/bin/bash', '-euo', 'pipefail' + + script: + """ + your_command --here + """ +} +``` + +It can also be specified in the Nextflow configuration: + +```groovy +process.shell = ['/bin/bash', '-euo', 'pipefail'] +``` diff --git a/docs/reference/process/directives/spack.mdx b/docs/reference/process/directives/spack.mdx new file mode 100644 index 0000000000..8e9ead7881 --- /dev/null +++ b/docs/reference/process/directives/spack.mdx @@ -0,0 +1,27 @@ +--- +title: spack +description: Reference for the `spack` process directive. +--- + +# `spack` + +The `spack` directive defines the set of [Spack](https://spack.io) packages required by each task. For example: + +```nextflow +process hello { + spack 'bwa@0.7.15' + + script: + """ + your_command --here + """ +} +``` + +Nextflow automatically creates a Spack environment for each unique set of packages. + +Multiple packages can be specified separating them with a blank space, e.g. `bwa@0.7.15 fastqc@0.11.5`. + +The `spack` directive also accepts a Spack environment file path or the path of an existing Spack environment. See [Spack][spack-page] for more information. + +[spack-page]: ../../../spack diff --git a/docs/reference/process/directives/stage-in-mode.mdx b/docs/reference/process/directives/stage-in-mode.mdx new file mode 100644 index 0000000000..c4a2455e18 --- /dev/null +++ b/docs/reference/process/directives/stage-in-mode.mdx @@ -0,0 +1,26 @@ +--- +title: stageInMode +description: Reference for the `stageInMode` process directive. +--- + +# `stageInMode` + +The `stageInMode` directive defines how input files are staged into the task work directory. + +The following modes are supported: + +##### `'copy'` + +Input files are staged in the task work directory by creating a copy. + +##### `'link'` + +Input files are staged in the task work directory by creating a hard link for each of them. + +##### `'rellink'` + +Input files are staged in the task work directory by creating a symbolic link with a relative path for each of them. + +##### `'symlink'` + +Input files are staged in the task work directory by creating a symbolic link with an absolute path for each of them (default). diff --git a/docs/reference/process/directives/stage-out-mode.mdx b/docs/reference/process/directives/stage-out-mode.mdx new file mode 100644 index 0000000000..2eb88f1fff --- /dev/null +++ b/docs/reference/process/directives/stage-out-mode.mdx @@ -0,0 +1,36 @@ +--- +title: stageOutMode +description: Reference for the `stageOutMode` process directive. +--- + +# `stageOutMode` + +The `stageOutMode` directive defines how output files are staged out from the scratch directory to the task work directory. + +The following modes are supported: + +##### `'copy'` + +Output files are copied from the scratch directory to the work directory. + +##### `'fcp'` + + + +Output files are copied from the scratch directory to the work directory by using the [fcp](https://github.com/Svetlitski/fcp) utility (note: it must be available in the task environment). + +##### `'move'` + +Output files are moved from the scratch directory to the work directory. + +##### `'rclone'` + + + +Output files are copied from the scratch directory to the work directory by using the [rclone](https://rclone.org) utility (note: it must be available in the task environment). + +##### `'rsync'` + +Output files are copied from the scratch directory to the work directory by using the `rsync` utility. + +See also: [scratch](./scratch) diff --git a/docs/reference/process/directives/store-dir.mdx b/docs/reference/process/directives/store-dir.mdx new file mode 100644 index 0000000000..a76db4de19 --- /dev/null +++ b/docs/reference/process/directives/store-dir.mdx @@ -0,0 +1,40 @@ +--- +title: storeDir +description: Reference for the `storeDir` process directive. +--- + +# `storeDir` + +The `storeDir` directive stores task outputs in a permanent *store directory* instead of the work directory. + +On subsequent runs, each task is executed only if the declared output files do not exist in the store directory. When the files are present, the task is skipped and these files are used as the task outputs. + +The following example shows how to use the `storeDir` directive to create a directory containing a BLAST database for each species specified by an input parameter: + +```nextflow +process make_blast_db { + storeDir '/db/genomes' + + input: + path species + + output: + path "${dbName}.*" + + script: + dbName = species.baseName + """ + makeblastdb -dbtype nucl -in ${species} -out ${dbName} + """ +} +``` + +Caveats: + +- The `env`, `eval`, and `stdout` output qualifiers cannot be used with `storeDir` because they rely on helper files in the task directory. Use `path` outputs instead. + +- If a process uses `storeDir` and all of its outputs are optional, the process will always be skipped, even if the store directory is empty. Avoid this issue by specifying at least one required file output. + +- The `storeDir` directive is not a replacement for publishing outputs. Use the [publishDir](./publish-dir) directive or [workflow outputs][workflow-output-def] instead. + +[workflow-output-def]: ../../../workflow#outputs diff --git a/docs/reference/process/directives/tag.mdx b/docs/reference/process/directives/tag.mdx new file mode 100644 index 0000000000..f3aeab3ae5 --- /dev/null +++ b/docs/reference/process/directives/tag.mdx @@ -0,0 +1,50 @@ +--- +title: tag +description: Reference for the `tag` process directive. +--- + +# `tag` + +The `tag` directive defines a custom identifier for each task execution. For example: + +```nextflow +process hello { + tag "$code" + + input: + val code + + script: + """ + echo $code + """ +} + +workflow { + ch_codes = channel.of('alpha', 'gamma', 'omega') + hello(ch_codes) +} +``` + +The above example logs each task with its corresponding tag: + +``` +[6e/28919b] Submitted process > hello (alpha) +[d2/1c6175] Submitted process > hello (gamma) +[1c/3ef220] Submitted process > hello (omega) +``` + +Tags are a useful way to track related tasks in a pipeline run. Tasks can be identified by tag in the [execution log][execution-log] and the [trace report][trace-report]. + +:::note +The name of a task in both [reports][tracing-page] and [lineage][data-lineage-page] is defined as ` ()`. +::: + +:::note +The `tag` directive is not related to the [label](./label) directive. Process labels are only used for shared process configuration, not for tracking. +::: + +[data-lineage-page]: ../../../tutorials/data-lineage +[execution-log]: ../../../reports#execution-log +[trace-report]: ../../../reports#trace-file +[tracing-page]: ../../../reports diff --git a/docs/reference/process/directives/time.mdx b/docs/reference/process/directives/time.mdx new file mode 100644 index 0000000000..3f27539f5e --- /dev/null +++ b/docs/reference/process/directives/time.mdx @@ -0,0 +1,39 @@ +--- +title: time +description: Reference for the `time` process directive. +--- + +# `time` + +The `time` directive defines the maximum runtime for each task. For example: + +```nextflow +process hello { + time 1.h + + script: + """ + your_command --here + """ +} +``` + +The following suffixes can be used to specify duration values: + +- `ms`: milliseconds +- `s`: seconds +- `m`: minutes +- `h`: hours +- `d`: days + +See [Duration][stdlib-types-duration] for more information. + +:::note +This directive is only used by certain executors. Refer to the [Executors][executor-page] page to see which executors support this directive. +::: + +See also: [cpus](./cpus), [disk](./disk), [memory](./memory), [queue](./queue), [dynamic task resources][dynamic-task-resources] + +[dynamic-task-resources]: ../../../process#dynamic-task-resources +[executor-page]: ../../../executor +[stdlib-types-duration]: ../../stdlib-types/duration diff --git a/docs/reference/process/inputs-outputs-typed.mdx b/docs/reference/process/inputs-outputs-typed.mdx new file mode 100644 index 0000000000..d9ad1f7af1 --- /dev/null +++ b/docs/reference/process/inputs-outputs-typed.mdx @@ -0,0 +1,88 @@ +--- +title: Inputs and outputs (typed) +description: Reference for the stage directives and output functions available in a typed process. +--- + +# Inputs and outputs (typed) + + + +:::note +Typed processes require the `nextflow.enable.types` feature flag to be enabled in every script that uses them. +::: + +### Stage directives + +The following directives can be used in the `stage:` section of a typed process: + +##### `env( name: String, String value )` + +Declares an environment variable with the specified name and value in the task environment. + +##### `stageAs( value: Path, filePattern: String )` + +Stages a file into the task directory under the given alias. + +##### `stageAs( value: Iterable, filePattern: String )` + +Stages a collection of files into the task directory under the given alias. + +##### `stdin( value: String )` + +Stages the given value as the standard input (i.e., `stdin`) to the task script. + +### Outputs + +The following functions are available in the `output:` and `topic:` sections of a typed process: + +##### `env( name: String ) -> String` + +Returns the value of an environment variable from the task environment. + +##### `eval( command: String ) -> String` + +Returns the standard output of the specified command, which is executed in the task environment after the task script completes. + +##### `file( pattern: String, [options] ) -> Path` + +Returns a file from the task environment that matches the specified pattern. + +Available options: + +###### `followLinks: Boolean` + +When `true`, target files are returned in place of any matching symlink (default: `true`). + +###### `glob: Boolean` + +When `true`, the file name is interpreted as a glob pattern (default: `true`). + +###### `hidden: Boolean` + +When `true`, hidden files are included in the matching output files (default: `false`). + +###### `includeInputs: Boolean` + +When `true` and the file name is a glob pattern, any input files matching the pattern are also included in the output (default: `false`). + +###### `maxDepth: Integer` + +Maximum number of directory levels to visit (default: no limit). + +###### `optional: Boolean` + +When `true`, the task will not fail if the given file is missing (default: `false`). + +###### `type: String` + +Type of paths returned, either `file`, `dir` or `any` (default: `any`, or `file` if the given file name contains a double star (`**`)). + +##### `files( pattern: String, [options] ) -> Set` + +Returns files from the task environment that match the given pattern. + +Supports the same options as `file()` (except for `optional`). + +##### `stdout() -> String` + +Returns the standard output of the task script. diff --git a/docs/reference/process/inputs-outputs.mdx b/docs/reference/process/inputs-outputs.mdx new file mode 100644 index 0000000000..774f544b24 --- /dev/null +++ b/docs/reference/process/inputs-outputs.mdx @@ -0,0 +1,165 @@ +--- +title: Inputs and outputs (legacy) +description: Reference for the input and output qualifiers available in a legacy process. +--- + +# Inputs and outputs (legacy) + +### Inputs + +##### `val( identifier )` + +Declare a variable input. The received value can be any type, and it will be made available to the process body (i.e. `script`, `shell`, `exec`) as a variable given by `identifier`. + +##### `file( identifier | stageName )` + + +Use `path` instead. + + +Declare a file input. The received value can be any type, and it will be staged into the task directory. If the received value is not a file or collection of files, it is implicitly converted to a string and written to a file. + +The argument can be an identifier or string. If an identifier, the received value will be made available to the process body as a variable. If a string, the received value will be staged into the task directory under the given alias. + +##### `path( identifier | stageName )` + +Declare a file input. The received value should be a file or collection of files and will be staged into the task directory. + +:::tip +See [Multiple input files][process-multiple-input-files] for more information about accepting collections of files. +::: + +The argument can be an identifier or string. If an identifier, the received value will be made available to the process body as a variable. If a string, the received value will be staged into the task directory under the given alias. + +Available options: + +###### `arity` + + + +Specify the number of expected files. Can be a number, e.g. `'1'`, or a range, e.g. `'1..*'`. If a task receives an invalid number of files for this `path` input, it will fail. + +###### `name` + +Specify how the file should be named in the task work directory. Can be a name or a pattern. + +###### `stageAs` + +Alias of `name`. + +##### `env( name )` + +Declare an environment variable input. The received value should be a string, and it will be exported to the task environment as an environment variable given by `name`. + +##### `stdin` + +Declare a `stdin` input. The received value should be a string, and it will be provided as the standard input (i.e. `stdin`) to the task script. It should be declared only once for a process. + +##### `tuple( arg1, arg2, ... )` + +Declare a tuple input. Each argument should be an input declaration such as `val`, `path`, `env`, or `stdin`. + +The received value should be a tuple with the same number of elements as the `tuple` declaration, and each received element should be compatible with the corresponding `tuple` argument. Each tuple element is treated the same way as if it were a standalone input. + +### Outputs + +##### `val( value )` + +Declare a variable output. The argument can be any value, and it can reference any output variables defined in the process body (i.e. variables declared without the `def` keyword). + +##### `file( pattern )` + + +Use `path` instead. + + +Declare a file output. It receives the output files from the task environment that match the given pattern. + +Multiple patterns can be specified using the colon separator (`:`). The union of all files matched by each pattern will be collected. + +##### `path( pattern, [options] )` + +Declare a file output. It receives the output files from the task environment that match the given pattern. + +Available options: + +###### `arity` + + + +Specify the number of expected files. Can be a number or a range. If a task produces an invalid number of files for this `path` output, it will fail. + +If the arity is `1`, a single file will be emitted. Otherwise, a list will always be emitted, even if only one file is produced. + +:::warning +If the arity is not specified, a single file or list will be emitted based on whether a single file or multiple files are produced at runtime, resulting potentially in an output channel with a mixture of files and file collections. +::: + +###### `followLinks` + +When `true`, target files are returned in place of any matching symlink (default: `true`). + +###### `glob` + +When `true`, the specified name is interpreted as a glob pattern (default: `true`). + +###### `hidden` + +When `true`, hidden files are included in the matching output files (default: `false`). + +###### `includeInputs` + +When `true` and the output path is a glob pattern, any input files matching the pattern are also included in the output (default: `false`). + +###### `maxDepth` + +Maximum number of directory levels to visit (default: no limit). + +###### `type` + +Type of paths returned, either `file`, `dir` or `any` (default: `any`, or `file` if the specified file name pattern contains a double star (`**`)). + +##### `env( name )` + +Declare an environment variable output. It receives the value of the environment variable (given by `name`) from the task environment. + + +Prior to this version, if the environment variable contained multiple lines of output, the output would be compressed to a single line by converting newlines to spaces. + + +##### `stdout` + +Declare a `stdout` output. It receives the standard output of the task script. + +##### `eval( command )` + + + +Declare an `eval` output. It receives the standard output of the given command, which is executed in the task environment after the task script. + +If the command fails, the task will also fail. + +##### `tuple( arg1, arg2, ... )` + +Declare a tuple output. Each argument should be an output declaration such as `val`, `path`, `env`, `stdin`, or `eval`. Each tuple element is treated the same way as if it were a standalone output. + +### Generic options + +The following options are available for all process outputs: + +##### `emit: ` + +Defines the name of the output channel. + +##### `optional: true | false` + +When `true`, the task will not fail if the specified output is missing (default: `false`). + +##### `topic: ` + + + +Send the output to a [topic channel][channel-topic] with the given name. + +[channel-topic]: ../stdlib-namespaces/channel#topic +[process-multiple-input-files]: ../../process#multiple-input-files diff --git a/docs/reference/process/task-properties.mdx b/docs/reference/process/task-properties.mdx new file mode 100644 index 0000000000..3654f26e20 --- /dev/null +++ b/docs/reference/process/task-properties.mdx @@ -0,0 +1,73 @@ +--- +title: Task properties +description: Reference for the task properties available in a process body. +--- + +# Task properties + +The following task properties are defined in the process body: + +##### `task.attempt` + +The current task attempt. + +##### `task.exitStatus` + +*Available only in `script:` and `shell:` blocks* + +The exit code returned by the task script. + +The exit code is only available after the task has been executed (e.g., the [errorStrategy](./directives/error-strategy) directive). + +##### `task.hash` + +*Available only in `exec:` blocks* + +The task hash. + +##### `task.index` + +The process-level task index. + +##### `task.name` + +*Available only in `exec:` blocks* + +The task name. + +##### `task.previousException` + + + +The exception reported by the previous task attempt. + +Since the exception is available after a failed task attempt, it can only be accessed when retrying a failed task execution, i.e., when `task.attempt` is greater than 1. + +##### `task.previousTrace` + + + +The trace record associated with the previous task attempt. + +Since the trace record is available after a failed task attempt, it can only be accessed when retrying a failed task execution, i.e., when `task.attempt` is greater than 1. See [Trace file][trace-report] for a list of available fields. + +:::note +The trace fields `%cpu` and `%mem` can be accessed as `pcpu` and `pmem`, respectively. +::: + +##### `task.process` + +The name of the process that spawned the task. + +##### `task.workDir` + +*Available only in `exec:` blocks* + +The unique directory path for the task. + +:::note +[Directive values](./directives) for a task can be accessed via `task.`. See [Task directive values][task-directive-values] for more information. +::: + +[task-directive-values]: ../../process#using-task-directive-values +[trace-report]: ../../reports#trace-file diff --git a/docs/reference/semantics.mdx b/docs/reference/semantics.mdx index 774259d8c4..105d4d56af 100644 --- a/docs/reference/semantics.mdx +++ b/docs/reference/semantics.mdx @@ -1026,7 +1026,7 @@ A value is also cast *implicitly* in the following situations: - **String coercion** -- when a value is interpolated into a [string](#strings), it is converted to a string with its `toString()` method. [operator-page]: ./operator -[process-error-strategy]: ./process#errorstrategy +[process-error-strategy]: ./process/directives/error-strategy [process-page]: ../process [script-page]: ../script [stdlib-error]: ./stdlib-namespaces/global diff --git a/docs/reference/stdlib-namespaces/workflow.mdx b/docs/reference/stdlib-namespaces/workflow.mdx index 58a250c621..d458145830 100644 --- a/docs/reference/stdlib-namespaces/workflow.mdx +++ b/docs/reference/stdlib-namespaces/workflow.mdx @@ -191,4 +191,4 @@ Define an action to take when the workflow completes (whether successful or not) Define an action to take if the workflow is terminated due to a runtime error or task failure. [config-manifest]: ../config/manifest -[process-error-strategy]: ../process#errorstrategy +[process-error-strategy]: ../process/directives/error-strategy diff --git a/docs/reference/syntax.mdx b/docs/reference/syntax.mdx index 1ce5503e24..f5abac769e 100644 --- a/docs/reference/syntax.mdx +++ b/docs/reference/syntax.mdx @@ -958,7 +958,7 @@ See [strict syntax][strict-syntax-page] for more information. [operator-page]: ./operator [process-page]: ../process [process-reference]: ./process -[inputs-and-outputs-typed]: ./process#inputs-and-outputs-typed +[inputs-and-outputs-typed]: ./process/inputs-outputs-typed [process-typed]: ../process-typed [stdlib-types-iterable]: ./stdlib-types/iterable [strict-syntax-page]: ../strict-syntax diff --git a/docs/secrets.mdx b/docs/secrets.mdx index f9c6f725a8..0582e77d48 100644 --- a/docs/secrets.mdx +++ b/docs/secrets.mdx @@ -112,4 +112,4 @@ Secrets are made available as environment variables in the process script. To pr ::: [cli-secrets]: ./reference/cli/secrets -[process-secret]: ./reference/process#secret \ No newline at end of file +[process-secret]: ./reference/process/directives/secret \ No newline at end of file diff --git a/docs/spack.mdx b/docs/spack.mdx index ddb4b6e188..ad3afeabf1 100644 --- a/docs/spack.mdx +++ b/docs/spack.mdx @@ -178,6 +178,6 @@ Spack advanced configuration settings are described in the [spack][config-spack] [config-profiles]: ./config#config-profiles [config-spack]: ./reference/config/spack -[process-arch]: ./reference/process#arch -[process-spack]: ./reference/process#spack +[process-arch]: ./reference/process/directives/arch +[process-spack]: ./reference/process/directives/spack [process-stub]: ./process#stub diff --git a/docs/tutorials/static-types.mdx b/docs/tutorials/static-types.mdx index 2c912b5929..d2abf41c1f 100644 --- a/docs/tutorials/static-types.mdx +++ b/docs/tutorials/static-types.mdx @@ -719,7 +719,7 @@ See the following links to learn more about static typing: [operator-combine]: ../reference/operator#combine [operator-page]: ../reference/operator [process-input-each]: ../process#input-repeaters-each -[process-reference-typed]: ../reference/process#inputs-and-outputs-typed +[process-reference-typed]: ../reference/process/inputs-outputs-typed [process-typed-page]: ../process-typed [rnaseq-nf-page]: ./rnaseq-nf [script-records]: ../script#records diff --git a/docs/tutorials/workflow-outputs.mdx b/docs/tutorials/workflow-outputs.mdx index 33e40e331c..7e8f3c1055 100644 --- a/docs/tutorials/workflow-outputs.mdx +++ b/docs/tutorials/workflow-outputs.mdx @@ -303,7 +303,7 @@ In the future, if you add a tool with per-sample outputs, you only need to join [config-process-selectors]: ../config#process-selectors [migrating-static-types]: ./static-types -[process-publishdir]: ../reference/process#publishdir +[process-publishdir]: ../reference/process/directives/publish-dir [rnaseq-nf-page]: ./rnaseq-nf [workflow-output-def]: ../workflow#outputs [workflow-outputs-final]: ./workflow-outputs diff --git a/docs/wave.mdx b/docs/wave.mdx index d500eae79f..4a3fcce442 100644 --- a/docs/wave.mdx +++ b/docs/wave.mdx @@ -246,4 +246,4 @@ Wave advanced configuration settings are described in the [Wave][config-wave] se [config-wave]: ./reference/config/wave [fusion-page]: ./fusion [module-directory]: ./modules#local-modules -[process-conda]: ./reference/process#conda +[process-conda]: ./reference/process/directives/conda diff --git a/docs/workflow.mdx b/docs/workflow.mdx index b01a9fe17a..125a7524c8 100644 --- a/docs/workflow.mdx +++ b/docs/workflow.mdx @@ -778,7 +778,7 @@ Workflows can also be invoked recursively: [process-naming-outputs]: ./process#naming-outputs [process-out-singleton]: ./process#singleton-outputs [process-page]: ./process -[process-publishdir]: ./reference/process#publishdir +[process-publishdir]: ./reference/process/directives/publish-dir [stdlib-types-value]: ./reference/stdlib-types/value [syntax-include]: ./reference/syntax#include [workflow-outputs-first-preview]: ./migrations/24-04#workflow-outputs-first-preview