diff --git a/adr/module-spec-schema.json b/adr/module-spec-schema.json index 36e2d2d7f1..80ff56b253 100644 --- a/adr/module-spec-schema.json +++ b/adr/module-spec-schema.json @@ -9,7 +9,7 @@ "type": "string", "description": "Module name. Can be a simple identifier (e.g., 'fastqc', 'bwa_mem') for local/nf-core modules, or a fully qualified scoped name (e.g., 'nf-core/fastqc', 'myorg/custom') for registry modules.", "examples": ["fastqc", "bwa_mem", "nf-core/fastqc", "myorg/salmon-quant"], - "pattern": "^([a-z0-9][a-z0-9-]*/)?[a-z][a-z0-9_-]*$" + "pattern": "^([a-z0-9][a-z0-9._-]*/)?[a-z][a-z0-9._-]*(/[a-z][a-z0-9._-]*)*$" }, "version": { "type": "string", @@ -17,6 +17,11 @@ "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(-[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?(\\+[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?$", "examples": ["1.0.0", "2.1.3", "1.0.0-beta.1"] }, + "kind": { + "type": "string", + "description": "Module kind: 'Process' (standalone process, default when absent) or 'Workflow' (standalone workflow / subworkflow). Metadata only; does not affect storage location.", + "enum": ["Process", "Workflow"] + }, "description": { "type": "string", "description": "Brief description of what the module does", @@ -64,6 +69,15 @@ "description": "Nextflow version constraint using comparison operators", "examples": [">=24.04.0", ">=24.04.0,<25.0.0"], "pattern": "^[<>=!]+[0-9]+\\.[0-9]+\\.[0-9]+(-[a-zA-Z0-9]+)?(,\\s*[<>=!]+[0-9]+\\.[0-9]+\\.[0-9]+(-[a-zA-Z0-9]+)?)*$" + }, + "modules": { + "type": "array", + "description": "Direct module dependencies (workflow modules): 'scope/name@' references pinned to an exact semver, including the git-sha pre-release form (e.g. 0.0.0-4e3e10e). Version constraints/ranges are NOT supported — under nested per-module vendoring each dependency is installed at its pinned version in isolation.", + "items": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*/[a-z0-9][a-z0-9_/-]*@(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(-[0-9A-Za-z-.]+)?$" + }, + "uniqueItems": true } }, "additionalProperties": false @@ -218,7 +232,7 @@ }, "type": { "type": "string", - "description": "Data type of the parameter value", + "description": "Data type of the parameter value. The generic tags 'channel' (a typed workflow channel, e.g. Channel) and 'custom-record' (a user-defined record type) are documentation only for statically-typed processes/workflows -- the authoritative types are declared in the source take:/emit: and typed input/output, and module run parses them from the code.", "enum": [ "boolean", "float", @@ -227,7 +241,9 @@ "list", "map", "file", - "directory" + "directory", + "channel", + "custom-record" ] }, "description": { diff --git a/docs/modules/developing-modules.mdx b/docs/modules/developing-modules.mdx index 224528cdca..283ce8ecd3 100644 --- a/docs/modules/developing-modules.mdx +++ b/docs/modules/developing-modules.mdx @@ -27,6 +27,12 @@ The command creates a module directory with the following files: - `meta.yml`: The module spec describing metadata, inputs, and outputs. - `README.md`: Documentation for the module. +By default this scaffolds a *process* module. Use `-kind Workflow` to scaffold a *workflow* module, and `-typed` to generate statically-typed declarations. A typed workflow module can be executed directly with `nextflow module run`; an untyped one can only be included in a pipeline: + +```console +$ nextflow module create myorg/my-workflow -kind Workflow -typed +``` + See [module create][cli-module-create] for the full command reference. ## Module structure @@ -185,6 +191,11 @@ $ nextflow module spec \ When updating an existing module spec, it is incorporated into the new file. +For a workflow module, the `input` and `output` sections are derived from the workflow's `take:` and +`emit:` declarations. Types are inferred when the workflow is statically typed; for an untyped workflow +the parameter names are captured but the types are left as `TODO` placeholders, since they cannot be +inferred from the source. + See [module spec][cli-module-spec] for the full command reference. ## Validating a module diff --git a/docs/modules/using-modules.mdx b/docs/modules/using-modules.mdx index 81d7313846..9c8733062f 100644 --- a/docs/modules/using-modules.mdx +++ b/docs/modules/using-modules.mdx @@ -148,6 +148,22 @@ Use the `-force` flag to overwrite local changes: $ nextflow module install nf-core/fastqc -version 0.0.0-c9h0bv4 -force ``` +### Updating a workflow module's dependencies + +A workflow module declares its dependencies in `meta.yml` (`requires.modules`), and they are vendored +under the module's own nested `modules/` directory. If you edit the declared dependency versions of an +already-installed module by hand, use `-update-deps` to re-vendor them to match the edited `meta.yml`, +without reinstalling the module itself: + +```console +$ nextflow module install nf-core/my-workflow -update-deps +``` + +This installs newly declared dependencies, updates changed versions, and removes dependencies that are no +longer declared. A vendored dependency with local modifications is not overwritten or removed; an error is +raised instead. The module itself is left untouched, so its local (unpublished) modification status is +preserved. The flag is ignored if the module is not installed, and cannot be combined with `-force`. + ## Removing modules Use the `module remove` command to uninstall a module from your project: diff --git a/docs/reference/cli.mdx b/docs/reference/cli.mdx index 2e632eefab..ba490c0da7 100644 --- a/docs/reference/cli.mdx +++ b/docs/reference/cli.mdx @@ -1337,10 +1337,20 @@ The `module` command provides a comprehensive system for managing registry-based #### Subcommands -##### `create [namespace/name]` {#module-create} +##### `create [options] [namespace/name]` {#module-create} Create a new module with a basic `main.nf`, `meta.yml`, and `README.md`. +The following options are available: + +###### `-kind` (`Process`) + +The kind of module to scaffold: `Process` (default) or `Workflow`. + +###### `-typed` + +Scaffold a statically-typed module: adds `nextflow.enable.types = true` and typed `input`/`output` (process) or `take`/`emit` (workflow) declarations using basic types. A typed workflow module can be executed directly with `nextflow module run`; an untyped workflow module can only be included in a pipeline. + ##### `install [options] [namespace/name]` {#module-install} Install a module from the registry into your project. @@ -1360,7 +1370,7 @@ Force reinstall even if the module exists locally with modifications. Without th ##### `list [options]` {#module-list} List all modules currently installed in your project. -Shows each module's name, version, and integrity status (whether it has been modified locally). +Shows each module's name, version, kind (`Process` or `Workflow`), and integrity status (whether it has been modified locally). The following options are available: diff --git a/modules/nextflow/build.gradle b/modules/nextflow/build.gradle index 966f1f440f..6223f61fa6 100644 --- a/modules/nextflow/build.gradle +++ b/modules/nextflow/build.gradle @@ -71,8 +71,8 @@ dependencies { api 'io.seqera:lib-trace:0.1.0' api 'com.fasterxml.woodstox:woodstox-core:7.1.1' api 'org.apache.commons:commons-compress:1.27.1' // For tar.gz extraction - api 'io.seqera:npr-api:0.22.0' - api 'io.seqera:npr-client:0.22.0' + api 'io.seqera:npr-api:0.24.10' + api 'io.seqera:npr-client:0.24.10' api 'com.networknt:json-schema-validator:1.5.6' testImplementation 'org.subethamail:subethasmtp:3.1.7' diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleCreate.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleCreate.groovy index 6a6cf29596..bee679f215 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleCreate.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleCreate.groovy @@ -38,6 +38,12 @@ class CmdModuleCreate extends CmdBase { @Parameter(description = "[namespace/name]") List args + @Parameter(names = ['-kind'], description = "Module kind: Process (default) or Workflow") + String kind + + @Parameter(names = ['-typed'], description = "Generate a statically-typed module (usable directly with `nextflow module run`)", arity = 0) + boolean typed + @Override String getName() { return 'create' @@ -86,8 +92,18 @@ class CmdModuleCreate extends CmdBase { validateSegment('namespace', namespace) validateSegments('name', name) - createModule(namespace, name) + createModule(namespace, name, normalizeKind(kind), typed) + } + + static private String normalizeKind(String value) { + if( !value ) + return 'Process' + final k = value.toLowerCase().capitalize() + if( k != 'Process' && k != 'Workflow' ) + throw new AbortOperationException("Invalid module kind '${value}' -- must be 'Process' or 'Workflow'") + return k } + static private void validateSegment(String field, String value) { if( !value.matches('[a-zA-Z0-9][a-zA-Z0-9._\\-]*') ) throw new AbortOperationException("Invalid module $field '${value}' -- only alphanumeric characters, hyphens, underscores and dots are allowed, and must start with an alphanumeric character") @@ -103,7 +119,7 @@ class CmdModuleCreate extends CmdBase { return Path.of('modules') } - protected void createModule(String namespace, String name) { + protected void createModule(String namespace, String name, String kind = 'Process', boolean typed = false) { final moduleDir = modulesBase().resolve(namespace).resolve(name) if( Files.exists(moduleDir) ) throw new AbortOperationException("Module directory already exists: $moduleDir") @@ -112,22 +128,31 @@ class CmdModuleCreate extends CmdBase { Files.createDirectories(moduleDir) // create main.nf - moduleDir.resolve('main.nf').text = mainNf(namespace, name) + moduleDir.resolve('main.nf').text = mainNf(namespace, name, kind, typed) // create README.md moduleDir.resolve('README.md').text = readmeMd(namespace, name) // create meta.yml - moduleDir.resolve('meta.yml').text = metaYml(namespace, name) + moduleDir.resolve('meta.yml').text = metaYml(namespace, name, kind, typed) // create .module-info so it's recognised as a Nextflow managed module Files.createFile(moduleDir.resolve(ModuleInfo.MODULE_INFO_FILE)) + final defName = name.replaceAll('[^a-zA-Z0-9_]', '_').toUpperCase() println "Module created successfully at path: $moduleDir" println "" - println "To run the module:" - println "" - println " nextflow module run $namespace/$name --greeting 'Hello world!'" + // an untyped workflow module cannot be run directly (`module run` requires typed take:/emit:) + if( kind == 'Workflow' && !typed ) { + println "Include the workflow module in a pipeline:" + println "" + println " include { $defName } from '$namespace/$name'" + } + else { + println "To run the module:" + println "" + println " nextflow module run $namespace/$name --greeting 'Hello world!'" + } } static private String readLine() { @@ -137,13 +162,84 @@ class CmdModuleCreate extends CmdBase { : new BufferedReader(new InputStreamReader(System.in)).readLine() } - static String mainNf(String namespace, String name) { - """\ + static String mainNf(String namespace, String name, String kind = 'Process', boolean typed = false) { + final defName = name.replaceAll('[^a-zA-Z0-9_]', '_').toUpperCase() + + if( kind == 'Workflow' && typed ) { + return """\ + /* + * Workflow module: ${namespace}/${name} + * TODO: rename the workflow, replace the example take/emit and types, and implement the logic. + */ + + nextflow.enable.types = true + + workflow ${defName} { + take: + greeting: String + + main: + // TODO: implement the workflow logic + message = greeting + + emit: + result: String = message + } + """.stripIndent() + } + + if( kind == 'Workflow' ) { + return """\ + /* + * Workflow module: ${namespace}/${name} + * TODO: rename the workflow, replace the example take/emit, and implement the logic. + */ + + workflow ${defName} { + take: + ch_input + + main: + // TODO: implement the workflow logic + ch_output = ch_input + + emit: + output = ch_output + } + """.stripIndent() + } + + if( typed ) { + return """\ + /* + * Module: ${namespace}/${name} + * TODO: rename the process, replace the example input/output and types, and implement the script. + */ + + nextflow.enable.types = true + + process ${defName} { + input: + greeting: String + + output: + message: String = stdout() + + script: + \"\"\" + echo '\${greeting}' + \"\"\" + } + """.stripIndent() + } + + return """\ /* * Module: ${namespace}/${name} + * TODO: rename the process, replace the example input/output, and implement the script. */ - process ${name.replaceAll('[^a-zA-Z0-9_]', '_').toUpperCase()} { + process ${defName} { input: val greeting @@ -158,8 +254,69 @@ class CmdModuleCreate extends CmdBase { """.stripIndent() } - static String metaYml(String namespace, String name) { - """\ + static String metaYml(String namespace, String name, String kind = 'Process', boolean typed = false) { + if( kind == 'Workflow' && typed ) { + // typed workflow: derive input/output from the scaffold's take:/emit: + // (typed workflows require Nextflow 26.04.0+) + return """\ + name: ${namespace}/${name} + version: 1.0.0 + kind: Workflow + description: A brief description of the ${namespace}/${name} workflow module + license: Apache-2.0 + requires: + nextflow: ">=26.04.0" + input: + - name: greeting + type: string + description: A greeting string + output: + - name: result + type: string + description: The greeting message + """.stripIndent() + } + if( kind == 'Workflow' ) { + // untyped workflow: take/emit have no declared types, but the generated scaffold's + // take/emit are channels, so the interface is documented with the generic channel type + return """\ + name: ${namespace}/${name} + version: 1.0.0 + kind: Workflow + description: A brief description of the ${namespace}/${name} workflow module + license: Apache-2.0 + requires: + nextflow: ">=24.04.0" + input: + - name: ch_input + type: channel + description: The input channel + output: + - name: output + type: channel + description: The output channel + """.stripIndent() + } + if( typed ) { + // typed process (requires Nextflow 25.10.0+) + return """\ + name: ${namespace}/${name} + version: 1.0.0 + description: A brief description of the ${namespace}/${name} module + license: Apache-2.0 + requires: + nextflow: ">=25.10.0" + input: + - name: greeting + type: string + description: A greeting string + output: + - name: message + type: string + description: The greeting message + """.stripIndent() + } + return """\ name: ${namespace}/${name} version: 1.0.0 description: A brief description of the ${namespace}/${name} module diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy index 6ebbcec4c8..fd28cc641f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy @@ -51,6 +51,9 @@ class CmdModuleInstall extends CmdBase { @Parameter(names = ["-force"], description = "Force reinstall even if already installed", arity = 0) boolean force = false + @Parameter(names = ["-update-deps"], description = "For an already-installed module, update its vendored dependencies to match meta.yml, without reinstalling the module (ignored if the module is not installed)", arity = 0) + boolean updateDeps = false + @Parameter(description = "[scope/name]", required = true) List args @@ -71,6 +74,10 @@ class CmdModuleInstall extends CmdBase { throw new AbortOperationException("Incorrect number of arguments") } + if( updateDeps && force ) { + throw new AbortOperationException("Options -update-deps and -force cannot be used together") + } + def reference = ModuleReference.parse(args[0]) // Get config @@ -85,7 +92,16 @@ class CmdModuleInstall extends CmdBase { def resolver = new ModuleResolver(baseDir, client ?: RegistryClientFactory.forConfig(registryConfig)) try { - def installedMainFile = resolver.installModule(reference, version, force) + // -update-deps: for an already-installed module, refresh only its vendored dependencies + // to match meta.yml (the module itself is left untouched). Ignored if not installed. + if( updateDeps && resolver.isInstalled(reference) ) { + resolver.updateDependencies(reference) + println "Module ${reference} dependencies updated successfully" + return + } + + // Install the module together with its transitive requires.modules dependencies + def installedMainFile = resolver.installWithDependencies(reference, version, force) // Read the installed version from meta.yml to avoid a redundant registry call def installedVersion = ModuleSpecFactory.fromYaml(installedMainFile.parent.resolve('meta.yml')).version diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy index a9ecf87aca..b96bf751c7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy @@ -106,12 +106,12 @@ class CmdModuleList extends CmdBase { println "" println "Installed modules:" println "" - println "Module".padRight(40) + "Version".padRight(15) + "Status" - println("-" * 70) + println "Module".padRight(40) + "Version".padRight(15) + "Kind".padRight(12) + "Status" + println("-" * 82) installed.each { module -> def status = getStatusString(module.integrity) - println "${module.reference.toString().padRight(40)}${(module.installedVersion ?: 'unknown').padRight(15)}${status}" + println "${module.reference.toString().padRight(40)}${(module.installedVersion ?: 'unknown').padRight(15)}${(module.kind ?: 'Process').padRight(12)}${status}" } println "" } @@ -121,6 +121,7 @@ class CmdModuleList extends CmdBase { [ name : module.reference.toString(), version : module.installedVersion ?: 'unknown', + kind : module.kind ?: 'Process', integrity: module.integrity.toString(), directory: module.directory.toString(), registry : module.registryUrl ?: 'unknown' diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy index 493831530e..453d8f1b11 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy @@ -27,9 +27,11 @@ import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException import nextflow.module.ModuleChecksum import nextflow.module.ModuleInfo +import nextflow.module.ModuleSchemaValidator import nextflow.module.ModuleSpec import nextflow.module.ModuleSpecFactory import nextflow.module.ModuleReference +import nextflow.module.ModuleResolver import nextflow.module.ModuleValidator import nextflow.module.RegistryClientFactory import nextflow.module.ModuleStorage @@ -55,6 +57,9 @@ class CmdModulePublish extends CmdBase { @Parameter(names = ["-registry"], description = "Target registry URL.") String registryUrl + @Parameter(names = ["-schema"], description = "URL or local path of the JSON schema used to validate meta.yml") + String schema + @Parameter(description = "Module directory path or scope/name") List args @@ -83,7 +88,9 @@ class CmdModulePublish extends CmdBase { log.info "Publishing module from: ${moduleDir}" // Step 1: Validate module structure and spec - def validationErrors = ModuleValidator.validate(moduleDir) + def manifestPath = moduleDir.resolve(ModuleStorage.MODULE_MANIFEST_FILE) + def schemaLocation = ModuleSchemaValidator.resolveSchemaLocation(manifestPath, schema) + def validationErrors = ModuleValidator.validate(moduleDir, schemaLocation) if (!validationErrors.isEmpty()) { throw new AbortOperationException( "Module validation failed:\n" + validationErrors.collect { " - ${it}" }.join('\n') @@ -91,9 +98,30 @@ class CmdModulePublish extends CmdBase { } // Step 2: Load spec for publish metadata - def manifestPath = moduleDir.resolve(ModuleStorage.MODULE_MANIFEST_FILE) def spec = ModuleSpecFactory.fromYaml(manifestPath) + // Config / registry access (needed to verify dependencies and to publish) + def config = new ConfigBuilder() + .setOptions(launcher.options) + .setBaseDir(moduleDir) + .build() + def registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig() + + // Step 3: Verify declared dependencies resolve in the registry. Dependencies are not + // bundled with the module -- consumers re-resolve them from the registry at install time -- + // so each declared `requires.modules` reference must exist there at its pinned version. + if( spec.requiresModules ) { + def resolver = new ModuleResolver(moduleDir, client ?: RegistryClientFactory.forConfig(registryConfig)) + def missing = resolver.findMissingDependencies(spec.requiresModules) + if( missing ) { + throw new AbortOperationException( + "Module validation failed:\n" + missing.collect { + " - Declared dependency '${it}' was not found in the registry".toString() + }.join('\n') + ) + } + } + log.info "Module validated: ${spec.name}@${spec.version}" if (dryRun) { @@ -101,14 +129,6 @@ class CmdModulePublish extends CmdBase { return } - // Step 3: Get authentication token - def config = new ConfigBuilder() - .setOptions(launcher.options) - .setBaseDir(moduleDir) - .build() - - def registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig() - publishModule(moduleDir, registryConfig, spec) } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy index 779d6606a1..49c5493a7b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy @@ -104,7 +104,9 @@ class CmdModuleRun extends CmdRun { final registryConfig = new RegistryConfig(config.registry as Map ?: Collections.emptyMap()) try { final resolver = new ModuleResolver(baseDir, client ?: RegistryClientFactory.forConfig(registryConfig)) - return resolver.installModule(reference, version) + // vendor the module together with its transitive deps (pinned) before running, + // consistent with `module install` and include-time auto-install + return resolver.installWithDependencies(reference, version) } catch( Exception e ) { throw new AbortOperationException("Unable to install module: ${name}", e) } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleSearch.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleSearch.groovy index be274a3f82..c09096a494 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleSearch.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleSearch.groovy @@ -122,6 +122,14 @@ class CmdModuleSearch extends CmdBase { } } + /** + * The module kind, defaulting to {@code Process} when the registry does not + * report one (older registries or process modules that predate the field). + */ + private static String kindOf(ModuleSearchResult result) { + return result.kind != null ? result.kind.toString() : 'Process' + } + private void printFormattedResults(SearchModulesResponse response) { println "" println "Top ${response.totalResults} matching module(s):" @@ -129,6 +137,7 @@ class CmdModuleSearch extends CmdBase { response.results.each { ModuleSearchResult result -> println " ${result.name}" + println " Kind: ${kindOf(result)}" if( result.description ) { println " Description: ${result.description}" } @@ -141,6 +150,7 @@ class CmdModuleSearch extends CmdBase { [ name : result.name, repositoryPath: result.repositoryPath, + kind : kindOf(result), description : result.description, relevanceScore: result.relevanceScore, keywords : result.keywords, diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleValidate.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleValidate.groovy index 47ae2579de..91b1e57cb0 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleValidate.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleValidate.groovy @@ -61,7 +61,8 @@ class CmdModuleValidate extends CmdBase { throw new AbortOperationException("Incorrect number of arguments -- usage: nextflow module validate ") final moduleDir = determineModuleDir(args[0]) - final schemaLocation = schema ?: ModuleSchemaValidator.DEFAULT_SCHEMA_URL + final manifest = moduleDir.resolve(ModuleStorage.MODULE_MANIFEST_FILE) + final schemaLocation = ModuleSchemaValidator.resolveSchemaLocation(manifest, schema) final errors = ModuleValidator.validate(moduleDir, schemaLocation) if( errors ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleView.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleView.groovy index 829fabc1c0..e45da0b0b6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleView.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleView.groovy @@ -137,11 +137,20 @@ class CmdModuleView extends CmdBase { } } + /** + * The module kind, defaulting to {@code Process} when the registry does not report one + * (older registries or process modules that predate the field). + */ + private static String kindOf(ModuleMetadata metadata) { + return metadata.kind != null ? metadata.kind.toString() : 'Process' + } + private void printFormattedInfo(ModuleReference reference, ModuleRelease release, String moduleUrl) { ModuleMetadata metadata = release.metadata println "" println "Module: ${reference}" println "Version: ${release.version}" + println "Kind: ${kindOf(metadata)}" println "URL: ${moduleUrl}" println "Description: ${metadata.description ?: release.description ?: 'N/A'}" @@ -288,6 +297,7 @@ class CmdModuleView extends CmdBase { name : reference.toString(), fullName : reference.fullName, version : release.version, + kind : kindOf(metadata), url : moduleUrl, description: metadata.description ?: release.description, authors : metadata.authors, diff --git a/modules/nextflow/src/main/groovy/nextflow/module/DefaultRemoteModuleResolver.groovy b/modules/nextflow/src/main/groovy/nextflow/module/DefaultRemoteModuleResolver.groovy index f0b28dc3fd..04ef7990c6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/DefaultRemoteModuleResolver.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/DefaultRemoteModuleResolver.groovy @@ -45,8 +45,8 @@ import java.nio.file.Path class DefaultRemoteModuleResolver implements RemoteModuleResolver { @Override - Path resolve(String moduleName, Path projectDir) { - final baseDir = projectDir ?: Path.of('.').toAbsolutePath() + Path resolve(String moduleName, Path baseDir0) { + final baseDir = baseDir0 ?: Path.of('.').toAbsolutePath() final config = Global.config ?: new ConfigBuilder().setBaseDir(baseDir).build() final registryConfig = config.navigate('registry') as RegistryConfig diff --git a/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy b/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy index e951df7c59..a65b7a3840 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy @@ -41,6 +41,7 @@ class InstalledModule { Path manifestFile Path moduleInfoFile String installedVersion + String kind String expectedChecksum String registryUrl diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy index c59d2b96d3..3b971990f1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy @@ -81,7 +81,7 @@ class ModuleResolver { if( version && installed.installedVersion != version ) { if( autoInstall ) { log.info "Upgrading module ${reference} from ${installed.installedVersion} to ${version}" - return installModule(reference, version) + return installWithDependencies(reference, version) } else { throw new AbortOperationException( "Module ${reference} version mismatch: " + @@ -97,7 +97,7 @@ class ModuleResolver { // Module not installed if( autoInstall ) { - return installModule(reference, version) + return installWithDependencies(reference, version) } else { throw new AbortOperationException( "Module ${reference} is not installed. " + @@ -123,19 +123,35 @@ class ModuleResolver { * @return Path to the installed module's main.nf file */ Path installModule(ModuleReference reference, String version = null, boolean force = false) { + return installInto(storage, reference, version, force).mainFile + } + + /** + * Install or update a module into the given storage and return the resulting + * {@link InstalledModule}. The storage's base directory determines where the + * module is placed -- the project root for a top-level module, or a parent + * module's own directory for a nested (vendored) dependency. + * + * @param store the target storage (defines the install base directory) + * @param reference the module reference + * @param version specific version (null = latest) + * @param force force reinstall even if locally modified + * @return the installed module + */ + private InstalledModule installInto(ModuleStorage store, ModuleReference reference, String version, boolean force) { // Check if already installed locally before hitting the registry - if( storage.isInstalled(reference) ) { - def installed = storage.getInstalledModule(reference) + if( store.isInstalled(reference) ) { + def installed = store.getInstalledModule(reference) // No specific version requested -- use the local module as-is if( !version ) { log.debug "Module ${reference}@${installed.installedVersion} is already installed locally" - return installed.mainFile + return installed } if( installed.installedVersion == version ) { log.debug "Module ${reference}@${installed.installedVersion} is already installed (version $version)" - return installed.mainFile + return installed } // Version mismatch -- check for local modifications before overwriting @@ -166,10 +182,10 @@ class ModuleResolver { def downloadUrl = registryClient.downloadModuleRelease(reference.fullName, version, tempFile) // Install to modules directory (will compute directory checksum for future integrity checks) - InstalledModule installed = storage.installModule(reference, version, tempFile, downloadUrl) + InstalledModule installed = store.installModule(reference, version, tempFile, downloadUrl) log.info "Module ${reference}@${version} installed successfully at ${installed.mainFile.parent.toAbsolutePath()}" - return installed.mainFile + return installed } finally { // Clean up temporary file @@ -179,4 +195,182 @@ class ModuleResolver { } } + /** + * Return the declared {@code requires.modules} dependencies that cannot be resolved in the + * registry (i.e. do not exist at their pinned version). Used to validate a module's declared + * dependencies before publishing -- the dependencies are re-resolved from the registry by + * consumers at install time, so they must exist there (they are not bundled with the module). + * + * @param deps the declared dependency references ({@code scope/name@version}) + * @return the subset of {@code deps} that could not be resolved in the registry + */ + List findMissingDependencies(List deps) { + final missing = new ArrayList() + for( final dep : deps ) { + final parsed = parseDependency(dep) + if( !dependencyExists(parsed.reference.fullName, parsed.version) ) + missing.add(dep) + } + return missing + } + + private boolean dependencyExists(String name, String version) { + try { + return version + ? registryClient.getModuleRelease(name, version) != null + : registryClient.getModule(name) != null + } + catch( Exception e ) { + log.debug "Registry lookup failed for ${name}${version ? "@${version}" : ''}: ${e.message}" + return false + } + } + + /** + * A parsed {@code requires.modules} entry: a module reference plus an + * optional pinned version (the part after {@code @}). + */ + @groovy.transform.TupleConstructor + static class DependencySpec { + ModuleReference reference + String version + } + + /** + * Parse a {@code requires.modules} entry of the form + * {@code scope/name[@]}. + */ + protected static DependencySpec parseDependency(String dep) { + final idx = dep.lastIndexOf('@') + if( idx < 0 ) + return new DependencySpec(ModuleReference.parse(dep), null) + final name = dep.substring(0, idx) + final version = dep.substring(idx + 1) + return new DependencySpec(ModuleReference.parse(name), version ?: null) + } + + /** + * Install a module together with its transitive {@code requires.modules} + * dependencies, using nested per-module vendoring (ADR v2, PR #7342). + * + * The module installs under this resolver's base (`/modules///`); + * each of its *direct* dependencies is vendored under the module's own nested + * `modules/` directory, and each dependency likewise vendors its own + * dependencies (arbitrary depth). Dependencies are installed at their pinned + * version in isolation -- there is no cross-module flattening or conflict + * resolution, so the same module may be vendored more than once in the tree. + * Dependency cycles are detected and reported rather than followed. + * + * @param reference the root module reference + * @param version optional explicit version for the root (null = latest) + * @param force force reinstall of locally modified modules + * @return the path to the root module's main.nf file + */ + Path installWithDependencies(ModuleReference reference, String version = null, boolean force = false) { + return walkDependencies(storage, reference, version, force, new LinkedHashSet()) + } + + /** + * @return true if the given module is installed at this resolver's base directory. + */ + boolean isInstalled(ModuleReference reference) { + return storage.isInstalled(reference) + } + + /** + * Update the vendored dependencies of an already-installed module so that its nested + * {@code modules/} directory matches the module's (possibly locally-edited) meta.yml + * {@code requires.modules}, without reinstalling the module itself. + * + * This automates the manual workflow of editing a module's meta.yml dependency versions and + * then re-vendoring them. The parent module is left untouched -- in particular its checksum is + * not refreshed -- so its (unpublished) modification status is preserved. Each declared + * dependency is installed/updated at its pinned version (transitively); a locally-modified + * dependency is not silently overwritten (an error is raised instead). A vendored dependency + * that is no longer declared is pruned, unless it has local modifications, in which case an + * error is raised rather than discarding the changes. + * + * @param reference the installed module whose dependencies should be updated + */ + void updateDependencies(ModuleReference reference) { + final installed = storage.getInstalledModule(reference) + if( installed == null ) + throw new AbortOperationException("Module ${reference} is not installed") + + // declared direct dependencies from the installed (possibly locally-edited) meta.yml + final deps = ModuleSpecFactory.fromYaml(installed.manifestFile).requiresModules + final nestedStore = storage.nestedFor(reference) + + // remove vendored dependencies that are no longer declared + pruneOrphanDependencies(nestedStore, deps) + + // install/update each declared dependency at its pinned version (transitively) + final onStack = new LinkedHashSet() + onStack.add(reference.fullName) + for( final dep : deps ) { + final parsed = parseDependency(dep) + walkDependencies(nestedStore, parsed.reference, parsed.version, false, onStack) + } + } + + /** + * Remove vendored dependencies under {@code nestedStore} that are not in {@code declaredDeps}. + * A modified orphan is not removed -- an error is raised so the user can reconcile it. + */ + private static void pruneOrphanDependencies(ModuleStorage nestedStore, List declaredDeps) { + final declared = new HashSet() + for( final dep : declaredDeps ) + declared.add(parseDependency(dep).reference.fullName) + + for( final vendored : nestedStore.listInstalled() ) { + if( vendored.reference.fullName in declared ) + continue + if( vendored.integrity == ModuleIntegrity.MODIFIED ) + throw new AbortOperationException( + "Vendored dependency '${vendored.reference}' has local modifications and is no longer " + + "declared in meta.yml -- refusing to remove it. Restore it in meta.yml or discard the changes.") + log.info "Removing dependency no longer declared in meta.yml: ${vendored.reference}" + nestedStore.removeModule(vendored.reference, true) + } + } + + private Path walkDependencies(ModuleStorage store, ModuleReference reference, String version, boolean force, + Set onStack) { + // cycle detection keys on the module name only (ignoring version): this guarantees + // termination on the finite set of module names. A diamond (same module in sibling + // branches) is fine — the name is popped from onStack when a branch completes. + final key = reference.fullName + + if( key in onStack ) + throw new AbortOperationException("Module dependency cycle detected: ${(onStack.toList() << key).join(' -> ')}") + + // whether the module is already present at the requested version -- installInto will reuse + // it as-is, so we must not touch its checksum (preserving any local modification status) + final existing = store.getInstalledModule(reference) + final reused = existing != null && (!version || existing.installedVersion == version) + + // install this module at the current level + final installed = installInto(store, reference, version, force) + + // read its direct dependencies from the installed spec (requiresModules is never null) + final deps = ModuleSpecFactory.fromYaml(installed.manifestFile).requiresModules + + // dependencies are vendored under THIS module's own nested `modules/` directory + final nestedStore = store.nestedFor(reference) + onStack.add(key) + for( final dep : deps ) { + final parsed = parseDependency(dep) + walkDependencies(nestedStore, parsed.reference, parsed.version, force, onStack) + } + onStack.remove(key) + + // if this module was (re)installed in this run, refresh its checksum now that its nested + // dependencies are vendored, so the stored checksum covers the full subtree and a clean + // install reads as VALID (the checksum is saved by installModule *before* deps are present) + if( !reused ) + store.refreshChecksum(reference) + + return installed.mainFile + } + } diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleSchemaValidator.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleSchemaValidator.groovy index fdd137f3db..9985b79579 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleSchemaValidator.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleSchemaValidator.groovy @@ -30,6 +30,7 @@ import com.networknt.schema.ValidationMessage import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.BuildInfo +import nextflow.SysEnv import nextflow.exception.AbortOperationException import org.yaml.snakeyaml.Yaml @@ -45,8 +46,47 @@ class ModuleSchemaValidator { static final String DEFAULT_SCHEMA_URL = 'https://raw.githubusercontent.com/nextflow-io/schemas/refs/heads/main/module/v1/schema.json' + /** + * Env var to override the schema location used to validate module specs, e.g. while + * the published remote schema does not yet include newer fields (kind, requires.modules). + */ + static final String SCHEMA_ENV_VAR = 'NXF_MODULE_SPEC_SCHEMA' + private static final ObjectMapper JSON_MAPPER = new ObjectMapper() + /** + * Resolve the schema location to validate a {@code meta.yml} against, honoring, in order: + * an explicit value (e.g. a {@code -schema} CLI flag), the {@code NXF_MODULE_SPEC_SCHEMA} + * env var, the {@code $schema} field declared in the meta.yml, then the default remote + * schema URL. + * + * @param metaYaml the meta.yml being validated (may be used to read its `$schema` field) + * @param explicit an explicit override (e.g. from a CLI flag), or null + * @return the schema location (URL, file: URI, or local path) + */ + static String resolveSchemaLocation(Path metaYaml, String explicit) { + if( explicit ) + return explicit + final fromEnv = SysEnv.get(SCHEMA_ENV_VAR, '') + if( fromEnv ) + return fromEnv + final fromMeta = readSchemaField(metaYaml) + if( fromMeta ) + return fromMeta + return DEFAULT_SCHEMA_URL + } + + private static String readSchemaField(Path metaYaml) { + try( final stream = Files.newInputStream(metaYaml) ) { + final data = new Yaml().load(stream) + return data instanceof Map ? (((Map) data).get('$schema') as String) : null + } + catch( Exception e ) { + // meta.yml unreadable/absent here -- structure validation reports it later + return null + } + } + /** * Validate a meta.yml file against the JSON schema located at the given * URL or local file path. @@ -65,7 +105,7 @@ class ModuleSchemaValidator { } static List validate(Path metaYaml) { - return validate(metaYaml, DEFAULT_SCHEMA_URL) + return validate(metaYaml, resolveSchemaLocation(metaYaml, null)) } private static JsonNode parseSchema(String schemaText, String schemaLocation) { diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpec.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpec.groovy index 9a7ace6adb..60a8b3f716 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpec.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpec.groovy @@ -34,6 +34,10 @@ class ModuleSpec { static final String TODO_DESCRIPTION = 'TODO: Add description' + static final String KIND_PROCESS = 'Process' + + static final String KIND_WORKFLOW = 'Workflow' + static final String YAML_HEADER = """\ # This file was auto-generated by `nextflow module spec`. # @@ -56,12 +60,14 @@ class ModuleSpec { String name String version + String kind String description List keywords String license List authors List maintainers Map requires + List requiresModules List tools List inputs List outputs @@ -118,6 +124,14 @@ class ModuleSpec { return validate().isEmpty() } + /** + * @return true if this is a workflow module (kind: Workflow), false otherwise + * (a null/absent kind defaults to a process module). + */ + boolean isWorkflow() { + return kind == KIND_WORKFLOW + } + /** * Render the module spec to YAML. */ @@ -145,6 +159,8 @@ class ModuleSpec { result['name'] = name if( version ) result['version'] = version + if( kind ) + result['kind'] = kind if( description ) result['description'] = description if( keywords ) @@ -155,8 +171,14 @@ class ModuleSpec { result['authors'] = authors if( maintainers ) result['maintainers'] = maintainers - if( requires ) - result['requires'] = requires + if( requires || requiresModules ) { + final req = new LinkedHashMap() + if( requires ) + req.putAll(requires) + if( requiresModules ) + req['modules'] = requiresModules + result['requires'] = req + } if( tools ) result['tools'] = tools if( inputs ) diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpecFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpecFactory.groovy index 1e643a39ce..60d60b5151 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpecFactory.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpecFactory.groovy @@ -22,13 +22,16 @@ import java.nio.file.Path import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.exception.AbortOperationException +import nextflow.script.ast.ProcessNode import nextflow.script.ast.ProcessNodeV1 import nextflow.script.ast.ProcessNodeV2 import nextflow.script.ast.ScriptNode +import nextflow.script.ast.WorkflowNode import nextflow.script.control.ScriptParser import org.yaml.snakeyaml.Yaml import static nextflow.module.ModuleSpec.ModuleParam +import static nextflow.script.ast.ASTUtils.asBlockStatements /** * Factory methods for module specs. @@ -43,6 +46,7 @@ class ModuleSpecFactory { private static final List MODULE_SPEC_FIELDS = [ 'name', 'version', + 'kind', 'description', 'keywords', 'license', @@ -91,32 +95,32 @@ class ModuleSpecFactory { spec.authors = opts.authors as List ?: oldSpec.authors spec.maintainers = oldSpec.maintainers spec.requires = oldSpec.requires + spec.requiresModules = oldSpec.requiresModules spec.tools = oldSpec.tools spec._passthrough = oldSpec._passthrough - // load script - final parser = new ScriptParser() - final sourceUnit = parser.parse(path.toFile()) - parser.analyze() + // parse with semantic analysis so that statically-typed declarations resolve their types + // (an unresolved type cannot be inferred and is rendered as a TODO placeholder) + final scriptNode = parseScript(path) + final processes = scriptNode.getProcesses() + // ignore the (unnamed) entry workflow -- a module publishes a named workflow + final List workflows = scriptNode.getWorkflows().findAll { WorkflowNode w -> !w.isEntry() } - final scriptNode = sourceUnit.getAST() - if( scriptNode !instanceof ScriptNode ) - throw new AbortOperationException("Error parsing module script -- run `nextflow lint ${path}` to check for errors") + if( !processes.isEmpty() ) + return fromProcessScript(opts, path, oldSpec, spec, processes) + if( !workflows.isEmpty() ) + return fromWorkflowScript(opts, oldSpec, spec, workflows) - final errors = sourceUnit.getErrorCollector().getErrors() - if( errors != null && !errors.isEmpty() ) - throw new AbortOperationException("Error parsing module script -- run `nextflow lint ${path}` to check for errors") + throw new AbortOperationException("Module script does not define any process or workflow: ${path}") + } - // get process definition in script - final processes = ((ScriptNode) scriptNode).getProcesses() - if( processes.isEmpty() ) - throw new AbortOperationException("Module script does not define any processes: ${path}") + private static ModuleSpec fromProcessScript(Map opts, Path path, ModuleSpec oldSpec, ModuleSpec spec, List processes) { if( processes.size() > 1 ) throw new AbortOperationException("Module script defines multiple processes: ${path}") // infer module spec properties from process definition final process = processes[0] - + spec.kind = oldSpec.kind spec.name = opts.name ?: "${opts.namespace}/${process.name.toLowerCase()}" if( process instanceof ProcessNodeV1 ) { @@ -135,10 +139,88 @@ class ModuleSpecFactory { return spec } + private static ModuleSpec fromWorkflowScript(Map opts, ModuleSpec oldSpec, ModuleSpec spec, List workflows) { + if( workflows.size() > 1 ) + throw new AbortOperationException("Module script defines multiple workflows") + + // infer module spec properties from the workflow's take:/emit: interface. Types are taken + // from the source when statically typed; an untyped take/emit renders a TODO placeholder. + final workflow = workflows[0] + spec.kind = ModuleSpec.KIND_WORKFLOW + spec.name = opts.name ?: "${opts.namespace}/${workflow.name.toLowerCase()}" + + final visitor = new ModuleSpecVisitorV2(oldSpec) + spec.inputs = visitor.visitTakes(workflow) + spec.outputs = visitor.visitEmits(workflow) + + return spec + } + static ModuleSpec fromScript(Map opts = [:], Path path) { return fromScript(opts, path, new ModuleSpec()) } + /** + * Parse a module script and return its AST root, failing on parse errors. + * + * @param path the module script + * @param analyze whether to run semantic analysis (include resolution etc.); disable it when + * only the syntactic structure is needed, so that includes of not-yet-installed + * modules do not cause a failure + */ + private static ScriptNode parseScript(Path path, boolean analyze = true) { + final parser = new ScriptParser() + final sourceUnit = parser.parse(path.toFile()) + if( analyze ) + parser.analyze() + + final scriptNode = sourceUnit.getAST() + if( scriptNode !instanceof ScriptNode ) + throw new AbortOperationException("Error parsing module script -- run `nextflow lint ${path}` to check for errors") + + final errors = sourceUnit.getErrorCollector().getErrors() + if( errors != null && !errors.isEmpty() ) + throw new AbortOperationException("Error parsing module script -- run `nextflow lint ${path}` to check for errors") + + return (ScriptNode) scriptNode + } + + /** + * @return true if the given module script defines at least one workflow. Parsing is + * syntactic only, so a workflow that includes not-yet-installed modules still validates. + */ + static boolean definesWorkflow(Path path) { + return !parseScript(path, false).getWorkflows().isEmpty() + } + + /** + * The take/emit arity of a workflow, used to reconcile a workflow module's + * {@code take:}/{@code emit:} interface with its meta.yml input/output. + */ + static class WorkflowInterface { + final int takes + final int emits + + WorkflowInterface(int takes, int emits) { + this.takes = takes + this.emits = emits + } + } + + /** + * The take/emit arity of every workflow defined by the given script. Parsing is + * syntactic only, so a workflow that includes not-yet-installed modules still parses. + */ + static List workflowInterfaces(Path path) { + final List result = [] + for( final WorkflowNode wf : parseScript(path, false).getWorkflows() ) { + final takes = wf.getParameters() != null ? wf.getParameters().length : 0 + final emits = asBlockStatements(wf.emits).size() + result.add(new WorkflowInterface(takes, emits)) + } + return result + } + /** * Load a module spec from a yaml file * @@ -155,12 +237,13 @@ class ModuleSpecFactory { final spec = new ModuleSpec() spec.name = data.name as String spec.version = data.version as String + spec.kind = data.kind as String spec.description = data.description as String spec.keywords = data.keywords as List ?: [] spec.license = data.license as String spec.authors = data.authors as List ?: [] spec.maintainers = data.maintainers as List ?: [] - spec.requires = data.requires as Map ?: [:] + parseRequires(data.requires, spec) spec.tools = data.tools as List ?: [] final inputs = data.input @@ -197,6 +280,39 @@ class ModuleSpecFactory { } } + /** + * Parse the `requires` block, separating the transitive module dependency + * list (`requires.modules`) from scalar constraints (e.g. `requires.nextflow`). + * + * @param requires the raw `requires` value from the parsed yaml + * @param spec the module spec to populate + */ + private static void parseRequires(Object requires, ModuleSpec spec) { + if( requires == null ) { + spec.requires = [:] + spec.requiresModules = [] + return + } + if( requires !instanceof Map ) + throw new Exception("invalid spec requires") + final reqMap = requires as Map + final mods = reqMap.get('modules') + if( mods instanceof List ) + spec.requiresModules = mods.collect { it as String } + else if( mods != null ) + throw new Exception("invalid spec requires.modules") + else + spec.requiresModules = [] + final scalar = new LinkedHashMap() + for( final entry : reqMap.entrySet() ) { + if( entry.key == 'modules' ) + continue + if( entry.value != null ) + scalar[entry.key as String] = entry.value as String + } + spec.requires = scalar + } + /** * Load a module input/output from the YAML representation. * diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpecVisitorV2.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpecVisitorV2.groovy index ff0c9ab249..318727fa82 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpecVisitorV2.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpecVisitorV2.groovy @@ -21,6 +21,7 @@ import groovy.util.logging.Slf4j import nextflow.script.ast.AssignmentExpression import nextflow.script.ast.ProcessNodeV2 import nextflow.script.ast.TupleParameter +import nextflow.script.ast.WorkflowNode import org.codehaus.groovy.ast.ClassHelper import org.codehaus.groovy.ast.ClassNode import org.codehaus.groovy.ast.Parameter @@ -62,6 +63,22 @@ class ModuleSpecVisitorV2 { return moduleTopics(asBlockStatements(node.topics), oldSpec.topics) } + /** + * Extract the module inputs from a workflow's {@code take:} parameters. An untyped take + * yields a null type, which is rendered as a {@code TODO: Add type} placeholder. + */ + List visitTakes(WorkflowNode node) { + return moduleInputs(node.getParameters(), oldSpec.inputs) + } + + /** + * Extract the module outputs from a workflow's {@code emit:} statements. An untyped emit + * yields a null type, which is rendered as a {@code TODO: Add type} placeholder. + */ + List visitEmits(WorkflowNode node) { + return moduleOutputs(asBlockStatements(node.emits), oldSpec.outputs) + } + private static List moduleInputs(Parameter[] params, List oldParams) { final result = new ArrayList(params.length) for( int i = 0; i < params.length; i++ ) { @@ -201,9 +218,21 @@ class ModuleSpecVisitorV2 { } private static final ClassNode PATH_TYPE = ClassHelper.makeCached(java.nio.file.Path) + private static final ClassNode CHANNEL_TYPE = ClassHelper.makeCached(nextflow.script.types.Channel) + private static final ClassNode RECORD_TYPE = ClassHelper.makeCached(nextflow.script.types.Record) private static String paramType(ClassNode type) { - if( !type || !type.isResolved() ) + if( !type ) + return null + + // generic documentation tags for statically-typed declarations -- the authoritative + // types live in the source take:/emit: and typed input/output (parsed at `module run`) + if( isChannelType(type) ) + return 'channel' + if( isRecordType(type) ) + return 'custom-record' + + if( !type.isResolved() ) return null if( type.implementsInterface(ClassHelper.ITERABLE_TYPE) && !type.equals(PATH_TYPE) ) { @@ -231,6 +260,14 @@ class ModuleSpecVisitorV2 { } } + private static boolean isChannelType(ClassNode type) { + return type.equals(CHANNEL_TYPE) || type.implementsInterface(CHANNEL_TYPE) + } + + private static boolean isRecordType(ClassNode type) { + return type.equals(RECORD_TYPE) || type.implementsInterface(RECORD_TYPE) + } + private static String paramType(Expression node) { if( node instanceof MethodCallExpression ) { final name = node.getMethodAsString() diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy index 04f6c8e244..c56d7a30e5 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy @@ -48,6 +48,7 @@ import static nextflow.module.ModuleInfo.MODULE_INFO_FILE class ModuleStorage { public static final String MODULE_MANIFEST_FILE = "meta.yml" public static final String MODULE_README_FILE = "README.md" + public static final String MODULES_DIR = "modules" private final Path modulesDir /** @@ -56,7 +57,7 @@ class ModuleStorage { * @param baseDir The base directory (usually project root) */ ModuleStorage(Path baseDir) { - this.modulesDir = baseDir.resolve('modules') + this.modulesDir = baseDir.resolve(MODULES_DIR) } /** @@ -78,6 +79,18 @@ class ModuleStorage { return modulesDir.resolve(reference.scope).resolve(reference.name) } + /** + * Get a storage rooted at the given module's own directory, so that the + * module's vendored dependencies are installed/resolved under its nested + * {@code modules/} directory (nested per-module vendoring). + * + * @param reference the parent module + * @return a storage whose base directory is the parent module's directory + */ + ModuleStorage nestedFor(ModuleReference reference) { + return new ModuleStorage(getModuleDir(reference)) + } + /** * Get the module info path for a specific module * @@ -99,6 +112,20 @@ class ModuleStorage { return Files.exists(moduleDir) && Files.isDirectory(moduleDir) } + /** + * Recompute and persist a module's integrity checksum after its nested dependencies have + * been vendored, so the stored checksum covers the module's full installed subtree (its own + * files plus vendored deps). The registry origin is preserved. This makes a freshly installed + * module read as VALID, while any later edit -- to the module's own files or to a vendored + * dependency -- is detected as a modification. + * + * @param reference The module reference + */ + void refreshChecksum(ModuleReference reference) { + final moduleDir = getModuleDir(reference) + ModuleInfo.save(moduleDir, 'checksum', ModuleChecksum.compute(moduleDir)) + } + /** * Get an installed module * @@ -123,7 +150,9 @@ class ModuleStorage { Map infoProps = ModuleInfo.load(moduleDir) installed.expectedChecksum = infoProps?.checksum installed.registryUrl = infoProps?.registryUrl - installed.installedVersion = ModuleSpecFactory.fromYaml(installed.manifestFile).version + final spec = ModuleSpecFactory.fromYaml(installed.manifestFile) + installed.installedVersion = spec.version + installed.kind = spec.kind ?: ModuleSpec.KIND_PROCESS return installed } @@ -416,6 +445,13 @@ class ModuleStorage { return } + // Skip the nested vendored dependencies directory (modules/) at the module root: + // a module's requires.modules dependencies are re-resolved from the registry at + // install time, so they must not be shipped inside the parent module's bundle + if (Files.isDirectory(path) && path == sourceDir.resolve(MODULES_DIR)) { + return + } + def relativePath = sourceDir.relativize(path).toString() if (Files.isDirectory(path)) { diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleValidator.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleValidator.groovy index d3601c2969..5e8725a732 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleValidator.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleValidator.groovy @@ -59,16 +59,22 @@ class ModuleValidator { if( errors ) return errors - // Level 3: validate module input/output spec against process definition - final scriptPath = moduleDir.resolve("main.nf") - final sourceSpec = ModuleSpecFactory.fromScript(scriptPath) - errors.addAll(validateInputsOutputs(spec, sourceSpec)) + // Level 3: validate the module script against its kind + final scriptPath = moduleDir.resolve(Const.DEFAULT_MAIN_FILE_NAME) + if( spec.isWorkflow() ) { + errors.addAll(validateWorkflow(spec, scriptPath)) + } + else { + final sourceSpec = ModuleSpecFactory.fromScript(scriptPath) + errors.addAll(validateInputsOutputs(spec, sourceSpec)) + } return errors } static List validate(Path moduleDir) { - return validate(moduleDir, ModuleSchemaValidator.DEFAULT_SCHEMA_URL) + final manifest = moduleDir.resolve(ModuleStorage.MODULE_MANIFEST_FILE) + return validate(moduleDir, ModuleSchemaValidator.resolveSchemaLocation(manifest, null)) } /** @@ -136,4 +142,36 @@ class ModuleValidator { return errors } + + /** + * Validate a workflow module's script and reconcile its {@code take:}/{@code emit:} interface + * with the meta.yml. A workflow module must define exactly one workflow; when the meta.yml also + * declares {@code input}/{@code output} (e.g. a typed workflow) the counts must match the + * workflow's take/emit arity. When the meta.yml omits them the take:/emit: sections are the sole + * source of truth and there is nothing to reconcile. + * + * @param spec the parsed module spec + * @param scriptPath the module's main.nf + */ + static List validateWorkflow(ModuleSpec spec, Path scriptPath) { + final errors = new ArrayList() + + final interfaces = ModuleSpecFactory.workflowInterfaces(scriptPath) + if( interfaces.isEmpty() ) { + errors << "Workflow module '${spec.name}' must define a workflow in ${Const.DEFAULT_MAIN_FILE_NAME}".toString() + return errors + } + if( interfaces.size() > 1 ) { + errors << "Workflow module '${spec.name}' must define exactly one workflow in ${Const.DEFAULT_MAIN_FILE_NAME} (found ${interfaces.size()})".toString() + return errors + } + + final iface = interfaces[0] + if( spec.inputs != null && spec.inputs.size() != iface.takes ) + errors << "Module spec has ${spec.inputs.size()} inputs but workflow declares ${iface.takes} take(s)".toString() + if( spec.outputs != null && spec.outputs.size() != iface.emits ) + errors << "Module spec has ${spec.outputs.size()} outputs but workflow declares ${iface.emits} emit(s)".toString() + + return errors + } } diff --git a/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy index 3159c5282b..b9361329af 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy @@ -181,7 +181,11 @@ class IncludeDef { Path resolveRemoteModulePath(String moduleName) { // Use SPI to get the remote module resolver implementation def resolver = RemoteModuleResolverProvider.getInstance() - return resolver.resolve(moduleName, session.baseDir) + // Resolve relative to the including file's directory (context-relative) so that a + // workflow module's own dependencies are found under its nested `modules/` directory + // (nested vendoring, ADR v2). For the top-level script this is the project base dir. + final base = getOwnerPath()?.getParent() ?: session.baseDir + return resolver.resolve(moduleName, base) } @PackageScope diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleCreateTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleCreateTest.groovy index d800c6f071..720bc806ed 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleCreateTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleCreateTest.groovy @@ -40,6 +40,119 @@ class CmdModuleCreateTest extends Specification { content.contains("process HELLO") } + def 'should generate a workflow module main.nf with -kind Workflow'() { + when: + def content = CmdModuleCreate.mainNf('myorg', 'hello', 'Workflow') + + then: + content.contains("Workflow module: myorg/hello") + content.contains("workflow HELLO") + content.contains("take:") + content.contains("emit:") + !content.contains("process ") + } + + def 'should generate a workflow module meta.yml with kind Workflow'() { + when: + def content = CmdModuleCreate.metaYml('myorg', 'hello', 'Workflow') + + then: + content.contains("name: myorg/hello") + content.contains("kind: Workflow") + + and: 'the untyped scaffold take/emit are documented as channels' + content.contains("input:") + content.contains("name: ch_input") + content.contains("type: channel") + content.contains("name: output") + + and: 'untyped workflows require the baseline Nextflow version' + content.contains('nextflow: ">=24.04.0"') + } + + def 'generated untyped workflow scaffold passes validation'() { + given: + def moduleDir = tempDir.resolve('modules/myorg/hello') + Files.createDirectories(moduleDir) + moduleDir.resolve('main.nf').text = CmdModuleCreate.mainNf('myorg', 'hello', 'Workflow') + moduleDir.resolve('meta.yml').text = CmdModuleCreate.metaYml('myorg', 'hello', 'Workflow') + moduleDir.resolve('README.md').text = '# hello\n' + and: 'a permissive schema' + def schema = tempDir.resolve('schema.json') + schema.text = '{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object",' + + '"properties":{"name":{"type":"string"},"description":{"type":"string"}},"required":["name","description"]}' + + expect: 'no errors -- interface counts match take/emit and there are no TODO placeholders' + nextflow.module.ModuleValidator.validate(moduleDir, schema.toString()).isEmpty() + } + + def 'should default to a process module'() { + expect: + CmdModuleCreate.mainNf('myorg', 'hello').contains("process HELLO") + !CmdModuleCreate.metaYml('myorg', 'hello').contains("kind:") + } + + def 'should generate a typed process module with -typed'() { + when: + def content = CmdModuleCreate.mainNf('myorg', 'hello', 'Process', true) + + then: + content.contains("nextflow.enable.types = true") + content.contains("process HELLO") + content.contains("greeting: String") // typed input + content.contains("message: String = stdout()") // named typed output bound to stdout + content.contains("script:") // keeps a shell script block + !content.contains("val greeting") // not the untyped form + !content.contains("exec:") + + and: + def meta = CmdModuleCreate.metaYml('myorg', 'hello', 'Process', true) + meta.contains("name: message") + meta.contains('nextflow: ">=25.10.0"') // typed processes require Nextflow 25.10.0+ + } + + def 'generated typed workflow scaffold parses as a workflow'() { + given: + def nf = tempDir.resolve('main.nf') + nf.text = CmdModuleCreate.mainNf('myorg', 'hello', 'Workflow', true) + + expect: + nextflow.module.ModuleSpecFactory.definesWorkflow(nf) + } + + def 'generated typed process scaffold parses cleanly'() { + given: + def nf = tempDir.resolve('main.nf') + nf.text = CmdModuleCreate.mainNf('myorg', 'hello', 'Process', true) + + expect: + // parses without error (no workflow defined -> false, but must not throw) + !nextflow.module.ModuleSpecFactory.definesWorkflow(nf) + } + + def 'should generate a typed workflow module with -kind Workflow -typed'() { + when: + def content = CmdModuleCreate.mainNf('myorg', 'hello', 'Workflow', true) + + then: + content.contains("nextflow.enable.types = true") + content.contains("workflow HELLO") + content.contains("greeting: String") // typed take + content.contains("result: String = message") // typed emit + !content.contains("process ") + + and: 'the meta.yml derives input/output from the take/emit' + def meta = CmdModuleCreate.metaYml('myorg', 'hello', 'Workflow', true) + meta.contains("kind: Workflow") + meta.contains("input:") + meta.contains("name: greeting") + meta.contains("output:") + meta.contains("name: result") + + and: 'typed workflows require Nextflow 26.04.0+' + meta.contains('nextflow: ">=26.04.0"') + } + def 'should translate special chars in process name to underscore and uppercase'() { expect: CmdModuleCreate.mainNf('myorg', name).contains("process ${expected}") diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInstallTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInstallTest.groovy index b626ce78bd..bddbfc567a 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInstallTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInstallTest.groovy @@ -90,6 +90,52 @@ class CmdModuleInstallTest extends Specification { Files.exists(moduleDir.resolve(ModuleInfo.MODULE_INFO_FILE)) } + def 'should reject -update-deps combined with -force'() { + given: + def cmd = new CmdModuleInstall() + cmd.args = ['nf-core/fastqc'] + cmd.root = tempDir + cmd.updateDeps = true + cmd.force = true + + when: + cmd.run() + + then: + def e = thrown(AbortOperationException) + e.message.contains('cannot be used together') + } + + def 'should ignore -update-deps and install normally when the module is not installed'() { + given: + def cmd = new CmdModuleInstall() + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.args = ['nf-core/fastqc'] + cmd.root = tempDir + cmd.updateDeps = true + + and: + def modulePackage = createModulePackage('nf-core', 'fastqc', '1.0.0') + def mockClient = Mock(RegistryClient) + mockClient.getModule('nf-core/fastqc') >> new Module( + name: 'nf-core/fastqc', + latest: new ModuleRelease(version: '1.0.0') + ) + mockClient.downloadModuleRelease('nf-core/fastqc', '1.0.0', _) >> { String name, String version, Path dest -> + Files.write(dest, modulePackage) + return dest + } + cmd.client = mockClient + + when: + cmd.run() + + then: 'a normal install happens (the flag is a no-op for a not-yet-installed module)' + Files.exists(tempDir.resolve('modules/nf-core/fastqc/main.nf')) + } + def 'should install module with specific version'() { given: def cmd = new CmdModuleInstall() diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy index dace2c8ba0..82258d4e8a 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy @@ -151,6 +151,49 @@ class CmdModuleListTest extends Specification { output.contains('myorg/custom') } + def 'should include the module kind in the listing'() { + given: + def storage = new ModuleStorage(tempDir) + createTestModule(storage, 'nf-core', 'fastqc', '1.0.0') // process (no kind) + createWorkflowModule(storage, 'nf-core', 'align_wf', '0.0.0-test') // kind: Workflow + + and: + def cmd = new CmdModuleList() + cmd.root = tempDir + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Kind') + output.contains('Process') // fastqc defaults to Process + output.contains('Workflow') // align_wf declares kind: Workflow + } + + private Path createWorkflowModule(ModuleStorage storage, String scope, String name, String version) { + def moduleDir = storage.getModuleDir(new ModuleReference(scope, name)) + Files.createDirectories(moduleDir) + moduleDir.resolve('main.nf').text = """ + workflow ${name.toUpperCase()} { + take: + ch_in + main: + ch_out = ch_in + emit: + out = ch_out + } + """.stripIndent() + moduleDir.resolve('meta.yml').text = """ + name: ${scope}/${name} + version: ${version} + kind: Workflow + description: Test workflow module + """.stripIndent() + ModuleInfo.save(moduleDir, [checksum: ModuleChecksum.compute(moduleDir), registryUrl: 'http://registry.com']) + return moduleDir + } + private Path createTestModule(ModuleStorage storage, String scope, String name, String version) { def moduleDir = storage.getModuleDir(new ModuleReference(scope, name)) Files.createDirectories(moduleDir) diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModulePublishTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModulePublishTest.groovy index 1daec349dd..d73579c46c 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModulePublishTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModulePublishTest.groovy @@ -16,7 +16,10 @@ package nextflow.cli.module +import io.seqera.npr.api.schema.v1.ModuleRelease +import io.seqera.npr.client.RegistryClient import nextflow.cli.Launcher +import nextflow.exception.AbortOperationException import nextflow.module.ModuleValidator import spock.lang.Specification import spock.lang.TempDir @@ -146,4 +149,70 @@ class CmdModulePublishTest extends Specification { then: noExceptionThrown() } + + private String permissiveSchema() { + final p = tempDir.resolve('schema.json') + p.text = '{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object",' + + '"properties":{"name":{"type":"string"},"description":{"type":"string"}},"required":["name","description"]}' + return p.toString() + } + + private Path workflowModuleWithDep(String depRef) { + final dir = tempDir.resolve('wf-module') + Files.createDirectories(dir) + dir.resolve('main.nf').text = 'workflow FOO {\n take:\n ch_in\n emit:\n ch_in\n}\n' + dir.resolve('README.md').text = '# wf' + dir.resolve('meta.yml').text = """\ + name: test/wf + version: 1.0.0 + kind: Workflow + description: Test workflow module + requires: + modules: + - ${depRef} + """.stripIndent() + return dir + } + + private CmdModulePublish publishCmd(Path dir, RegistryClient client) { + final launcher = new Launcher() + launcher.options = [:] + final cmd = new CmdModulePublish() + cmd.launcher = launcher + cmd.args = [dir.toString()] + cmd.schema = permissiveSchema() + cmd.dryRun = true + cmd.client = client + return cmd + } + + def 'should pass publish validation when a declared dependency exists in the registry' () { + given: + def dir = workflowModuleWithDep('nf-core/dep@1.2.0') + def cmd = publishCmd(dir, Mock(RegistryClient) { + getModuleRelease('nf-core/dep', '1.2.0') >> new ModuleRelease(version: '1.2.0') + }) + + when: + cmd.run() + + then: + noExceptionThrown() + } + + def 'should fail publish validation when a declared dependency is not in the registry' () { + given: + def dir = workflowModuleWithDep('nf-core/dep@1.2.0') + def cmd = publishCmd(dir, Mock(RegistryClient) { + getModuleRelease(_, _) >> { throw new RuntimeException('not found') } + }) + + when: + cmd.run() + + then: + def e = thrown(AbortOperationException) + e.message.contains('nf-core/dep@1.2.0') + e.message.contains('not found in the registry') + } } diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleSearchTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleSearchTest.groovy index 6a4d29f3f9..c8502c4569 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleSearchTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleSearchTest.groovy @@ -18,6 +18,7 @@ package nextflow.cli.module import groovy.json.JsonSlurper import io.seqera.npr.client.RegistryClient +import io.seqera.npr.api.schema.v1.ModuleKind import io.seqera.npr.api.schema.v1.ModuleSearchResult import io.seqera.npr.api.schema.v1.SearchModulesResponse import nextflow.exception.AbortOperationException @@ -44,12 +45,14 @@ class CmdModuleSearchTest extends Specification { def result1 = new ModuleSearchResult( name: 'nf-core/fastqc', repositoryPath: 'nf-core/modules', + kind: ModuleKind.WORKFLOW, description: 'FastQC quality control', relevanceScore: 0.95, keywords: ['quality-control', 'fastqc'], tools: ['fastqc'], revoked: false ) + // result2 has no kind -> should default to Process def result2 = new ModuleSearchResult( name: 'nf-core/multiqc', repositoryPath: 'nf-core/modules', @@ -93,18 +96,32 @@ class CmdModuleSearchTest extends Specification { output.contains('nf-core/multiqc') output.contains('MultiQC reporting') + and: 'the module kind is shown, defaulting to Process when absent' + output.contains('Kind: Workflow') + output.contains('Kind: Process') + } def 'should search and display results in JSON output'() { given: def result1 = new ModuleSearchResult( name: 'nf-core/fastqc', + kind: ModuleKind.WORKFLOW, description: 'FastQC quality control', relevanceScore: 0.95, keywords: ['quality-control'], tools: ['fastqc'], revoked: false ) + // result2 has no kind -> should default to Process in the JSON output + def result2 = new ModuleSearchResult( + name: 'nf-core/multiqc', + description: 'MultiQC reporting', + relevanceScore: 0.85, + keywords: ['reporting'], + tools: ['multiqc'], + revoked: false + ) and: def cmd = new CmdModuleSearch() @@ -120,8 +137,8 @@ class CmdModuleSearchTest extends Specification { def mockClient = Mock(RegistryClient) mockClient.searchModules('fastqc', 10) >> new SearchModulesResponse( query: 'fastqc', - totalResults: 1, - results: [result1] + totalResults: 2, + results: [result1, result2] ) cmd.client = mockClient @@ -135,11 +152,15 @@ class CmdModuleSearchTest extends Specification { then: json.query == 'fastqc' - json.totalResults == 1 - json.results.size() == 1 + json.totalResults == 2 + json.results.size() == 2 json.results[0].name == 'nf-core/fastqc' json.results[0].description == 'FastQC quality control' + and: 'kind is emitted, defaulting to Process when absent' + json.results[0].kind == 'Workflow' + json.results[1].kind == 'Process' + } def 'should handle no search results'() { diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleViewTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleViewTest.groovy index c448095d99..ac0aebe714 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleViewTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleViewTest.groovy @@ -20,6 +20,7 @@ import groovy.json.JsonSlurper import io.seqera.npr.client.RegistryClient import io.seqera.npr.api.schema.v1.Module import io.seqera.npr.api.schema.v1.ModuleChannel +import io.seqera.npr.api.schema.v1.ModuleKind import io.seqera.npr.api.schema.v1.ModuleChannelItem import io.seqera.npr.api.schema.v1.ModuleMetadata import io.seqera.npr.api.schema.v1.ModuleRelease @@ -49,6 +50,7 @@ class CmdModuleViewTest extends Specification { def 'should display module info in formatted output'() { given: def metadata = new ModuleMetadata( + kind: ModuleKind.WORKFLOW, description: 'FastQC quality control analysis', authors: ['nf-core', 'community'], keywords: ['quality-control', 'fastqc', 'reads'] @@ -94,6 +96,10 @@ class CmdModuleViewTest extends Specification { output.contains('Keywords:') output.contains('quality-control, fastqc, reads') output.contains('Usage Template:') + + and: 'the module kind is shown' + output.contains('Kind:') + output.contains('Workflow') } def 'should display module info with specific version'() { @@ -183,6 +189,9 @@ class CmdModuleViewTest extends Specification { json.authors == ['nf-core'] json.keywords == ['quality-control'] json.usageTemplate != null + + and: 'kind defaults to Process when the registry does not report one' + json.kind == 'Process' } def 'should display module info with tools'() { diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverDependencyTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverDependencyTest.groovy new file mode 100644 index 0000000000..71a7e3491d --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverDependencyTest.groovy @@ -0,0 +1,301 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import java.nio.file.Files +import java.nio.file.Path + +import io.seqera.npr.api.schema.v1.ListModuleReleasesResponse +import io.seqera.npr.api.schema.v1.Module +import io.seqera.npr.api.schema.v1.ModuleRelease +import io.seqera.npr.client.RegistryClient +import nextflow.exception.AbortOperationException +import nextflow.util.VersionNumber +import org.yaml.snakeyaml.Yaml +import spock.lang.Specification +import spock.lang.TempDir + +/** + * Tests for nested transitive dependency vendoring in + * {@link ModuleResolver#installWithDependencies} (ADR v2, PR #7342). + * + * @author Jorge Ejarque + */ +class ModuleResolverDependencyTest extends Specification { + + @TempDir + Path tempDir + + // module universe: fullName -> ( version -> requires.modules list ) + private Map>> universe = [:] + + private void module(String name, String version, List requires = []) { + universe.computeIfAbsent(name, { [:] }).put(version, requires) + } + + private void buildBundle(String name, String version, List requires, Path dest) { + final dir = Files.createTempDirectory('mod') + dir.resolve('main.nf').text = "workflow FOO {\n}\n" + final meta = [name: name, version: version, kind: 'Workflow', description: "test module ${name}".toString(), + requires: [nextflow: '>=24.04.0']] + if( requires ) + meta.requires.modules = requires + dir.resolve('meta.yml').text = new Yaml().dump(meta) + ModuleStorage.createBundle(dir, dest) + } + + private RegistryClient mockClient() { + final client = Mock(RegistryClient) + client.listModuleReleases(_) >> { String n -> + final resp = new ListModuleReleasesResponse() + resp.releases = universe.getOrDefault(n, [:]).keySet().collect { v -> new ModuleRelease().version(v) } + return resp + } + client.getModule(_) >> { String n -> + final versions = new ArrayList(universe.getOrDefault(n, [:]).keySet()) + final latest = versions.max { a, b -> new VersionNumber(a) <=> new VersionNumber(b) } + return new Module().latest(new ModuleRelease().version(latest)) + } + client.downloadModuleRelease(_, _, _) >> { String n, String v, Path dest -> + buildBundle(n, v, universe[n][v], dest) + return "oci://${n}:${v}" + } + return client + } + + private boolean installedAt(String relDir) { + return Files.exists(tempDir.resolve(relDir).resolve('main.nf')) + } + + private String versionAt(String relDir) { + return ModuleSpecFactory.fromYaml(tempDir.resolve(relDir).resolve('meta.yml')).version + } + + // simulate a user hand-editing an installed module's meta.yml requires.modules + private void editRequires(String relDir, List requires) { + final metaFile = tempDir.resolve(relDir).resolve('meta.yml') + final meta = new Yaml().load(metaFile.text) as Map + final req = (meta.requires ?: [:]) as Map + if( requires ) + req.modules = requires + else + req.remove('modules') + meta.requires = req + metaFile.text = new Yaml().dump(meta) + } + + def 'a freshly installed workflow module with dependencies reads as VALID' () { + given: + module('nf-core/aln', '1.0.0', ['nf-core/samtools/sort@1.0.0']) + module('nf-core/samtools/sort', '1.0.0') + def resolver = new ModuleResolver(tempDir, mockClient()) + def storage = new ModuleStorage(tempDir) + + when: + resolver.installWithDependencies(ModuleReference.parse('nf-core/aln'), '1.0.0') + + then: 'the checksum covers the vendored subtree, so no false MODIFIED status' + storage.getInstalledModule(ModuleReference.parse('nf-core/aln')).integrity == ModuleIntegrity.VALID + and: 'the vendored dependency is itself VALID' + new ModuleStorage(tempDir.resolve('modules/nf-core/aln')) + .getInstalledModule(ModuleReference.parse('nf-core/samtools/sort')).integrity == ModuleIntegrity.VALID + } + + def 'editing an installed module own file marks it MODIFIED' () { + given: + module('nf-core/aln', '1.0.0', ['nf-core/samtools/sort@1.0.0']) + module('nf-core/samtools/sort', '1.0.0') + def resolver = new ModuleResolver(tempDir, mockClient()) + def storage = new ModuleStorage(tempDir) + resolver.installWithDependencies(ModuleReference.parse('nf-core/aln'), '1.0.0') + + when: + def mainNf = tempDir.resolve('modules/nf-core/aln/main.nf') + mainNf.text = mainNf.text + "\n// local edit\n" + + then: + storage.getInstalledModule(ModuleReference.parse('nf-core/aln')).integrity == ModuleIntegrity.MODIFIED + } + + def 'editing a vendored dependency by hand marks the parent MODIFIED' () { + given: + module('nf-core/aln', '1.0.0', ['nf-core/samtools/sort@1.0.0']) + module('nf-core/samtools/sort', '1.0.0') + def resolver = new ModuleResolver(tempDir, mockClient()) + def storage = new ModuleStorage(tempDir) + resolver.installWithDependencies(ModuleReference.parse('nf-core/aln'), '1.0.0') + + when: 'a vendored dependency file is edited by hand' + def depMain = tempDir.resolve('modules/nf-core/aln/modules/nf-core/samtools/sort/main.nf') + depMain.text = depMain.text + "\n// hand edit of a vendored dep\n" + + then: 'the change is surfaced through the parent module integrity (Option 1: subtree checksum)' + storage.getInstalledModule(ModuleReference.parse('nf-core/aln')).integrity == ModuleIntegrity.MODIFIED + } + + def 'update-deps installs a newly declared dependency' () { + given: + module('nf-core/aln', '1.0.0', []) // parent initially has no dependencies + module('nf-core/samtools/sort', '1.0.0') + def resolver = new ModuleResolver(tempDir, mockClient()) + resolver.installWithDependencies(ModuleReference.parse('nf-core/aln'), '1.0.0') + assert !installedAt('modules/nf-core/aln/modules/nf-core/samtools/sort') + + when: 'the user declares a dependency in meta.yml and updates deps' + editRequires('modules/nf-core/aln', ['nf-core/samtools/sort@1.0.0']) + resolver.updateDependencies(ModuleReference.parse('nf-core/aln')) + + then: + installedAt('modules/nf-core/aln/modules/nf-core/samtools/sort') + } + + def 'update-deps updates a dependency to the version declared in meta.yml' () { + given: + module('nf-core/aln', '1.0.0', ['nf-core/samtools/sort@1.0.0']) + module('nf-core/samtools/sort', '1.0.0') + module('nf-core/samtools/sort', '2.0.0') + def resolver = new ModuleResolver(tempDir, mockClient()) + resolver.installWithDependencies(ModuleReference.parse('nf-core/aln'), '1.0.0') + assert versionAt('modules/nf-core/aln/modules/nf-core/samtools/sort') == '1.0.0' + + when: + editRequires('modules/nf-core/aln', ['nf-core/samtools/sort@2.0.0']) + resolver.updateDependencies(ModuleReference.parse('nf-core/aln')) + + then: + versionAt('modules/nf-core/aln/modules/nf-core/samtools/sort') == '2.0.0' + } + + def 'update-deps prunes a dependency removed from meta.yml' () { + given: + module('nf-core/aln', '1.0.0', ['nf-core/samtools/sort@1.0.0']) + module('nf-core/samtools/sort', '1.0.0') + def resolver = new ModuleResolver(tempDir, mockClient()) + resolver.installWithDependencies(ModuleReference.parse('nf-core/aln'), '1.0.0') + assert installedAt('modules/nf-core/aln/modules/nf-core/samtools/sort') + + when: 'the dependency is removed from meta.yml and deps are updated' + editRequires('modules/nf-core/aln', []) + resolver.updateDependencies(ModuleReference.parse('nf-core/aln')) + + then: + !installedAt('modules/nf-core/aln/modules/nf-core/samtools/sort') + } + + def 'update-deps errors and keeps a removed dependency that has local modifications' () { + given: + module('nf-core/aln', '1.0.0', ['nf-core/samtools/sort@1.0.0']) + module('nf-core/samtools/sort', '1.0.0') + def resolver = new ModuleResolver(tempDir, mockClient()) + resolver.installWithDependencies(ModuleReference.parse('nf-core/aln'), '1.0.0') + and: 'the vendored dependency is hand-edited' + def depMain = tempDir.resolve('modules/nf-core/aln/modules/nf-core/samtools/sort/main.nf') + depMain.text = depMain.text + "\n// local edit\n" + + when: 'the dependency is removed from meta.yml and deps are updated' + editRequires('modules/nf-core/aln', []) + resolver.updateDependencies(ModuleReference.parse('nf-core/aln')) + + then: + def e = thrown(AbortOperationException) + e.message.contains('local modifications') + and: 'the modified orphan is not removed' + installedAt('modules/nf-core/aln/modules/nf-core/samtools/sort') + } + + def 'update-deps leaves the parent module untouched (stays MODIFIED)' () { + given: + module('nf-core/aln', '1.0.0', []) + module('nf-core/samtools/sort', '1.0.0') + def resolver = new ModuleResolver(tempDir, mockClient()) + def storage = new ModuleStorage(tempDir) + resolver.installWithDependencies(ModuleReference.parse('nf-core/aln'), '1.0.0') + + when: 'the user edits the parent meta.yml to add a dependency, then updates deps' + editRequires('modules/nf-core/aln', ['nf-core/samtools/sort@1.0.0']) + resolver.updateDependencies(ModuleReference.parse('nf-core/aln')) + + then: 'the dependency is vendored' + installedAt('modules/nf-core/aln/modules/nf-core/samtools/sort') + and: 'the parent stays MODIFIED (unpublished meta.yml edit; checksum not refreshed)' + storage.getInstalledModule(ModuleReference.parse('nf-core/aln')).integrity == ModuleIntegrity.MODIFIED + } + + def 'should install a module and vendor its dependencies under its own nested modules/ dir' () { + given: + module('nf-core/aln', '1.0.0', ['nf-core/samtools/sort@1.0.0']) + module('nf-core/samtools/sort', '1.0.0') + def resolver = new ModuleResolver(tempDir, mockClient()) + + when: + resolver.installWithDependencies(ModuleReference.parse('nf-core/aln'), '1.0.0') + + then: + installedAt('modules/nf-core/aln') + // dependency is vendored under the module's OWN nested modules/ directory + installedAt('modules/nf-core/aln/modules/nf-core/samtools/sort') + } + + def 'should vendor a shared dependency independently per consumer (duplication, no flattening)' () { + given: + module('nf-core/top', '1.0.0', ['nf-core/a@1.0.0', 'nf-core/b@1.0.0']) + module('nf-core/a', '1.0.0', ['nf-core/c@1.0.0']) + module('nf-core/b', '1.0.0', ['nf-core/c@2.0.0']) // different version -- no conflict under nested model + module('nf-core/c', '1.0.0') + module('nf-core/c', '2.0.0') + def resolver = new ModuleResolver(tempDir, mockClient()) + + when: + resolver.installWithDependencies(ModuleReference.parse('nf-core/top'), '1.0.0') + + then: + // c is vendored twice -- once under a, once under b -- at each pinned version + versionAt('modules/nf-core/top/modules/nf-core/a/modules/nf-core/c') == '1.0.0' + versionAt('modules/nf-core/top/modules/nf-core/b/modules/nf-core/c') == '2.0.0' + } + + def 'should detect a dependency cycle' () { + given: + module('nf-core/a', '1.0.0', ['nf-core/b@1.0.0']) + module('nf-core/b', '1.0.0', ['nf-core/a@1.0.0']) + def resolver = new ModuleResolver(tempDir, mockClient()) + + when: + resolver.installWithDependencies(ModuleReference.parse('nf-core/a'), '1.0.0') + + then: + def e = thrown(AbortOperationException) + e.message.contains('cycle') + } + + def 'resolve with autoInstall installs a workflow module and its nested deps' () { + given: + module('nf-core/mafft_align', '0.0.0-test001', ['nf-core/mafft/align@0.0.0-test001']) + module('nf-core/mafft/align', '0.0.0-test001') + def resolver = new ModuleResolver(tempDir, mockClient()) + + when: + // this is the include-resolution path (autoInstall = true, version = null) + def main = resolver.resolve(ModuleReference.parse('nf-core/mafft_align'), null, true) + + then: + installedAt('modules/nf-core/mafft_align') + installedAt('modules/nf-core/mafft_align/modules/nf-core/mafft/align') + main.toString().endsWith('modules/nf-core/mafft_align/main.nf') + } + +} diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleSchemaValidatorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleSchemaValidatorTest.groovy index 318e1d6307..74816874f8 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleSchemaValidatorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleSchemaValidatorTest.groovy @@ -19,6 +19,7 @@ package nextflow.module import java.nio.file.Files import java.nio.file.Path +import nextflow.SysEnv import nextflow.exception.AbortOperationException import spock.lang.Specification import spock.lang.TempDir @@ -164,4 +165,50 @@ class ModuleSchemaValidatorTest extends Specification { def e = thrown(AbortOperationException) e.message.contains('Cannot determine JSON Schema draft') } + + def 'resolveSchemaLocation: explicit value wins over env, meta and default' () { + given: + def meta = writeMeta('name: x\n$schema: /from/meta.json\n') + SysEnv.push([(ModuleSchemaValidator.SCHEMA_ENV_VAR): '/from/env.json']) + + when: + def loc = ModuleSchemaValidator.resolveSchemaLocation(meta, '/explicit.json') + + then: + loc == '/explicit.json' + + cleanup: + SysEnv.pop() + } + + def 'resolveSchemaLocation: env var wins over meta $schema and default' () { + given: + def meta = writeMeta('name: x\n$schema: /from/meta.json\n') + SysEnv.push([(ModuleSchemaValidator.SCHEMA_ENV_VAR): '/from/env.json']) + + when: + def loc = ModuleSchemaValidator.resolveSchemaLocation(meta, null) + + then: + loc == '/from/env.json' + + cleanup: + SysEnv.pop() + } + + def 'resolveSchemaLocation: meta $schema used when no explicit or env override' () { + given: + def meta = writeMeta('name: x\n$schema: /from/meta.json\n') + + expect: + ModuleSchemaValidator.resolveSchemaLocation(meta, null) == '/from/meta.json' + } + + def 'resolveSchemaLocation: falls back to the default schema url' () { + given: + def meta = writeMeta('name: x\n') + + expect: + ModuleSchemaValidator.resolveSchemaLocation(meta, null) == ModuleSchemaValidator.DEFAULT_SCHEMA_URL + } } diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleSpecFactoryTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleSpecFactoryTest.groovy index 96944f0b43..bb627b96f4 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleSpecFactoryTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleSpecFactoryTest.groovy @@ -62,6 +62,129 @@ class ModuleSpecFactoryTest extends Specification { spec.requires == ['nextflow': '>=24.04.0'] } + def 'should load a workflow module spec with kind and requires.modules' () { + given: + def metaYaml = tempDir.resolve('meta.yml') + metaYaml.text = '''\ + name: nf-core/fastq_align_star + version: 0.0.0-4e3e10e + kind: Workflow + description: Align reads then sort with samtools + keywords: + - align + - star + license: MIT + requires: + nextflow: ">=24.04.0" + modules: + - nf-core/star/align@>=1.0.0 + - nf-core/samtools/sort@>=1.2.0,<2.0.0 + '''.stripIndent() + + when: + def spec = ModuleSpecFactory.fromYaml(metaYaml) + + then: + spec.name == 'nf-core/fastq_align_star' + spec.kind == 'Workflow' + spec.isWorkflow() + spec.requires == ['nextflow': '>=24.04.0'] + spec.requiresModules == ['nf-core/star/align@>=1.0.0', 'nf-core/samtools/sort@>=1.2.0,<2.0.0'] + } + + def 'should default to process kind and empty requiresModules when absent' () { + given: + def metaYaml = tempDir.resolve('meta.yml') + metaYaml.text = '''\ + name: nf-core/fastqc + version: 1.0.0 + description: FastQC quality control + requires: + nextflow: ">=24.04.0" + '''.stripIndent() + + when: + def spec = ModuleSpecFactory.fromYaml(metaYaml) + + then: + spec.kind == null + !spec.isWorkflow() + spec.requiresModules == [] + spec.requires == ['nextflow': '>=24.04.0'] + } + + def 'should round-trip kind and requires.modules through asMap' () { + given: + def spec = new ModuleSpec( + name: 'nf-core/fastq_align_star', + version: '0.0.0-abc', + kind: 'Workflow', + description: 'demo', + requires: ['nextflow': '>=24.04.0'], + requiresModules: ['nf-core/star/align@>=1.0.0'] + ) + + when: + def map = spec.asMap() + + then: + map['kind'] == 'Workflow' + map['requires'] == ['nextflow': '>=24.04.0', 'modules': ['nf-core/star/align@>=1.0.0']] + } + + def 'definesWorkflow should return true for a workflow script' () { + given: + def nf = tempDir.resolve('main.nf') + nf.text = '''\ + workflow FOO { + take: + ch_in + main: + ch_out = ch_in + emit: + ch_out + } + '''.stripIndent() + + expect: + ModuleSpecFactory.definesWorkflow(nf) + } + + def 'definesWorkflow should tolerate includes of not-yet-installed modules' () { + given: + def nf = tempDir.resolve('main.nf') + nf.text = '''\ + include { MAFFT_ALIGN as MAFFT_ALIGN_MODULE } from 'nf-core/mafft/align' + + workflow MAFFT_ALIGN { + take: + ch_fasta + main: + MAFFT_ALIGN_MODULE ( ch_fasta ) + emit: + alignment = MAFFT_ALIGN_MODULE.out.fas + } + '''.stripIndent() + + expect: + ModuleSpecFactory.definesWorkflow(nf) + } + + def 'definesWorkflow should return false for a process-only script' () { + given: + def nf = tempDir.resolve('main.nf') + nf.text = '''\ + process FOO { + """ + echo hello + """ + } + '''.stripIndent() + + expect: + !ModuleSpecFactory.definesWorkflow(nf) + } + def 'should fail to load non-existent spec' () { given: def metaYaml = tempDir.resolve('meta.yml') @@ -518,6 +641,65 @@ class ModuleSpecFactoryTest extends Specification { spec.topics[0].components[2].type == 'string' } + def 'should map a user-defined record type to the custom-record documentation tag'() { + given: + def mainNf = tempDir.resolve('main.nf') + mainNf.text = '''\ + nextflow.enable.types = true + + record Sample { + id: String + reads: Path + } + + process ALIGN { + input: + sample: Sample + + output: + bam: Path + + exec: + bam = file("${sample.id}.bam") + } + '''.stripIndent() + + when: + def spec = ModuleSpecFactory.fromScript(mainNf, namespace: 'my-namespace') + + then: + spec.inputs.size() == 1 + spec.inputs[0].name == 'sample' + spec.inputs[0].type == 'custom-record' + } + + def 'should map a Channel type to the channel documentation tag'() { + given: + def mainNf = tempDir.resolve('main.nf') + mainNf.text = '''\ + nextflow.enable.types = true + + process COLLECT { + input: + items: Channel + + output: + result: Path + + exec: + result = file('out.txt') + } + '''.stripIndent() + + when: + def spec = ModuleSpecFactory.fromScript(mainNf, namespace: 'my-namespace') + + then: + spec.inputs.size() == 1 + spec.inputs[0].name == 'items' + spec.inputs[0].type == 'channel' + } + def 'should extract tuple inputs and outputs'() { given: def mainNf = tempDir.resolve('main.nf') @@ -572,6 +754,91 @@ class ModuleSpecFactoryTest extends Specification { spec.topics[0].components[2].type == 'string' } + // ========================================================================= + // workflow module tests + // ========================================================================= + + def 'should extract a workflow spec with TODO types for an untyped workflow'() { + given: + def mainNf = tempDir.resolve('main.nf') + mainNf.text = '''\ + workflow ALIGN_WF { + take: + ch_reads + ch_index + + main: + ch_out = ch_reads + + emit: + aligned = ch_out + } + '''.stripIndent() + + when: + def spec = ModuleSpecFactory.fromScript(mainNf, namespace: 'my-namespace') + + then: + spec.kind == 'Workflow' + spec.name == 'my-namespace/align_wf' + + and: 'take names are extracted, types cannot be inferred (rendered as TODO)' + spec.inputs.size() == 2 + spec.inputs[0].name == 'ch_reads' + spec.inputs[0].type == null + spec.inputs[1].name == 'ch_index' + spec.inputs[1].type == null + + and: 'emit names are extracted, types cannot be inferred' + spec.outputs.size() == 1 + spec.outputs[0].name == 'aligned' + spec.outputs[0].type == null + + and: 'untyped params render the TODO placeholder in the YAML' + def parsed = new Yaml().load(spec.toYaml()) as Map + (parsed['input'] as List)[0]['type'] == 'TODO: Add type' + (parsed['output'] as List)[0]['type'] == 'TODO: Add type' + } + + def 'should extract a workflow spec with inferred types for a typed workflow'() { + given: + def mainNf = tempDir.resolve('main.nf') + mainNf.text = '''\ + nextflow.enable.types = true + + workflow ALIGN_WF { + take: + greeting: String + count: Integer + + main: + message = greeting + + emit: + result: String = message + } + '''.stripIndent() + + when: + def spec = ModuleSpecFactory.fromScript(mainNf, namespace: 'my-namespace') + + then: + spec.kind == 'Workflow' + spec.name == 'my-namespace/align_wf' + + and: 'typed takes yield inferred types' + spec.inputs.size() == 2 + spec.inputs[0].name == 'greeting' + spec.inputs[0].type == 'string' + spec.inputs[1].name == 'count' + spec.inputs[1].type == 'integer' + + and: 'typed emits yield inferred types' + spec.outputs.size() == 1 + spec.outputs[0].name == 'result' + spec.outputs[0].type == 'string' + } + // ========================================================================= // legacy process tests // ========================================================================= diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy index 042696f82e..7da3fa59ad 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy @@ -20,7 +20,9 @@ import java.nio.file.Files import java.nio.file.Path import java.util.zip.GZIPOutputStream import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream import nextflow.exception.AbortOperationException @@ -531,4 +533,48 @@ class ModuleStorageTest extends Specification { // Cleanup temp directory tempModuleDir.deleteDir() } + + def 'should exclude the nested vendored modules directory from the publish bundle'() { + given: 'a module with its own files, a resources dir, and a vendored dependency' + def moduleDir = tempDir.resolve('mod') + Files.createDirectories(moduleDir) + Files.writeString(moduleDir.resolve('main.nf'), "workflow FOO { }\n") + Files.writeString(moduleDir.resolve('meta.yml'), "name: nf-core/foo\nversion: 1.0.0\nkind: Workflow\ndescription: demo\n") + Files.writeString(moduleDir.resolve('README.md'), "# foo\n") + Files.createDirectories(moduleDir.resolve('resources')) + Files.writeString(moduleDir.resolve('resources').resolve('data.txt'), "hello\n") + and: 'a vendored dependency under the nested modules/ directory' + def dep = moduleDir.resolve('modules').resolve('nf-core').resolve('dep') + Files.createDirectories(dep) + Files.writeString(dep.resolve('main.nf'), "workflow DEP { }\n") + Files.writeString(dep.resolve('meta.yml'), "name: nf-core/dep\nversion: 2.0.0\nkind: Workflow\ndescription: dep\n") + + when: + def bundle = tempDir.resolve('bundle.tar.gz') + ModuleStorage.createBundle(moduleDir, bundle) + def entries = listEntries(bundle) + + then: 'the module own files and resources are bundled' + entries.contains('main.nf') + entries.contains('meta.yml') + entries.contains('README.md') + entries.any { it.startsWith('resources/') } + + and: 'nothing from the nested vendored modules/ directory is bundled' + !entries.any { it == 'modules/' || it.startsWith('modules/') } + } + + private List listEntries(Path bundle) { + final result = new ArrayList() + Files.newInputStream(bundle).withCloseable { fis -> + new GzipCompressorInputStream(fis).withCloseable { gzis -> + new TarArchiveInputStream(gzis).withCloseable { tis -> + TarArchiveEntry entry + while ((entry = tis.nextTarEntry) != null) + result.add(entry.name) + } + } + } + return result + } } diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleValidatorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleValidatorTest.groovy new file mode 100644 index 0000000000..99499e4d43 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleValidatorTest.groovy @@ -0,0 +1,209 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import java.nio.file.Files +import java.nio.file.Path + +import spock.lang.Specification +import spock.lang.TempDir + +/** + * Tests for workflow-module validation in {@link ModuleValidator}. + * + * @author Jorge Ejarque + */ +class ModuleValidatorTest extends Specification { + + @TempDir + Path tempDir + + private static final String PERMISSIVE_SCHEMA = '''\ + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { "name": {"type": "string"}, "description": {"type": "string"} }, + "required": ["name", "description"] + } + '''.stripIndent() + + private String schema() { + final p = tempDir.resolve('schema.json') + Files.writeString(p, PERMISSIVE_SCHEMA) + return p.toString() + } + + private Path moduleDir(String mainNf, String metaYml) { + final d = Files.createDirectories(tempDir.resolve('mod')) + Files.writeString(d.resolve('main.nf'), mainNf) + Files.writeString(d.resolve('meta.yml'), metaYml) + Files.writeString(d.resolve('README.md'), '# test module\n') + return d + } + + def 'a workflow module that defines a workflow passes validation' () { + given: + def dir = moduleDir( + '''\ + workflow FOO { + take: + ch_in + main: + ch_out = ch_in + emit: + ch_out + } + '''.stripIndent(), + '''\ + name: nf-core/demo_wf + version: 1.0.0 + kind: Workflow + description: a demo workflow module + '''.stripIndent()) + + when: + def errors = ModuleValidator.validate(dir, schema()) + + then: + errors.isEmpty() + } + + def 'a workflow module without a workflow definition fails validation' () { + given: + def dir = moduleDir( + '''\ + process FOO { + """ + echo hello + """ + } + '''.stripIndent(), + '''\ + name: nf-core/demo_wf + version: 1.0.0 + kind: Workflow + description: a demo workflow module + '''.stripIndent()) + + when: + def errors = ModuleValidator.validate(dir, schema()) + + then: + errors.any { it.contains('must define a workflow') } + } + + def 'a workflow module defining more than one workflow fails validation' () { + given: + def dir = moduleDir( + '''\ + workflow FOO { + take: + ch_in + emit: + ch_in + } + workflow BAR { + take: + ch_in + emit: + ch_in + } + '''.stripIndent(), + '''\ + name: nf-core/demo_wf + version: 1.0.0 + kind: Workflow + description: a demo workflow module + '''.stripIndent()) + + when: + def errors = ModuleValidator.validate(dir, schema()) + + then: + errors.any { it.contains('exactly one workflow') } + } + + def 'a workflow meta.yml with matching input/output counts passes validation' () { + given: + def dir = moduleDir( + '''\ + workflow FOO { + take: + ch_a + ch_b + main: + ch_out = ch_a.mix(ch_b) + emit: + ch_out + } + '''.stripIndent(), + '''\ + name: nf-core/demo_wf + version: 1.0.0 + kind: Workflow + description: a demo workflow module + input: + - name: ch_a + type: channel + description: first input + - name: ch_b + type: channel + description: second input + output: + - name: ch_out + type: channel + description: the output + '''.stripIndent()) + + when: + def errors = ModuleValidator.validate(dir, schema()) + + then: + errors.isEmpty() + } + + def 'a workflow meta.yml with mismatched interface counts fails validation' () { + given: + def dir = moduleDir( + '''\ + workflow FOO { + take: + ch_a + ch_b + emit: + ch_a + } + '''.stripIndent(), + '''\ + name: nf-core/demo_wf + version: 1.0.0 + kind: Workflow + description: a demo workflow module + input: + - name: ch_a + type: channel + description: only one declared, but the workflow takes two + '''.stripIndent()) + + when: + def errors = ModuleValidator.validate(dir, schema()) + + then: + errors.any { it.contains('1 inputs but workflow declares 2 take') } + } + +} diff --git a/modules/nf-lang/src/main/java/nextflow/module/spi/FallbackRemoteModuleResolver.java b/modules/nf-lang/src/main/java/nextflow/module/spi/FallbackRemoteModuleResolver.java index 4f993323f8..f0518e25f9 100644 --- a/modules/nf-lang/src/main/java/nextflow/module/spi/FallbackRemoteModuleResolver.java +++ b/modules/nf-lang/src/main/java/nextflow/module/spi/FallbackRemoteModuleResolver.java @@ -31,8 +31,8 @@ public class FallbackRemoteModuleResolver implements RemoteModuleResolver { @Override - public Path resolve(String moduleName, Path projectDir) { - var baseDir = projectDir != null ? projectDir : Path.of(".").toAbsolutePath(); + public Path resolve(String moduleName, Path baseDir0) { + var baseDir = baseDir0 != null ? baseDir0 : Path.of(".").toAbsolutePath(); var modulesDir = baseDir.resolve("modules").normalize(); var resolved = modulesDir.resolve(moduleName).normalize(); if( !resolved.startsWith(modulesDir) ) { diff --git a/modules/nf-lang/src/main/java/nextflow/module/spi/RemoteModuleResolver.java b/modules/nf-lang/src/main/java/nextflow/module/spi/RemoteModuleResolver.java index 6eb8b3bbdc..24a00caba8 100644 --- a/modules/nf-lang/src/main/java/nextflow/module/spi/RemoteModuleResolver.java +++ b/modules/nf-lang/src/main/java/nextflow/module/spi/RemoteModuleResolver.java @@ -47,11 +47,13 @@ public interface RemoteModuleResolver { * * * @param moduleName The module reference string (e.g., '@scope/name' or '@scope/name@version') - * @param projectDir The base directory for the project (used to locate the modules directory) + * @param baseDir The directory relative to which the {@code modules/} directory is located: + * the project root for a top-level include, or the including module's own + * directory for a nested (vendored) include * @return Path to the resolved module's main.nf file * @throws IllegalArgumentException if the module reference is invalid or resolution fails */ - Path resolve(String moduleName, Path projectDir); + Path resolve(String moduleName, Path baseDir); /** * Get the priority of this resolver. Higher priority resolvers are tried first. diff --git a/modules/nf-lang/src/main/java/nextflow/script/control/ModuleResolver.java b/modules/nf-lang/src/main/java/nextflow/script/control/ModuleResolver.java index 9070106e14..13081cffca 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/control/ModuleResolver.java +++ b/modules/nf-lang/src/main/java/nextflow/script/control/ModuleResolver.java @@ -91,14 +91,18 @@ private SourceUnit resolveInclude(IncludeNode node, SourceUnit sourceUnit, Funct } private URI getIncludeUri(URI uri, String source) { + var parent = Path.of(uri).getParent(); if( isRemoteModule(source) ) { + // Resolve a remote module relative to the including file's directory (context-relative), + // so a workflow module's own dependencies are discovered under its nested `modules/` + // directory (nested vendoring). For the top-level script this parent is the project dir. + var base = parent != null ? parent : projectDir; return RemoteModuleResolverProvider.getInstance() - .resolve(source, projectDir) + .resolve(source, base) .normalize() .toUri(); } else { - var parent = Path.of(uri).getParent(); return getLocalIncludeUri(parent, source); } } diff --git a/modules/nf-lang/src/main/java/nextflow/script/control/ResolveIncludeVisitor.java b/modules/nf-lang/src/main/java/nextflow/script/control/ResolveIncludeVisitor.java index b298c5cfa1..ae3cc49285 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/control/ResolveIncludeVisitor.java +++ b/modules/nf-lang/src/main/java/nextflow/script/control/ResolveIncludeVisitor.java @@ -137,14 +137,18 @@ private static void setPlaceholderTargets(IncludeNode node) { } private URI getIncludeUri(String source) { + var parent = Path.of(uri).getParent(); if( ModuleResolver.isRemoteModule(source) ) { + // Resolve a remote module relative to the including file's directory (context-relative), + // so a workflow module's own dependencies are found under its nested `modules/` directory + // (nested vendoring). For the top-level script this parent is the project directory. + var base = parent != null ? parent : projectDir; return RemoteModuleResolverProvider.getInstance() - .resolve(source, projectDir) + .resolve(source, base) .normalize() .toUri(); } else { - var parent = Path.of(uri).getParent(); return getLocalIncludeUri(parent, source); } }