From f5b6b86b08ab4de5d0086ffb51485dbbf741c5fb Mon Sep 17 00:00:00 2001 From: jorgee Date: Wed, 28 Jan 2026 12:46:56 +0100 Subject: [PATCH 01/23] Add first implementation of module CLI commands Signed-off-by: jorgee --- modules/nextflow/build.gradle | 1 + .../main/groovy/nextflow/cli/CmdBase.groovy | 5 + .../main/groovy/nextflow/cli/CmdModule.groovy | 148 ++++++ .../main/groovy/nextflow/cli/Launcher.groovy | 9 +- .../nextflow/cli/module/ModuleInstall.groovy | 101 ++++ .../nextflow/cli/module/ModuleList.groovy | 122 +++++ .../nextflow/cli/module/ModulePublish.groovy | 251 +++++++++ .../nextflow/cli/module/ModuleRemove.groovy | 124 +++++ .../nextflow/cli/module/ModuleRun.groovy | 130 +++++ .../nextflow/cli/module/ModuleSearch.groovy | 143 ++++++ .../nextflow/config/ModulesConfig.groovy | 93 ++++ .../nextflow/config/RegistryConfig.groovy | 132 +++++ .../nextflow/module/InstalledModule.groovy | 89 ++++ .../nextflow/module/ModuleChecksum.groovy | 157 ++++++ .../nextflow/module/ModuleManifest.groovy | 117 +++++ .../nextflow/module/ModuleReference.groovy | 89 ++++ .../module/ModuleRegistryClient.groovy | 481 ++++++++++++++++++ .../nextflow/module/ModuleResolver.groovy | 168 ++++++ .../nextflow/module/ModuleStorage.groovy | 425 ++++++++++++++++ .../nextflow/util/NextflowSpecFile.groovy | 147 ++++++ .../cli/module/ModulePublishTest.groovy | 143 ++++++ .../module/InstalledModuleTest.groovy | 201 ++++++++ .../nextflow/module/ModuleChecksumTest.groovy | 415 +++++++++++++++ .../nextflow/module/ModuleManifestTest.groovy | 161 ++++++ .../module/ModuleReferenceTest.groovy | 231 +++++++++ .../nextflow/module/ModuleStorageTest.groovy | 361 +++++++++++++ 26 files changed, 4443 insertions(+), 1 deletion(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/cli/CmdModule.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleList.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRemove.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/config/ModulesConfig.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/config/RegistryConfig.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/module/ModuleManifest.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/module/ModuleReference.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/util/NextflowSpecFile.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/cli/module/ModulePublishTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/module/InstalledModuleTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/module/ModuleChecksumTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/module/ModuleManifestTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/module/ModuleReferenceTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy diff --git a/modules/nextflow/build.gradle b/modules/nextflow/build.gradle index 78bff819f0..dd46d41e36 100644 --- a/modules/nextflow/build.gradle +++ b/modules/nextflow/build.gradle @@ -54,6 +54,7 @@ dependencies { api 'dev.failsafe:failsafe:3.1.0' api 'io.seqera:lib-trace:0.1.0' api 'com.fasterxml.woodstox:woodstox-core:7.1.1' + api 'org.apache.commons:commons-compress:1.27.1' // For tar.gz extraction testImplementation 'org.subethamail:subethasmtp:3.1.7' testImplementation (project(':nf-lineage')) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdBase.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdBase.groovy index 75b4cde37b..543b43f0af 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdBase.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdBase.groovy @@ -26,9 +26,14 @@ import com.beust.jcommander.Parameter abstract class CmdBase implements Runnable { private Launcher launcher + private List unknownOptions abstract String getName() + protected List getUnknownOptions(){ return this.unknownOptions } + + void setUnknownOptions(List options){ this.unknownOptions = options } + Launcher getLauncher() { launcher } void setLauncher( Launcher value ) { this.launcher = value } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdModule.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdModule.groovy new file mode 100644 index 0000000000..d43cbf3ddc --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdModule.groovy @@ -0,0 +1,148 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli + +import com.beust.jcommander.JCommander +import com.beust.jcommander.Parameter +import com.beust.jcommander.ParameterException +import com.beust.jcommander.Parameters +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.cli.module.ModuleInstall +import nextflow.cli.module.ModuleList +import nextflow.cli.module.ModulePublish +import nextflow.cli.module.ModuleRemove +import nextflow.cli.module.ModuleRun +import nextflow.cli.module.ModuleSearch +import nextflow.exception.AbortOperationException + +/** + * Implements `module` command + * + * @author Jorge Ejarque + */ +@CompileStatic +@Slf4j +@Parameters(commandDescription = "Manage Nextflow modules") +class CmdModule extends CmdBase implements UsageAware { + + static final public String NAME = 'module' + + private JCommander jCommander + + static final List commands = new ArrayList<>() + + static { + commands << new ModuleInstall() + commands << new ModuleRun() + commands << new ModuleList() + commands << new ModuleRemove() + commands << new ModuleSearch() + commands << new ModulePublish() + } + + protected JCommander commander(){ + if (!this.jCommander) { + this.jCommander = new JCommander(this) + this.jCommander.setProgramName('nextflow module') + // Register all subcommands + commands.each { cmd -> + cmd.launcher = this.launcher + this.jCommander.addCommand(cmd.getName(), cmd, new String[0]) + } + } + return jCommander + } + + @Parameter + List args + + @Override + String getName() { + return NAME + } + + @Override + void run() { + + + try { + final jc = commander() + final moduleArgs = args + unknownOptions + jc.parse(moduleArgs as String[]) + + final parsedCommand = jc.getParsedCommand() + if (!parsedCommand) { + jc.usage() + return + } + + // Get the parsed subcommand instance + final subcommand = jc.getCommands() + .get(parsedCommand) + .getObjects()[0] as CmdBase + + // Execute with fields already populated by JCommander + subcommand.run() + + } catch ( ParameterException e) { + throw new AbortOperationException("${e.getMessage()} -- Check the available commands and options and syntax with 'nextflow module -h'") + } + } + + private CmdBase findCmd(String name) { + commands.find { it.name == name } + } + + /** + * Print the command usage help + */ + @Override + void usage() { + usage(args) + } + + /** + * Print the command usage help + * + * @param args The arguments as entered by the user + */ + @Override + void usage(List args) { + def result = [] + if (!args) { + result << 'Usage: nextflow module [options]' + result << '' + result << 'Commands:' + commands.each { + def description = it.getClass().getAnnotation(Parameters)?.commandDescription() + result << " ${it.name.padRight(12)}${description}" + } + result << '' + println result.join('\n').toString() + } + else { + final sub = findCmd(args[0]) + if (sub) { + commander().usage(args[0]) + } + else { + throw new AbortOperationException("Unknown module sub-command: ${args[0]}") + } + } + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy index 95753b9757..9b3db289fb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy @@ -111,7 +111,8 @@ class Launcher { new CmdPlugin(), new CmdInspect(), new CmdLint(), - new CmdLineage() + new CmdLineage(), + new CmdModule() ] if(SecretsLoader.isEnabled()) @@ -129,6 +130,9 @@ class Launcher { jcommander.addCommand(cmd.name, cmd, aliases(cmd)) } jcommander.setProgramName( APP_NAME ) + + //Allow unknown options for module command + jcommander.getCommands().get(CmdModule.NAME)?.setAcceptUnknownOptions(true) } private static final String[] EMPTY = new String[0] @@ -154,6 +158,9 @@ class Launcher { jcommander.parse( normalizedArgs as String[] ) fullVersion = '-version' in normalizedArgs command = allCommands.find { it.name == jcommander.getParsedCommand() } + //Attach unknown options to command in case of needed + final unknownOptions = jcommander.commands.get(jcommander.getParsedCommand()).getUnknownOptions() + command.setUnknownOptions(unknownOptions) // whether is running a daemon daemonMode = command instanceof CmdNode // set the log file name diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy new file mode 100644 index 0000000000..55d7b9826c --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy @@ -0,0 +1,101 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli.module + +import com.beust.jcommander.Parameter +import com.beust.jcommander.Parameters +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.cli.CmdBase +import nextflow.config.ConfigBuilder +import nextflow.config.ModulesConfig +import nextflow.config.RegistryConfig +import nextflow.exception.AbortOperationException +import nextflow.module.ModuleReference +import nextflow.module.ModuleResolver +import nextflow.util.NextflowSpecFile + +import java.nio.file.Paths + +/** + * Module install subcommand + * + * @author Jorge Ejarque + */ +@Slf4j +@Parameters(commandDescription = "Install a module from the registry") +@CompileStatic +class ModuleInstall extends CmdBase { + + @Parameter(names = ["-version"], description = "Module version") + String version + + @Parameter(names = ["-force"], description = "Force reinstall even if already installed", arity = 0) + boolean force = false + + @Parameter(description = "[scope/name]", required = true) + List args + + @Override + String getName() { + return 'install' + } + + @Override + void run() { + if (!args || args.size() != 1) { + throw new AbortOperationException("Incorrect number of arguments") + } + + def moduleRef = '@' + args[0] + + def reference = ModuleReference.parse(moduleRef) + + // Get config + def baseDir = Paths.get('.').toAbsolutePath().normalize() + def config = new ConfigBuilder() + .setOptions(launcher.options) + .setBaseDir(baseDir) + .build() + def registryConfig = config.navigate('registry') as RegistryConfig + + //TODO: Decide final location of modules currently in nextflow_spec.json. + // Alternative: Use nextflow config. It requires to implement nextflow.config updater features + // def modulesConfig = config.navigate('modules') as ModulesConfig + def specFile = new NextflowSpecFile(baseDir) + def modulesConfig = new ModulesConfig(specFile.getModules()) + + // Create resolver and install + def resolver = new ModuleResolver(baseDir, modulesConfig, registryConfig) + + try { + def installedMainFile = resolver.installModule(reference, version, force) + + // Update nextflow_spec.json with the installed module version + def installedVersion = version ?: resolver.resolveVersion(reference) + specFile.addModuleEntry(reference.fullName, installedVersion) + + println "Module ${reference.nameWithoutPrefix}@${installedVersion} installed and configured successfully" + } + catch (AbortOperationException e) { + throw e + } + catch (Exception e) { + throw new AbortOperationException("Installation failed: ${e.message}", e) + } + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleList.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleList.groovy new file mode 100644 index 0000000000..d2fa1dec62 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleList.groovy @@ -0,0 +1,122 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli.module + +import com.beust.jcommander.Parameter +import com.beust.jcommander.Parameters +import groovy.json.JsonOutput +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.cli.CmdBase +import nextflow.exception.AbortOperationException +import nextflow.module.InstalledModule +import nextflow.module.ModuleIntegrity +import nextflow.module.ModuleStorage + +import java.nio.file.Paths + +/** + * Module list subcommand + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +@Parameters(commandDescription = "List all installed modules") +class ModuleList extends CmdBase { + + @Parameter(names = ["-json"], description = "Output in JSON format", arity=0) + boolean jsonOutput = false + + @Override + String getName() { + return 'list' + } + + @Override + void run() { + + // Get config + def baseDir = Paths.get('.').toAbsolutePath().normalize() + + + // Create resolver and list modules + def storage = new ModuleStorage(baseDir) + + try { + def installed = storage.listInstalled() + + if (installed.isEmpty()) { + println "No modules installed" + return + } + + if (jsonOutput) { + printJsonList(installed) + } else { + printFormattedList(installed) + } + } + catch (Exception e) { + log.error("Failed to list modules", e) + throw new AbortOperationException("List failed: ${e.message}", e) + } + } + + private void printFormattedList(List installed) { + println "" + println "Installed modules:" + println "" + println "Module".padRight(40) + "Version".padRight(15) + "Status" + println ("-" * 70) + + installed.each { module -> + def status = getStatusString(module.integrity) + println "${module.reference.nameWithoutPrefix.padRight(40)}${(module.installedVersion ?: 'unknown').padRight(15)}${status}" + } + println "" + } + + private void printJsonList(List installed) { + def modules = installed.collect { module -> + [ + name: module.reference.nameWithoutPrefix, + version: module.installedVersion ?: 'unknown', + integrity: module.integrity.toString(), + directory: module.directory.toString() + ] + } + + // Simple JSON output (could use groovy.json.JsonOutput for better formatting) + println JsonOutput.toJson(modules: modules) + } + + private String getStatusString(ModuleIntegrity integrity) { + switch (integrity) { + case ModuleIntegrity.VALID: + return 'OK' + case ModuleIntegrity.MODIFIED: + return 'MODIFIED' + case ModuleIntegrity.MISSING_CHECKSUM: + return 'NO CHECKSUM' + case ModuleIntegrity.CORRUPTED: + return 'CORRUPTED' + default: + return 'UNKNOWN' + } + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy new file mode 100644 index 0000000000..ec53b9d88b --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy @@ -0,0 +1,251 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli.module + +import com.beust.jcommander.Parameter +import com.beust.jcommander.Parameters +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.Const +import nextflow.cli.CmdBase +import nextflow.config.ConfigBuilder +import nextflow.config.RegistryConfig +import nextflow.exception.AbortOperationException +import nextflow.module.ModuleManifest +import nextflow.module.ModuleReference +import nextflow.module.ModuleRegistryClient +import nextflow.module.ModuleStorage + +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +/** + * Module publish subcommand + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +@Parameters(commandDescription = "Publish a module to the registry") +class ModulePublish extends CmdBase { + + @Parameter(names = ["-dry-run"], description = "Validate without uploading", arity=0) + boolean dryRun = false + + @Parameter(names = ["-registry"], description = "Target registry URL") + String registryUrl = RegistryConfig.DEFAULT_REGISTRY_URL + + @Parameter(description = "Module directory path or scope/name") + List args + + @Override + String getName() { + return 'publish' + } + + @Override + void run() { + if (!args || args.size() != 1) { + throw new AbortOperationException("Incorrect number of arguments") + } + + Path moduleDir = determineModuleDir(args[0]) + + log.info "Publishing module from: ${moduleDir}" + + // Step 1: Validate module structure + def validationErrors = validateModuleStructure(moduleDir) + if (!validationErrors.isEmpty()) { + throw new AbortOperationException( + "Module validation failed:\n" + validationErrors.collect { " - ${it}" }.join('\n') + ) + } + + // Step 2: Load and validate manifest + def manifestPath = moduleDir.resolve(ModuleStorage.MODULE_MANIFEST_FILE) + def manifest = ModuleManifest.load(manifestPath) + + def manifestErrors = manifest.validate() + if (!manifestErrors.isEmpty()) { + throw new AbortOperationException( + "Module manifest validation failed:\n" + manifestErrors.collect { " - ${it}" }.join('\n') + ) + } + + log.info "Module validated: ${manifest.name}@${manifest.version}" + + if (dryRun) { + printDryRunInfo(manifest) + return + } + + // Step 3: Get authentication token + def config = new ConfigBuilder() + .setOptions(launcher.options) + .setBaseDir(moduleDir) + .build() + + def registryConfig = config.navigate('registry') as RegistryConfig + + publishModule(moduleDir, registryConfig, manifest) + + } + + private void publishModule(Path moduleDir, RegistryConfig registryConfig, ModuleManifest manifest){ + log.info "Creating module bundle..." + def storage = new ModuleStorage(moduleDir.parent) + def tempBundleFile = Files.createTempFile("nf-module-publish-", ".tar.gz") + + try { + storage.createBundle(moduleDir, tempBundleFile) + + // Compute bundle checksum + def checksum = storage.computeBundleChecksum(tempBundleFile) + log.info "Bundle checksum: ${checksum}" + + // Read bundle content as bytes + def bundleBytes = Files.readAllBytes(tempBundleFile) + + // Create publish request as a map (npr-api will serialize it) + def request = [ + version: manifest.version, + bundle: bundleBytes + ] + + // Publish to registry + log.info "Publishing module to registry: ${registryUrl}" + def registryClient = new ModuleRegistryClient(registryConfig) + def response = registryClient.publishModule(manifest.name, request, registryUrl) + + println "✓ Module published successfully!" + println "" + println "Module details:" + println " Name: ${manifest.name}" + println " Version: ${manifest.version}" + println " DownloadUrl: ${response.downloadUrl}" + + println "" + println "Others can now install this module using:" + println " nextflow module install ${manifest.name}" + + } finally { + // Clean up temporary bundle file + if (Files.exists(tempBundleFile)) { + try { + Files.delete(tempBundleFile) + } catch (Exception e) { + log.warn "Failed to clean up temporary bundle file: ${e.message}" + } + } + } + } + + private void printDryRunInfo(ModuleManifest manifest) { + println "✓ Module structure is valid" + println "" + println "Module details:" + println " Name: ${manifest.name}" + println " Version: ${manifest.version}" + println " Description: ${manifest.description}" + println " License: ${manifest.license}" + if( manifest.authors ) { + println " Authors: ${manifest.authors.join(', ')}" + } + if( manifest.keywords ) { + println " Keywords: ${manifest.keywords.join(', ')}" + } + if( manifest.requires ) { + println " Requires:" + manifest.requires.each { name, version -> + println " - ${name}: ${version}" + } + } + println "" + println "Dry run complete. Module is ready to publish." + println "Run without --dry-run to publish to the registry." + } + + /** + * Validate that the module directory has the required structure + * + * @param moduleDir The module directory path + * @return List of validation error messages (empty if valid) + */ + private List validateModuleStructure(Path moduleDir) { + List errors = [] + + if (!Files.exists(moduleDir) || !Files.isDirectory(moduleDir)) { + errors << "Module directory does not exist: ${moduleDir}".toString() + return errors + } + + // Check for required files + def mainNf = moduleDir.resolve(Const.DEFAULT_MAIN_FILE_NAME) + if (!Files.exists(mainNf)) { + errors << "Missing required file: $Const.DEFAULT_MAIN_FILE_NAME".toString() + } + + def metaYaml = moduleDir.resolve(ModuleStorage.MODULE_MANIFEST_FILE) + if (!Files.exists(metaYaml)) { + errors << "Missing required file: $ModuleStorage.MODULE_MANIFEST_FILE".toString() + } + + def readme = moduleDir.resolve(ModuleStorage.MODULE_README_FILE) + if (!Files.exists(readme)) { + errors << "Missing required file: $ModuleStorage.MODULE_README_FILE".toString() + } + + // Check bundle size (1MB uncompressed limit) + try { + long totalSize = Files.walk(moduleDir) + .filter { Files.isRegularFile(it) } + .mapToLong { Files.size(it) } + .sum() + + def maxSize = 1024 * 1024 // 1MB in bytes + if (totalSize > maxSize) { + def sizeMB = totalSize / (1024 * 1024) + errors << "Module size exceeds 1MB limit (current: ${String.format('%.2f', sizeMB)}MB)".toString() + } + } catch (Exception e) { + log.warn "Failed to check module size: ${e.message}" + } + + return errors + } + /** + * Determine if the specified module is a local path or a reference + * @param module + * @return + */ + private Path determineModuleDir(String module) { + //If local path exists return this path as module dir + if (Paths.get(module).exists()){ + return Paths.get(module).toAbsolutePath().normalize() + } + + final ref = ModuleReference.parse('@' + module) + final localStorage = new ModuleStorage(Paths.get('.').toAbsolutePath().normalize()) + + if (!localStorage.isInstalled(ref)){ + throw new AbortOperationException("No module diretory found for $module") + } + + return localStorage.getModuleDir(ref) + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRemove.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRemove.groovy new file mode 100644 index 0000000000..8eb7f4cb25 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRemove.groovy @@ -0,0 +1,124 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli.module + +import com.beust.jcommander.Parameter +import com.beust.jcommander.Parameters +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.cli.CmdBase +import nextflow.exception.AbortOperationException +import nextflow.module.ModuleReference +import nextflow.module.ModuleStorage +import nextflow.util.NextflowSpecFile + +import java.nio.file.Paths + +/** + * Module remove subcommand + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +@Parameters(commandDescription = "Remove an installed module") +class ModuleRemove extends CmdBase { + + @Parameter(description = "", required = true) + List args + + @Parameter(names = ["-keep-config"], description = "Remove local files but keep the entry in nextflow_spec.json", arity = 0) + boolean keepConfig = false + + @Parameter(names = ["-keep-files"], description = "Remove from config but keep local files", arity = 0) + boolean keepFiles = false + + @Override + String getName() { + return 'remove' + } + + @Override + void run() { + if (!args || args.size() != 1) { + throw new AbortOperationException("Incorrect number of arguments") + } + + // Validate flags + if (keepConfig && keepFiles) { + throw new AbortOperationException("Cannot use both -keep-config and -keep-files flags together") + } + + def moduleRef = '@' + args[0] + + def reference = ModuleReference.parse(moduleRef) + + // Get config + def baseDir = Paths.get('.').toAbsolutePath().normalize() + + //TODO: Decide final location of modules currently in nextflow_spec.json. + def specFile = new NextflowSpecFile(baseDir) + + // Create resolver and spec file manager + def storage = new ModuleStorage(baseDir) + + try { + def filesRemoved = false + def configRemoved = false + + // Remove local files unless -keep-files is set + if (!keepFiles) { + println "Removing module files for ${reference.nameWithoutPrefix}..." + filesRemoved = storage.removeModule(reference) + if (filesRemoved) { + println "Module files removed successfully" + } else { + println "Module ${reference.nameWithoutPrefix} was not installed locally" + } + } else { + println "Keeping module files for ${reference.nameWithoutPrefix} (due to -keep-files flag)" + } + + // Remove config entry unless -keep-config is set + if (!keepConfig) { + println "Removing module entry from nextflow_spec.json..." + configRemoved = specFile.removeModuleEntry(reference.fullName) + if (configRemoved) { + println "Module entry removed from configuration" + } else { + println "Module ${reference.nameWithoutPrefix} was not configured in nextflow_spec.json" + } + } else { + println "Keeping module entry in nextflow_spec.json (due to -keep-config flag)" + } + + // Summary + if (filesRemoved || configRemoved) { + println "\nModule ${reference.nameWithoutPrefix} removal completed" + } else { + println "\nModule ${reference.nameWithoutPrefix} was not found" + } + } + catch (AbortOperationException e) { + throw e + } + catch (Exception e) { + log.error("Failed to remove module", e) + throw new AbortOperationException("Removal failed: ${e.message}", e) + } + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy new file mode 100644 index 0000000000..11ae422a0c --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy @@ -0,0 +1,130 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli.module + +import com.beust.jcommander.Parameter +import com.beust.jcommander.Parameters +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.cli.CmdRun +import nextflow.config.ConfigBuilder +import nextflow.config.ModulesConfig +import nextflow.config.RegistryConfig +import nextflow.exception.AbortOperationException +import nextflow.module.ModuleReference +import nextflow.module.ModuleResolver +import nextflow.util.NextflowSpecFile + +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +/** + * Module run subcommand + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +@Parameters(commandDescription = "Run a module directly from the registry") +class ModuleRun extends CmdRun { + @Parameter(names = ["-version"], description = "Module version") + String version + + @Override + String getName() { + return 'run' + } + + @Override + void run() { + if (!args ) { + throw new AbortOperationException("Arguments not provided") + } + + // Parse module reference (first argument starting with @) + String moduleRef = '@' + args[0] + + // Parse and validate module reference + ModuleReference reference + try { + reference = ModuleReference.parse(moduleRef) + } catch (Exception e) { + throw new AbortOperationException("Invalid module reference: ${moduleRef}", e) + } + + // Get config + def baseDir = Paths.get('.').toAbsolutePath().normalize() + def config = new ConfigBuilder() + .setOptions(launcher.options) + .setBaseDir(baseDir) + .build() + + def registryConfig = config.navigate('registry') as RegistryConfig + + //TODO: Decide final location of modules currently in nextflow_spec.json. + // Alternative: Use nextflow config. It requires to implement nextflow.config updater features + // def modulesConfig = config.navigate('modules') as ModulesConfig + def specFile = new NextflowSpecFile(baseDir) + def modulesConfig = new ModulesConfig(specFile.getModules()) + + //TODO: Decide if create resolver with a temporarily storage or use current ./modules + def tempDir = Files.createTempDirectory("nf-module-run-") + def resolver = new ModuleResolver(tempDir, modulesConfig, registryConfig) + try{ + Path moduleFile = resolver.installModule(reference, version) + if( moduleFile ) { + println "Executing module..." + args[0] = moduleFile.toAbsolutePath().toString() + super.run() + } + } + catch (AbortOperationException e) { + throw e + } + catch (Exception e) { + log.error("Failed to run module", e) + throw new AbortOperationException("Module run failed: ${e.message}", e) + } + finally { + // Clean up temporary directory + if (tempDir && Files.exists(tempDir)) { + try { + deleteDirectory(tempDir) + log.debug "Cleaned up temporary directory: ${tempDir}" + } catch (Exception e) { + log.warn "Failed to clean up temporary directory: ${tempDir}", e + } + } + } + } + + /** + * Delete a directory recursively + * + * @param dir The directory to delete + */ + private void deleteDirectory(Path dir) { + if (!Files.exists(dir)) { + return + } + + Files.walk(dir) + .sorted(Comparator.reverseOrder()) + .each { Path path -> Files.delete(path) } + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy new file mode 100644 index 0000000000..9027e9a64d --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy @@ -0,0 +1,143 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli.module + +import com.beust.jcommander.Parameter +import com.beust.jcommander.Parameters +import groovy.json.JsonOutput +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.seqera.npr.api.schema.v1.ModuleSearchResult +import io.seqera.npr.api.schema.v1.SearchModulesResponse +import nextflow.cli.CmdBase +import nextflow.config.ConfigBuilder +import nextflow.config.RegistryConfig +import nextflow.exception.AbortOperationException +import nextflow.module.ModuleRegistryClient + +import java.nio.file.Paths + +/** + * Module search subcommand + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +@Parameters(commandDescription = "Search for modules in the registry") +class ModuleSearch extends CmdBase { + + @Parameter(names = ["-limit"], description = "Maximum number of results") + int limit = 20 + + @Parameter(names = ["-json"], description = "Output in JSON format", arity=0) + boolean jsonOutput = false + + @Parameter(description = "", required = true) + List args + + @Override + String getName() { + return 'search' + } + + @Override + void run() { + if (!args && args.size() != 1 ) { + throw new AbortOperationException("Unexpected number of parameters") + } + String query = args[0] + + // Get config + def baseDir = Paths.get('.').toAbsolutePath().normalize() + def config = new ConfigBuilder() + .setOptions(launcher.options) + .setBaseDir(baseDir) + .build() + + final registryConfig = config.navigate('registry') as RegistryConfig + + // Create client to seach + final client = new ModuleRegistryClient(registryConfig) + + try { + println "Searching for '${query}'..." + final results = client.search(query, limit) + + if (results.totalResults == 0 || !results.results || results.results.isEmpty()) { + println "No modules found" + return + } + + if (jsonOutput) { + printJsonResults(results) + } else { + printFormattedResults(results) + } + } + catch (AbortOperationException e) { + throw e + } + catch (Exception e) { + log.error("Failed to search modules", e) + throw new AbortOperationException("Search failed: ${e.message}", e) + } + } + + private void printFormattedResults(SearchModulesResponse response) { + println "" + println "Found ${response.totalResults} module(s):" + println "" + + response.results.each { ModuleSearchResult result -> + println " ${result.name}" + if (result.relevanceScore != null) { + println " Relevance: ${String.format('%.2f', result.relevanceScore)}" + } + if (result.description) { + println " Description: ${result.description}" + } + if (result.keywords && !result.keywords.isEmpty()) { + println " Keywords: ${result.keywords.join(', ')}" + } + if (result.tools && !result.tools.isEmpty()) { + println " Tools: ${result.tools.join(', ')}" + } + println "" + } + } + + private void printJsonResults(SearchModulesResponse response) { + final modules = response.results.collect { ModuleSearchResult result -> + [ + name: result.name, + repositoryPath: result.repositoryPath, + description: result.description, + relevanceScore: result.relevanceScore, + keywords: result.keywords, + tools: result.tools, + revoked: result.revoked + ] + } + + println JsonOutput.toJson( + query: response.query, + totalResults: response.totalResults, + results: modules + ) + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ModulesConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ModulesConfig.groovy new file mode 100644 index 0000000000..558adec552 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/config/ModulesConfig.groovy @@ -0,0 +1,93 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.config + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.config.spec.ConfigOption +import nextflow.config.spec.ConfigScope +import nextflow.config.spec.ScopeName +import nextflow.script.dsl.Description + +/** + * Configuration scope for module version declarations + * + * @author Jorge Ejarque + */ +@Slf4j +@ScopeName("modules") +@Description(""" + The `modules` scope provides module version declarations for the Nextflow module system. + Each entry maps a module reference to a specific version. +""") +@CompileStatic +class ModulesConfig implements ConfigScope { + + @ConfigOption + @Description("Module version mappings (module name -> version)") + private Map modules = [:] + + /* required by extension point -- do not remove */ + ModulesConfig() {} + + ModulesConfig(Map opts) { + if (opts) { + opts.each { key, value -> + modules[key.toString()] = value.toString() + } + } + } + + /** + * Get the configured version for a module + * + * @param moduleName The module name (e.g., "@nf-core/fastqc") + * @return The configured version, or null if not configured + */ + String getVersion(String moduleName) { + return modules.get(moduleName) + } + + /** + * Get all configured modules + * + * @return Map of module name to version + */ + Map getModules() { + return Collections.unmodifiableMap(modules) + } + + /** + * Set a module version + * + * @param moduleName The module name + * @param version The version to set + */ + void setVersion(String moduleName, String version) { + modules[moduleName] = version + } + + /** + * Check if a module version is configured + * + * @param moduleName The module name + * @return true if version is configured + */ + boolean hasVersion(String moduleName) { + return modules.containsKey(moduleName) + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/config/RegistryConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/config/RegistryConfig.groovy new file mode 100644 index 0000000000..c87f3021a1 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/config/RegistryConfig.groovy @@ -0,0 +1,132 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.config + +import groovy.transform.CompileStatic +import nextflow.config.spec.ConfigOption +import nextflow.config.spec.ConfigScope +import nextflow.config.spec.ScopeName +import nextflow.script.dsl.Description + +/** + * Configuration scope for module registry settings + * + * @author Jorge Ejarque + */ +@ScopeName("registry") +@Description(""" + The `registry` scope provides configuration for the Nextflow module registry. + This includes registry URL(s) and authentication settings. +""") +@CompileStatic +class RegistryConfig implements ConfigScope { + + final static public String DEFAULT_REGISTRY_URL = 'https://registry.nextflow.io' + + @ConfigOption + @Description("Primary registry URL") + private String url + + @ConfigOption + @Description("List of registry URLs to try in order") + private List urls + + @ConfigOption + @Description("Authentication configuration per registry (registry URL -> token)") + private Map auth + + /* required by extension point -- do not remove */ + RegistryConfig() { + this.url = DEFAULT_REGISTRY_URL + this.urls = [] + this.auth = [:] + } + + RegistryConfig(Map opts) { + this.url = opts.url ? opts.url as String : DEFAULT_REGISTRY_URL + this.urls = opts.urls ? opts.urls as List : [] + this.auth = opts.auth ? opts.auth as Map : [:] + } + + /** + * Get the primary registry URL + * + * @return The registry URL + */ + String getUrl() { + return url + } + + /** + * Get all registry URLs (primary + fallbacks) + * + * @return List of registry URLs + */ + List getAllUrls() { + List result = [] + if (urls && !urls.isEmpty()) { + result.addAll(urls) + } else if (url) { + result.add(url) + } else { + result.add(DEFAULT_REGISTRY_URL) + } + return result + } + + /** + * Get authentication token for a registry + * + * @param registryUrl The registry URL + * @return The authentication token, or null if not configured + */ + String getAuthToken(String registryUrl) { + return auth?.get(registryUrl) + } + + /** + * Get authentication token from environment variable or config + * + * @param registryUrl The registry URL + * @return The authentication token, or null if not found + */ + String getAuthTokenResolved(String registryUrl) { + // First check config + def token = getAuthToken(registryUrl) + if (token) { + // Resolve environment variable references like ${NXF_REGISTRY_TOKEN} + if (token.startsWith('${') && token.endsWith('}')) { + def envVar = token.substring(2, token.length() - 1) + token = System.getenv(envVar) + } + return token + } + + // Fallback to NXF_REGISTRY_TOKEN environment variable + return System.getenv('NXF_REGISTRY_TOKEN') + } + + /** + * Check if authentication is configured for a registry + * + * @param registryUrl The registry URL + * @return true if authentication is available + */ + boolean hasAuth(String registryUrl) { + return getAuthTokenResolved(registryUrl) != null + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy b/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy new file mode 100644 index 0000000000..96e4b988a1 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy @@ -0,0 +1,89 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import groovy.transform.CompileStatic +import groovy.transform.ToString +import groovy.util.logging.Slf4j +import io.seqera.npr.api.schema.v1.ModuleMetadata +import org.yaml.snakeyaml.Yaml + +import java.nio.file.Files +import java.nio.file.Path + +/** + * Represents a module installed in the local modules/ directory + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +@ToString(includeNames = true) +class InstalledModule { + + ModuleReference reference + Path directory + Path mainFile + Path manifestFile + Path checksumFile + String installedVersion + String expectedChecksum + + /** + * Get the integrity status of this installed module + * + * @return ModuleIntegrity status + */ + ModuleIntegrity getIntegrity() { + // Check if main.nf exists + if (!Files.exists(mainFile) || !Files.exists(manifestFile)) { + return ModuleIntegrity.CORRUPTED + } + + // Check if checksum file exists + if (!Files.exists(checksumFile)) { + return ModuleIntegrity.MISSING_CHECKSUM + } + + try { + // Compute actual checksum + def actualChecksum = ModuleChecksum.compute(directory) + + // Compare with expected + if (actualChecksum == expectedChecksum) { + return ModuleIntegrity.VALID + } else { + log.debug("Actual: $actualChecksum, expected: $expectedChecksum") + return ModuleIntegrity.MODIFIED + } + } catch (Exception e) { + log.warn "Failed to compute checksum for module ${reference.nameWithoutPrefix}: ${e.message}" + return ModuleIntegrity.CORRUPTED + } + } +} + +/** + * Module integrity status + */ +@CompileStatic +enum ModuleIntegrity { + VALID, // Checksum matches + MODIFIED, // Checksum mismatch (local changes) + MISSING_CHECKSUM, // No .checksum file + CORRUPTED // Missing required files +} diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy new file mode 100644 index 0000000000..12822f54bf --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy @@ -0,0 +1,157 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j + +import java.nio.file.Files +import java.nio.file.Path +import java.security.MessageDigest + +/** + * Utility class for computing SHA-256 checksums of module directories + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +class ModuleChecksum { + + public static final String CHECKSUM_ALGORITHM = "SHA-256" + public static final String CHECKSUM_FILE = ".checksum" + + /** + * Compute the SHA-256 checksum of a module directory + * + * @param moduleDir The module directory path + * @return The hex-encoded SHA-256 checksum + */ + static String compute(Path moduleDir) { + if (!Files.exists(moduleDir) || !Files.isDirectory(moduleDir)) { + throw new IllegalArgumentException("Module directory does not exist or is not a directory: ${moduleDir}") + } + + try { + def digest = MessageDigest.getInstance(CHECKSUM_ALGORITHM) + + // Collect all files in sorted order for consistent checksums + List files = [] + Files.walk(moduleDir) + .filter { Path path -> Files.isRegularFile(path) } + .filter { Path path -> !path.fileName.toString().equals(CHECKSUM_FILE) } + .sorted() + .each { Path path -> files.add(path) } + + // Compute checksum over all file contents + for (Path file : files) { + // Include relative path in checksum for directory structure integrity + def relativePath = moduleDir.relativize(file).toString() + digest.update(relativePath.bytes) + + // Include file contents + def bytes = Files.readAllBytes(file) + digest.update(bytes) + } + + def hashBytes = digest.digest() + return bytesToHex(hashBytes) + } + catch (Exception e) { + log.error("Failed to compute checksum for module directory: ${moduleDir}", e) + throw new RuntimeException("Failed to compute module checksum", e) + } + } + + /** + * Save a checksum to the .checksum file in the module directory + * + * @param moduleDir The module directory path + * @param checksum The checksum to save + */ + static void save(Path moduleDir, String checksum) { + def checksumFile = moduleDir.resolve(CHECKSUM_FILE) + Files.writeString(checksumFile, checksum) + } + + /** + * Load a checksum from the .checksum file in the module directory + * + * @param moduleDir The module directory path + * @return The checksum, or null if file doesn't exist + */ + static String load(Path moduleDir) { + def checksumFile = moduleDir.resolve(CHECKSUM_FILE) + if (!Files.exists(checksumFile)) { + return null + } + return checksumFile.text + } + + /** + * Verify that a module directory matches the expected checksum + * + * @param moduleDir The module directory path + * @param expectedChecksum The expected checksum + * @return true if checksums match, false otherwise + */ + static boolean verify(Path moduleDir, String expectedChecksum) { + def actualChecksum = compute(moduleDir) + return actualChecksum == expectedChecksum + } + + /** + * Compute the checksum of a single file + * + * @param file The file path + * @param type checksum algorithm (sha-256 if not provided) + * @return The hex-encoded checksum + */ + static String computeFile(Path file, String type = CHECKSUM_ALGORITHM) { + if (!Files.exists(file) || !Files.isRegularFile(file)) { + throw new IllegalArgumentException("File does not exist or is not a regular file: ${file}") + } + + try { + final digest = MessageDigest.getInstance(type) + final bytes = Files.readAllBytes(file) + digest.update(bytes) + final hashBytes = digest.digest() + return bytesToHex(hashBytes) + } + catch (Exception e) { + log.error("Failed to compute checksum for file: ${file}", e) + throw new RuntimeException("Failed to compute file checksum", e) + } + } + + /** + * Convert byte array to hex string + * + * @param bytes The byte array + * @return Hex-encoded string + */ + private static String bytesToHex(byte[] bytes) { + def hexChars = new char[bytes.length * 2] + for (int i = 0; i < bytes.length; i++) { + int v = bytes[i] & 0xFF + hexChars[i * 2] = Character.forDigit(v >>> 4, 16) + hexChars[i * 2 + 1] = Character.forDigit(v & 0x0F, 16) + } + return new String(hexChars) + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleManifest.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleManifest.groovy new file mode 100644 index 0000000000..d78cea126f --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleManifest.groovy @@ -0,0 +1,117 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.exception.AbortOperationException +import org.yaml.snakeyaml.Yaml + +import java.nio.file.Files +import java.nio.file.Path + +/** + * Represents a module manifest (meta.yaml) with validation + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +class ModuleManifest { + + String name + String version + String description + List authors + String license + List keywords + Map requires + + /** + * Load a module manifest from a meta.yaml file + * + * @param metaYamlPath Path to meta.yaml + * @return ModuleManifest instance + */ + static ModuleManifest load(Path metaYamlPath) { + if (!Files.exists(metaYamlPath)) { + throw new AbortOperationException("Module manifest not found: ${metaYamlPath}") + } + + try { + def yaml = new Yaml() + def data = yaml.load(Files.newInputStream(metaYamlPath)) as Map + + def manifest = new ModuleManifest() + manifest.name = data.name as String + manifest.version = data.version as String + manifest.description = data.description as String + manifest.authors = data.authors as List ?: [] + manifest.license = data.license as String + manifest.keywords = data.keywords as List ?: [] + manifest.requires = data.requires as Map ?: [:] + + return manifest + } + catch (Exception e) { + throw new AbortOperationException("Failed to parse module manifest: ${metaYamlPath}", e) + } + } + + /** + * Validate the module manifest for required fields + * + * @return List of validation errors (empty if valid) + */ + List validate() { + List errors = [] + + if (!name) { + errors << "Missing required field: name" + } + if (!version) { + errors << "Missing required field: version" + } + if (!description) { + errors << "Missing required field: description" + } + if (!license) { + errors << "Missing required field: license" + } + + // Validate version format (semantic versioning) + if (version && !version.matches(/^\d+\.\d+\.\d+(-[\w.-]+)?$/)) { + errors << "Invalid version format: ${version} (expected semantic versioning, e.g., 1.0.0)".toString() + } + + // Validate name format (scope/name) + if (name && !name.matches(/^[a-zA-Z0-9_-]+\/[a-zA-Z0-9_-]+$/)) { + errors << "Invalid module name format: ${name} (expected scope/name, e.g., nf-core/fastqc)".toString() + } + + return errors + } + + /** + * Check if the manifest is valid + * + * @return true if valid, false otherwise + */ + boolean isValid() { + return validate().isEmpty() + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleReference.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleReference.groovy new file mode 100644 index 0000000000..76bbbcfc37 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleReference.groovy @@ -0,0 +1,89 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import nextflow.exception.AbortOperationException + +import java.util.regex.Pattern + +/** + * Represents a reference to a module in DSL include statements + * + * @author Jorge Ejarqe + */ +@CompileStatic +@EqualsAndHashCode +class ModuleReference { + + // Pattern allows: optional @, scope with letters/digits/hyphens/dots/underscores, name segments separated by slashes (no trailing slash) + // Scope: starts with letter/digit, followed by letters/digits/dots/underscores/hyphens + // Name: one or more segments (each starting with letter, followed by letters/digits/underscores/hyphens), separated by slashes + private static final Pattern MODULE_NAME_PATTERN = ~/^@?([a-z0-9][a-z0-9._\-]*)\/([a-z][a-z0-9_\-]*(?:\/[a-z][a-z0-9_\-]*)*)$/ + + final String scope + final String name + final String fullName + + ModuleReference(String scope, String name) { + this.scope = scope + this.name = name + this.fullName = "@${scope}/${name}" + } + + /** + * Parse a module reference from a string in "@scope/name" or "scope/name" format + * + * @param source The module reference string + * @return A ModuleReference object + * @throws AbortOperationException if the format is invalid + */ + static ModuleReference parse(String source) { + if (!source) { + throw new AbortOperationException("Module reference cannot be empty") + } + + // Trim whitespace + source = source.trim() + + def matcher = MODULE_NAME_PATTERN.matcher(source) + if (!matcher.matches()) { + throw new AbortOperationException( + "Invalid module reference: '${source}'. " + + "Expected format: [@]scope/name where scope is lowercase alphanumeric with dots/underscores/hyphens " + + "and name is lowercase alphanumeric with underscores/hyphens, optionally with slash-separated segments" + ) + } + + return new ModuleReference(matcher.group(1), matcher.group(2)) + } + + /** + * Get the module name without the @ prefix + * + * @return Module name in format "scope/name" + */ + String getNameWithoutPrefix() { + return "${scope}/${name}" + } + + @Override + String toString() { + return fullName + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy new file mode 100644 index 0000000000..960ee9b03e --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy @@ -0,0 +1,481 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import com.google.gson.Gson +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.seqera.http.HxClient +import io.seqera.npr.api.schema.v1.Module +import io.seqera.npr.api.schema.v1.ModuleRelease +import io.seqera.npr.api.schema.v1.PublishModuleResponse +import io.seqera.npr.api.schema.v1.SearchModulesResponse +import nextflow.config.RegistryConfig +import nextflow.exception.AbortOperationException +import nextflow.serde.gson.GsonEncoder +import nextflow.util.RetryConfig + +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.nio.file.Files +import java.nio.file.Path + +/** + * REST API client for Nextflow module registry using npr-api models + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +class ModuleRegistryClient { + + private final RegistryConfig config + private final HxClient httpClient + + ModuleRegistryClient(RegistryConfig config) { + this.config = config ?: new RegistryConfig() + this.httpClient = HxClient.newBuilder() + .retryConfig(RetryConfig.config()) + .followRedirects(HttpClient.Redirect.NORMAL) + .build() + } + + private String encodeName(String name) { + return URLEncoder.encode( + name.startsWith('@') ? name.substring(1) : name, + 'UTF-8' + ) + } + + /** + * Fetch module metadata from the registry + * + * @param name The module name (e.g., "nf-core/fastqc") + * @return Module object with metadata + */ + Module fetchModule(String name) { + def registryUrls = config.allUrls + + Exception lastError = null + for (String registryUrl : registryUrls) { + log.debug "Trying to fetch from $registryUrl" + try { + return fetchModuleFromRegistry(registryUrl, name) + } catch (Exception e) { + log.debug "Failed to fetch module from ${registryUrl}: ${e.message}" + lastError = e + } + } + + throw new AbortOperationException( + "Unable to fetch module ${name} from any configured registry", + lastError + ) + } + + /** + * Fetch module from a specific registry URL + */ + private Module fetchModuleFromRegistry(String registryUrl, String name) { + def endpoint = "${registryUrl}/api/modules/${encodeName(name)}" + def uri = URI.create(endpoint) + + def requestBuilder = HttpRequest.newBuilder() + .uri(uri) + .GET() + + // Add authentication if available + log.debug "Getting auth from: ${registryUrl}" + def token = config.getAuthTokenResolved(registryUrl) + if (token) { + requestBuilder.header("Authorization", "Bearer ${token}") + } + log.debug "Building request: ${registryUrl}" + def request = requestBuilder.build() + + try { + log.debug "Fetching module from: ${uri}" + def response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()) + def body = response.body() + + log.debug "Registry request: ${response.uri()}\n- code: ${response.statusCode()}\n- body: ${body}" + + if (response.statusCode() == 404) { + throw new AbortOperationException("Module not found: ${name}") + } + + if (response.statusCode() != 200) { + throw new AbortOperationException( + "Invalid response from registry: ${uri}\n" + + "- http status: ${response.statusCode()}\n" + + "- response: ${body}" + ) + } + + // Parse response using npr-api Module model + def encoder = new GsonEncoder() {} + return encoder.decode(body) + } + catch (AbortOperationException e) { + throw e + } + catch (Exception e) { + e.printStackTrace() + throw new AbortOperationException("Failed to fetch module from: ${uri}", e) + } + } + + /** + * Fetch specific module version/release + * + * @param name The module name + * @param version The version string + * @return ModuleRelease object from npr-api + */ + ModuleRelease fetchRelease(String name, String version) { + def registryUrls = config.allUrls + + Exception lastError = null + for (String registryUrl : registryUrls) { + try { + return fetchReleaseFromRegistry(registryUrl, name, version) + } catch (Exception e) { + log.debug "Failed to fetch release from ${registryUrl}: ${e.message}" + lastError = e + } + } + + throw new AbortOperationException( + "Unable to fetch module ${name}@${version} from any configured registry", + lastError + ) + } + + /** + * Fetch release from a specific registry URL + */ + private ModuleRelease fetchReleaseFromRegistry(String registryUrl, String name, String version) { + def endpoint = "${registryUrl}/api/modules/${encodeName(name)}/${version}" + def uri = URI.create(endpoint) + + def requestBuilder = HttpRequest.newBuilder() + .uri(uri) + .GET() + + def token = config.getAuthTokenResolved(registryUrl) + if (token) { + requestBuilder.header("Authorization", "Bearer ${token}") + } + + def request = requestBuilder.build() + + try { + log.debug "Fetching module release from: ${uri}" + def response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()) + def body = response.body() + + if (response.statusCode() == 404) { + throw new AbortOperationException("Module version not found: ${name}@${version}") + } + + if (response.statusCode() != 200) { + throw new AbortOperationException( + "Invalid response from registry: ${uri}\n" + + "- http status: ${response.statusCode()}\n" + + "- response: ${body}" + ) + } + + // Parse response using npr-api ModuleRelease model + return new GsonEncoder() {}.decode(body) + } + catch (AbortOperationException e) { + throw e + } + catch (Exception e) { + throw new AbortOperationException("Failed to fetch module release from: ${uri}", e) + } + } + + /** + * Download a module bundle from the registry + * + * @param name The module name + * @param version The module version + * @param targetPath The target path to download to + * @return Path with the downloaded file path + */ + Path downloadModule(String name, String version, Path targetPath) { + def registryUrls = config.allUrls + if (targetPath.exists()){ + targetPath.delete() + } + Exception lastError = null + for (String registryUrl : registryUrls) { + try { + return downloadModuleFromRegistry(registryUrl, name, version, targetPath) + } catch (Exception e) { + log.debug "Failed to download from ${registryUrl}: ${e.message}" + lastError = e + } + } + + throw new AbortOperationException( + "Unable to download module ${name}@${version} from any configured registry", + lastError + ) + } + + /** + * Download module from a specific registry URL + */ + private Path downloadModuleFromRegistry(String registryUrl, String name, String version, Path targetPath) { + def endpoint = "${registryUrl}/api/modules/${encodeName(name)}/${version}/download" + def uri = URI.create(endpoint) + + def requestBuilder = HttpRequest.newBuilder() + .uri(uri) + .GET() + + def token = config.getAuthTokenResolved(registryUrl) + if (token) { + requestBuilder.header("Authorization", "Bearer ${token}") + } + + def request = requestBuilder.build() + + try { + log.debug "Downloading module from: ${uri}" + def response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()) + + if (response.statusCode() == 404) { + throw new AbortOperationException("Module bundle not found: ${name}@${version}") + } + + if (response.statusCode() != 200) { + throw new AbortOperationException( + "Invalid response from registry: ${uri}\n" + + "- http status: ${response.statusCode()}" + ) + } + + // Create parent directories if needed + if (targetPath.parent) { + Files.createDirectories(targetPath.parent) + } + + // Write response body to file + Files.copy(response.body(), targetPath) + log.debug "Downloaded module to: ${targetPath}" + + validateDownloadIntegrity(response, uri, targetPath, name, version) + + return targetPath + } + catch (AbortOperationException e) { + throw e + } + catch (Exception e) { + throw new AbortOperationException("Failed to download module from: ${uri}", e) + } + } + + private void validateDownloadIntegrity(HttpResponse response, uri, Path targetPath, String name, String version) { + // Get checksum from headers (X-Checksum or Docker-Content-Digest) + def checksumType = ModuleChecksum.CHECKSUM_ALGORITHM + def checksum = response.headers().firstValue("X-Checksum").orElse(null) + + if( !checksum ) { + checksum = response.headers().firstValue("Docker-Content-Digest").orElse(null) + } + + if( !checksum ) { + log.warn "No X-Checksum or Docker-Content-Digest header found in response from ${uri}" + return + } + + // Check if checksum has a digest format including algorithm: "sha256:abc123..." + def parts = checksum.split(':', 2) + if( parts.length == 2 ) { + checksumType = parts[0].toLowerCase() + checksum = parts[1] + } + log.debug "Using checksum: ${checksumType}:${checksum}" + + def actualChecksum = ModuleChecksum.computeFile(targetPath, checksumType) + if( actualChecksum != checksum ) { + // Clean up downloaded file + Files.delete(targetPath) + throw new AbortOperationException( + "Downloaded module checksum mismatch for ${name}@${version}:\n" + + "- expected (${checksumType}): ${checksum}\n" + + "- actual: ${actualChecksum}\n" + + "The download may be corrupted or tampered with." + ) + } + log.debug "Checksum validated successfully: ${checksumType}:${checksum}" + } + + /** + * Search for modules in the registry + * + * @param query The search query + * @param limit Maximum number of results (default: 20) + * @return SearchModulesResponse with results + */ + SearchModulesResponse search(String query, int limit = 20) { + def registryUrls = config.allUrls + + Exception lastError = null + for (String registryUrl : registryUrls) { + try { + return searchInRegistry(registryUrl, query, limit) + } catch (Exception e) { + log.debug "Failed to search in ${registryUrl}: ${e.message}" + lastError = e + } + } + + throw new AbortOperationException( + "Unable to search modules in any configured registry", + lastError + ) + } + + /** + * Search in a specific registry + */ + private SearchModulesResponse searchInRegistry(String registryUrl, String query, int limit) { + def endpoint = "${registryUrl}/api/modules?query=${URLEncoder.encode(query, 'UTF-8')}&limit=${limit}" + def uri = URI.create(endpoint) + + def requestBuilder = HttpRequest.newBuilder() + .uri(uri) + .GET() + + def token = config.getAuthTokenResolved(registryUrl) + if (token) { + requestBuilder.header("Authorization", "Bearer ${token}") + } + + def request = requestBuilder.build() + + try { + log.debug "Searching modules: ${uri}" + def response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()) + def body = response.body() + + if (response.statusCode() != 200) { + throw new AbortOperationException( + "Invalid response from registry: ${uri}\n" + + "- http status: ${response.statusCode()}\n" + + "- response: ${body}" + ) + } + + // Parse response using npr-api SearchModulesResponse model + def encoder = new GsonEncoder() {} + return encoder.decode(body) + } + catch (AbortOperationException e) { + throw e + } + catch (Exception e) { + throw new AbortOperationException("Failed to search modules in: ${uri}", e) + } + } + + /** + * Publish a module to the registry (authenticated) + * + * @param name The module name + * @param request The publish request from npr-api + * @param authToken The authentication token + * @return PublishModuleResponse from npr-api + */ + PublishModuleResponse publishModule(String name, def request, String registry = null) { + final registryUrl = registry ?: config.url + final authToken = config.getAuthTokenResolved(registryUrl) + + if (!authToken) { + throw new AbortOperationException( + "Authentication required to publish modules.\n" + + "Please set NXF_REGISTRY_TOKEN environment variable or configure registry.auth in nextflow.config:\n\n" + + " registry {\n" + + " auth {\n" + + " '${registryUrl}' = '\${NXF_REGISTRY_TOKEN}'\n" + + " }\n" + + " }\n" + ) + } + try { + return publishModuleToRegistry(registryUrl, name, request, authToken) + } catch( Exception e ) { + throw new AbortOperationException("Failed to publish to ${registryUrl}", e) + } + } + + /** + * Publish module to a specific registry + */ + private PublishModuleResponse publishModuleToRegistry( + String registryUrl, + String name, + def request, + String authToken) { + + String endpoint = "${registryUrl}/api/modules/${encodeName(name)}".toString() + URI uri = URI.create(endpoint) + + // Serialize request to JSON + def gson = new Gson() + String requestBody = gson.toJson(request) + + HttpRequest httpRequest = HttpRequest.newBuilder() + .uri(uri) + .header("Content-Type", "application/json") + .header("Authorization", "Bearer ${authToken}".toString()) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build() + + try { + log.debug "Publishing module to: ${uri}" + log.trace "Request: \n\t${httpRequest}\n\theaders: ${httpRequest.headers()}\n\tbody: ${requestBody}" + + HttpResponse response = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()) + String body = response.body() + + if (response.statusCode() != 201) { + throw new AbortOperationException( + "Failed to publish module: ${uri}\n" + + "- http status: ${response.statusCode()}\n" + + "- response: ${body}" + ) + } + + // Parse response using npr-api PublishModuleResponse model + return new GsonEncoder() {}.decode(body) + } + catch (AbortOperationException e) { + throw e + } + catch (Exception e) { + throw new AbortOperationException("Failed to publish module to: ${uri}", e) + } + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy new file mode 100644 index 0000000000..1720382127 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy @@ -0,0 +1,168 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.config.ModulesConfig +import nextflow.config.RegistryConfig +import nextflow.exception.AbortOperationException + +import java.nio.file.Files +import java.nio.file.Path + +/** + * Core module resolution logic that coordinates registry, storage, and version management + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +class ModuleResolver { + + private final ModuleRegistryClient registryClient + private final ModuleStorage storage + private final ModulesConfig modulesConfig + private final RegistryConfig registryConfig + + ModuleResolver(Path baseDir, ModulesConfig modulesConfig = null, RegistryConfig registryConfig = null) { + this.registryConfig = registryConfig ?: new RegistryConfig() + this.registryClient = new ModuleRegistryClient(this.registryConfig) + this.storage = new ModuleStorage(baseDir) + this.modulesConfig = modulesConfig ?: new ModulesConfig() + } + + /** + * Resolve a module reference to an installed module path + * + * @param reference The module reference + * @param version Optional specific version (null = use config or latest) + * @param autoInstall Whether to auto-install if not present (default: false) + * @return Path to the module's main.nf file + */ + Path resolve(ModuleReference reference, String version = null, boolean autoInstall = false) { + // Determine version: explicit > config > latest + def targetVersion = version ?: modulesConfig.getVersion(reference.fullName) + + // Check if module is already installed + def installed = storage.getInstalledModule(reference) + + if (installed) { + // Check integrity + def integrity = installed.integrity + if (integrity == ModuleIntegrity.CORRUPTED) { + throw new AbortOperationException( + "Module ${reference.nameWithoutPrefix} is corrupted (missing required files). " + + "Please remove and reinstall." + ) + } + + if (integrity == ModuleIntegrity.MODIFIED) { + log.warn "Module ${reference.nameWithoutPrefix} has local modifications (checksum mismatch)" + } + + // Check if version matches + if (targetVersion && installed.installedVersion != targetVersion) { + if (autoInstall) { + log.info "Upgrading module ${reference.nameWithoutPrefix} from ${installed.installedVersion} to ${targetVersion}" + return installModule(reference, targetVersion) + } else { + throw new AbortOperationException( + "Module ${reference.nameWithoutPrefix} version mismatch: " + + "installed=${installed.installedVersion}, required=${targetVersion}. " + + "Run 'nextflow module install ${reference.nameWithoutPrefix}@${targetVersion}' to update." + ) + } + } + + // Module is installed and version matches + return installed.mainFile + } + + // Module not installed + if (autoInstall) { + return installModule(reference, targetVersion) + } else { + throw new AbortOperationException( + "Module ${reference.nameWithoutPrefix} is not installed. " + + "Run 'nextflow module install ${reference.nameWithoutPrefix}' to install." + ) + } + } + + String resolveVersion(ModuleReference reference){ + final version = modulesConfig.getVersion(reference.fullName) + ?: registryClient.fetchModule(reference.fullName).latest?.version + if (!version) { + throw new AbortOperationException("Module ${reference.nameWithoutPrefix} has no published versions") + } + return version + } + + /** + * Install or update a module + * + * @param reference The module reference + * @param version Optional specific version (null = latest) + * @param force Force reinstall even if already installed + * @return Path to the installed module's main.nf file + */ + Path installModule(ModuleReference reference, String version = null, boolean force = false) { + if (!version) + version = resolveVersion(reference) + // Check if already installed + if (storage.isInstalled(reference)) { + def installed = storage.getInstalledModule(reference) + if (installed.installedVersion == version) { + log.info "Module ${reference.nameWithoutPrefix}@${installed.installedVersion} is already installed (version $version)" + return installed.mainFile + } + + // No desired version, check for local modifications + def integrity = installed.integrity + if (integrity == ModuleIntegrity.MODIFIED && !force) { + throw new AbortOperationException( + "Module ${reference.nameWithoutPrefix} has local modifications. " + + "Use --force to override, or save your changes first." + ) + } + } + + + log.info "Installing module ${reference.nameWithoutPrefix}@${version}..." + + // Download module package to temporary location + Path tempFile = Files.createTempFile("nf-module-", ".tgz") + try { + // Download and validate integrity using server checksum + def downloadResult = registryClient.downloadModule(reference.fullName, version, tempFile) + + // Install to modules directory (will compute directory checksum for future integrity checks) + InstalledModule installed = storage.installModule(reference, version, tempFile) + + log.info "Module ${reference.nameWithoutPrefix}@${version} installed successfully at ${installed.mainFile.parent}" + return installed.mainFile + } + finally { + // Clean up temporary file + if (Files.exists(tempFile)) { + Files.delete(tempFile) + } + } + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy new file mode 100644 index 0000000000..df4a360599 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy @@ -0,0 +1,425 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.Const +import nextflow.exception.AbortOperationException +import nextflow.file.FileHelper +import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream + +import java.nio.file.Files +import java.nio.file.Path +import java.util.stream.Stream +import java.util.zip.ZipEntry +import java.util.zip.ZipInputStream + +/** + * Manages local filesystem storage for modules + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +class ModuleStorage { + public static final String MODULE_MANIFEST_FILE = "meta.yml" + public static final String MODULE_README_FILE = "README.md" + private final Path modulesDir + + /** + * Create a ModuleStorage instance + * + * @param baseDir The base directory (usually project root) + */ + ModuleStorage(Path baseDir) { + this.modulesDir = baseDir.resolve('modules') + } + + /** + * Get the modules directory path + * + * @return The modules directory + */ + Path getModulesDir() { + return modulesDir + } + + /** + * Get the directory path for a specific module + * + * @param reference The module reference + * @return The module directory path + */ + Path getModuleDir(ModuleReference reference) { + return modulesDir.resolve("@${reference.scope}").resolve(reference.name) + } + + /** + * Check if a module is installed locally + * + * @param reference The module reference + * @return true if the module directory exists + */ + boolean isInstalled(ModuleReference reference) { + def moduleDir = getModuleDir(reference) + return Files.exists(moduleDir) && Files.isDirectory(moduleDir) + } + + /** + * Get an installed module + * + * @param reference The module reference + * @return InstalledModule object, or null if not installed + */ + InstalledModule getInstalledModule(ModuleReference reference) { + def moduleDir = getModuleDir(reference) + if (!Files.exists(moduleDir) || !Files.isDirectory(moduleDir)) { + return null + } + + def installed = new InstalledModule( + reference: reference, + directory: moduleDir, + mainFile: moduleDir.resolve(Const.DEFAULT_MAIN_FILE_NAME), + manifestFile: moduleDir.resolve(MODULE_MANIFEST_FILE), + checksumFile: moduleDir.resolve(ModuleChecksum.CHECKSUM_FILE), + ) + + // Load checksum if available + installed.expectedChecksum = ModuleChecksum.load(moduleDir) + installed.installedVersion = ModuleManifest.load(installed.manifestFile).version + return installed + } + + /** + * List all installed modules + * + * @return List of InstalledModule objects + */ + List listInstalled() { + if (!Files.exists(modulesDir) || !Files.isDirectory(modulesDir)) { + return [] + } + + List modules = [] + + // Iterate over scope directories + Files.list(modulesDir).each { Path scopeDir -> + if (!Files.isDirectory(scopeDir)) return + + def scopeDirName = scopeDir.fileName.toString() + // Remove @ prefix from directory name to get scope + def scope = scopeDirName.startsWith('@') ? scopeDirName.substring(1) : scopeDirName + + // Iterate over module directories within scope + Files.list(scopeDir).each { Path moduleDir -> + if (!Files.isDirectory(moduleDir)) return + + def name = moduleDir.fileName.toString() + def reference = new ModuleReference(scope, name) + + def installed = getInstalledModule(reference) + if (installed) { + modules.add(installed) + } + } + } + + return modules + } + + /** + * Install a module from a downloaded package file + * + * @param reference The module reference + * @param version The module version + * @param packageFile The downloaded package file (zip or tar.gz) + * @return The InstalledModule object + */ + InstalledModule installModule(ModuleReference reference, String version, Path packageFile) { + def moduleDir = getModuleDir(reference) + + try { + // Remove existing installation if present + if (Files.exists(moduleDir)) { + log.debug "Removing existing module installation: ${moduleDir}" + FileHelper.deletePath(moduleDir) + } + + // Create module directory + Files.createDirectories(moduleDir) + + // Extract package - detect format by file extension + if (packageFile.toString().endsWith('.tgz') || packageFile.toString().endsWith('.tar.gz')) { + extractTarGz(packageFile, moduleDir) + } else { + extractZip(packageFile, moduleDir) + } + + // Compute and save checksum of extracted directory contents + // This checksum is used to detect local modifications + def checksum = ModuleChecksum.compute(moduleDir) + ModuleChecksum.save(moduleDir, checksum) + + log.debug "Installed module ${reference.nameWithoutPrefix}@${version} to ${moduleDir}" + + return getInstalledModule(reference) + } + catch (Exception e) { + // Clean up on failure + if (Files.exists(moduleDir)) { + try { + FileHelper.deletePath(moduleDir) + } catch (Exception cleanupError) { + log.warn "Failed to clean up after installation failure: ${cleanupError.message}" + } + } + throw new AbortOperationException("Failed to install module ${reference.nameWithoutPrefix}@${version}", e) + } + } + + /** + * Remove an installed module + * + * @param reference The module reference + * @return true if module was removed, false if not installed + */ + boolean removeModule(ModuleReference reference) { + def moduleDir = getModuleDir(reference) + + if (!Files.exists(moduleDir)) { + return false + } + + try { + FileHelper.deletePath(moduleDir) + log.debug "Removed module: ${reference.nameWithoutPrefix}" + + // Clean up empty scope directory + def scopeDir = moduleDir.parent + if (Files.exists(scopeDir) && isEmpty(scopeDir)) { + Files.delete(scopeDir) + } + + return true + } + catch (Exception e) { + throw new AbortOperationException("Failed to remove module ${reference.nameWithoutPrefix}", e) + } + } + + /** + * Extract a zip file to a target directory + * + * @param zipFile The zip file path + * @param targetDir The target directory + */ + private void extractZip(Path zipFile, Path targetDir) { + Files.newInputStream(zipFile).withCloseable { fis -> + new ZipInputStream(fis).withCloseable { zis -> + ZipEntry entry + while ((entry = zis.nextEntry) != null) { + def targetPath = targetDir.resolve(entry.name) + + // Security check: prevent zip slip + if (!targetPath.normalize().startsWith(targetDir.normalize())) { + throw new AbortOperationException("Invalid zip entry: ${entry.name}") + } + + if (entry.directory) { + Files.createDirectories(targetPath) + } else { + // Create parent directories if needed + if (targetPath.parent) { + Files.createDirectories(targetPath.parent) + } + + // Write file + Files.copy(zis, targetPath) + } + + zis.closeEntry() + } + } + } + } + + /** + * Extract a tar.gz file to a target directory + * + * @param tarGzFile The tar.gz file path + * @param targetDir The target directory + */ + private void extractTarGz(Path tarGzFile, Path targetDir) { + Files.newInputStream(tarGzFile).withCloseable { fis -> + new GzipCompressorInputStream(fis).withCloseable { gzis -> + new TarArchiveInputStream(gzis).withCloseable { tis -> + TarArchiveEntry entry + while ((entry = tis.nextTarEntry) != null) { + def targetPath = targetDir.resolve(entry.name) + + // Security check: prevent tar slip + if (!targetPath.normalize().startsWith(targetDir.normalize())) { + throw new AbortOperationException("Invalid tar entry: ${entry.name}") + } + + if (entry.directory) { + Files.createDirectories(targetPath) + } else { + // Create parent directories if needed + if (targetPath.parent) { + Files.createDirectories(targetPath.parent) + } + + // Write file + Files.copy(tis, targetPath) + } + } + } + } + } + } + + /** + * Check if a directory is empty + * + * @param dir The directory to check + * @return true if directory is empty + */ + private boolean isEmpty(Path dir) { + if (!Files.exists(dir) || !Files.isDirectory(dir)) { + return true + } + + try { + Stream entries = Files.list(dir) + try { + return !entries.findFirst().isPresent() + } finally { + entries.close() + } + } catch (Exception e) { + log.warn "Failed to check if directory is empty: ${dir}", e + return false + } + } + + /** + * Create a module bundle (tar.gz) from a module directory for publishing + * + * @param moduleDir The module directory to bundle + * @param targetFile The target bundle file path + * @return The created bundle file with its checksum + */ + Path createBundle(Path moduleDir, Path targetFile) { + if (!Files.exists(moduleDir) || !Files.isDirectory(moduleDir)) { + throw new AbortOperationException("Module directory not found: ${moduleDir}") + } + + try { + // Create parent directories if needed + if (targetFile.parent) { + Files.createDirectories(targetFile.parent) + } + + // Create tar.gz bundle + Files.newOutputStream(targetFile).withCloseable { fos -> + new GzipCompressorOutputStream(fos).withCloseable { gzos -> + new TarArchiveOutputStream(gzos).withCloseable { tos -> + // Add all files in module directory to bundle + addToTarArchive(tos, moduleDir, moduleDir) + } + } + } + + log.debug "Created module bundle: ${targetFile} (size: ${Files.size(targetFile)} bytes)" + return targetFile + } + catch (Exception e) { + // Clean up partial file on failure + if (Files.exists(targetFile)) { + try { + Files.delete(targetFile) + } catch (Exception cleanupError) { + log.warn "Failed to clean up after bundle creation failure: ${cleanupError.message}" + } + } + throw new AbortOperationException("Failed to create module bundle", e) + } + } + + /** + * Add files to tar archive recursively + * + * @param tos The tar archive output stream + * @param sourceDir The source directory being archived + * @param currentPath The current path being added + */ + private void addToTarArchive(TarArchiveOutputStream tos, Path sourceDir, Path currentPath) { + Files.list(currentPath).each { Path path -> + // Skip .checksum file when creating bundle + if (path.fileName.toString() == ModuleChecksum.CHECKSUM_FILE) { + return + } + + def relativePath = sourceDir.relativize(path).toString() + + if (Files.isDirectory(path)) { + // Add directory entry + def entry = new TarArchiveEntry(path.toFile(), "${relativePath}/") + tos.putArchiveEntry(entry) + tos.closeArchiveEntry() + + // Recursively add directory contents + addToTarArchive(tos, sourceDir, path) + } else { + // Add file entry + def entry = new TarArchiveEntry(path.toFile(), relativePath) + entry.setSize(Files.size(path)) + tos.putArchiveEntry(entry) + + // Copy file content + Files.copy(path, tos) + tos.closeArchiveEntry() + } + } + } + + /** + * Compute the checksum of a module bundle + * + * @param bundleFile The bundle file + * @return The SHA-256 checksum as hex string + */ + String computeBundleChecksum(Path bundleFile) { + if (!Files.exists(bundleFile)) { + throw new AbortOperationException("Bundle file not found: ${bundleFile}") + } + + try { + return ModuleChecksum.computeFile(bundleFile) + } + catch (Exception e) { + throw new AbortOperationException("Failed to compute bundle checksum", e) + } + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/util/NextflowSpecFile.groovy b/modules/nextflow/src/main/groovy/nextflow/util/NextflowSpecFile.groovy new file mode 100644 index 0000000000..07d34eab44 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/util/NextflowSpecFile.groovy @@ -0,0 +1,147 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.util + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j + +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption + +/** + * Manages the nextflow_spec.json file for module version declarations + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +class NextflowSpecFile { + + private static final String SPEC_FILE_NAME = 'nextflow_spec.json' + + private final Path baseDir + private final Path specFile + + NextflowSpecFile(Path baseDir) { + this.baseDir = baseDir + this.specFile = baseDir.resolve(SPEC_FILE_NAME) + } + + /** + * Add or update a module entry in the spec file + * + * @param moduleName The module name (e.g., "@nf-core/fastqc" or "nf-core/fastqc") + * @param version The module version + */ + void addModuleEntry(String moduleName, String version) { + // Normalize module name (ensure it starts with @) + def normalizedName = moduleName.startsWith('@') ? moduleName : '@' + moduleName + + def spec = readSpecFile() + + // Ensure modules map exists + if (!spec.modules) { + spec.modules = [:] + } + + // Check if already configured with same version + if (spec.modules[normalizedName] == version) { + log.info "Module ${normalizedName} already configured with version ${version} in ${SPEC_FILE_NAME}" + return + } + + // Add or update entry + spec.modules[normalizedName] = version + writeSpecFile(spec) + log.info "Added ${normalizedName}@${version} to ${SPEC_FILE_NAME}" + } + + /** + * Remove a module entry from the spec file + * + * @param moduleName The module name (e.g., "@nf-core/fastqc" or "nf-core/fastqc") + * @return true if entry was removed, false if it didn't exist + */ + boolean removeModuleEntry(String moduleName) { + // Normalize module name (ensure it starts with @) + def normalizedName = moduleName.startsWith('@') ? moduleName : '@' + moduleName + + def spec = readSpecFile() + + if (!spec.modules) { + return false + } + final modules = spec.modules as Map + modules.remove(normalizedName) + writeSpecFile(spec) + log.info "Removed ${normalizedName} from ${SPEC_FILE_NAME}" + return true + } + /** + * @return Modules Map stored in the spec file + */ + Map getModules() { + def spec = readSpecFile() + + if (!spec.modules) { + return [:] + } + return spec.modules as Map + } + + /** + * Check if the spec file exists + * + * @return true if the file exists + */ + boolean exists() { + return Files.exists(specFile) + } + + private Map readSpecFile() { + if (!Files.exists(specFile)) { + return [:] + } + + try { + def content = Files.readString(specFile) + if (content.trim().isEmpty()) { + return [:] + } + return new JsonSlurper().parseText(content) as Map + } catch (Exception e) { + throw new RuntimeException("Failed to read spec file ${specFile}: ${e.message}", e) + } + } + + private void writeSpecFile(Map spec) { + try { + // Create directory if it doesn't exist + if (!Files.exists(specFile.parent)) { + Files.createDirectories(specFile.parent) + } + + def jsonContent = JsonOutput.prettyPrint(JsonOutput.toJson(spec)) + Files.writeString(specFile, jsonContent, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING) + } catch (Exception e) { + throw new RuntimeException("Failed to write spec file ${specFile}: ${e.message}", e) + } + } +} \ No newline at end of file diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModulePublishTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/ModulePublishTest.groovy new file mode 100644 index 0000000000..7cf6b41a7e --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/ModulePublishTest.groovy @@ -0,0 +1,143 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli.module + +import nextflow.cli.Launcher +import spock.lang.Specification +import spock.lang.TempDir + +import java.nio.file.Files +import java.nio.file.Path + +/** + * Tests for ModulePublish command + * + * @author Jorge Ejarque + */ +class ModulePublishTest extends Specification { + + @TempDir + Path tempDir + + def 'should validate module structure' () { + given: + def moduleDir = tempDir.resolve('my-module') + Files.createDirectories(moduleDir) + + // Create required files + moduleDir.resolve('main.nf').text = 'process TEST { }' + moduleDir.resolve('README.md').text = '# Test Module' + moduleDir.resolve('meta.yml').text = ''' +name: test/module +version: 1.0.0 +description: Test module +license: MIT +''' + + and: + def cmd = new ModulePublish() + cmd.dryRun = true + cmd.args = [moduleDir.toString()] + + when: + def errors = cmd.invokeMethod('validateModuleStructure', moduleDir) + + then: + errors.isEmpty() + } + + def 'should detect missing required files' () { + given: + def moduleDir = tempDir.resolve('my-module') + Files.createDirectories(moduleDir) + + // Only create main.nf, missing meta.yaml and README.md + moduleDir.resolve('main.nf').text = 'process TEST { }' + + and: + def cmd = new ModulePublish() + + when: + def errors = cmd.invokeMethod('validateModuleStructure', moduleDir) + + then: + errors.size() == 2 + errors.any { it.contains('meta.yml') } + errors.any { it.contains('README.md') } + } + + def 'should detect oversized module' () { + given: + def moduleDir = tempDir.resolve('my-module') + Files.createDirectories(moduleDir) + + // Create required files + moduleDir.resolve('main.nf').text = 'process TEST { }' + moduleDir.resolve('README.md').text = '# Test Module' + moduleDir.resolve('meta.yml').text = ''' +name: test/module +version: 1.0.0 +description: Test module +license: MIT +''' + + // Create a large file (>1MB) + def largeFile = moduleDir.resolve('large-file.txt') + def content = 'x' * (1024 * 1024 + 1000) // 1MB + 1000 bytes + Files.writeString(largeFile, content) + + and: + def cmd = new ModulePublish() + + when: + def errors = cmd.invokeMethod('validateModuleStructure', moduleDir) + + then: + errors.size() == 1 + errors[0].contains('1MB limit') + } + + def 'should succeed in dry-run mode without authentication' () { + given: + def moduleDir = tempDir.resolve('my-module') + Files.createDirectories(moduleDir) + + // Create required files + moduleDir.resolve('main.nf').text = 'process TEST { }' + moduleDir.resolve('README.md').text = '# Test Module' + moduleDir.resolve('meta.yml').text = ''' +name: test/module +version: 1.0.0 +description: Test module +license: MIT +''' + + and: + def cmd = new ModulePublish() + def launcher = new Launcher() + launcher.options = [:] + cmd.launcher = launcher + cmd.args = [moduleDir.toString()] + cmd.dryRun = true + + when: + cmd.run() + + then: + noExceptionThrown() + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/module/InstalledModuleTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/InstalledModuleTest.groovy new file mode 100644 index 0000000000..850466ab48 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/module/InstalledModuleTest.groovy @@ -0,0 +1,201 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import java.nio.file.Files +import java.nio.file.Path + +import spock.lang.Specification + +/** + * Test suite for InstalledModule + * + * @author Jorge Ejarque + */ +class InstalledModuleTest extends Specification { + + Path tempDir + + def setup() { + tempDir = Files.createTempDirectory('nf-installed-module-test-') + } + + def cleanup() { + tempDir?.deleteDir() + } + + def 'should report VALID integrity when checksum matches'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + def mainFile = moduleDir.resolve('main.nf') + mainFile.text = 'process TEST { }' + + def metaFile = moduleDir.resolve('meta.yml') + metaFile.text = 'name: test/module\nversion: 0.0.1' + + // Compute actual checksum + def actualChecksum = ModuleChecksum.compute(moduleDir) + + // Save checksum + ModuleChecksum.save(moduleDir, actualChecksum) + + def installed = new InstalledModule( + reference: new ModuleReference('test', 'module'), + directory: moduleDir, + mainFile: mainFile, + manifestFile: metaFile, + checksumFile: moduleDir.resolve('.checksum'), + expectedChecksum: actualChecksum, + installedVersion: "0.0.1" + ) + + when: + def integrity = installed.getIntegrity() + + then: + integrity == ModuleIntegrity.VALID + } + + def 'should report MODIFIED integrity when checksum differs'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + def mainFile = moduleDir.resolve('main.nf') + mainFile.text = 'process TEST { }' + + def metaFile = moduleDir.resolve('meta.yml') + metaFile.text = 'name: test/module\nversion: 0.0.1' + + // Compute initial checksum + def originalChecksum = ModuleChecksum.compute(moduleDir) + ModuleChecksum.save(moduleDir, originalChecksum) + + // Modify the file + mainFile.text = 'process TEST { println "modified" }' + + def installed = new InstalledModule( + reference: new ModuleReference('test', 'module'), + directory: moduleDir, + mainFile: mainFile, + manifestFile: metaFile, + checksumFile: moduleDir.resolve('.checksum'), + expectedChecksum: originalChecksum, + installedVersion: "0.0.1" + ) + + when: + def integrity = installed.getIntegrity() + + then: + integrity == ModuleIntegrity.MODIFIED + } + + def 'should report CORRUPTED integrity when main.nf is missing'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + def mainFile = moduleDir.resolve('main.nf') + // Don't create main.nf + + def metaFile = moduleDir.resolve('meta.yml') + metaFile.text = 'name: test/module\nversion: 0.0.1' + + def checksumFile = moduleDir.resolve('.checksum') + checksumFile.text = 'some-checksum' + + def installed = new InstalledModule( + reference: new ModuleReference('test', 'module'), + directory: moduleDir, + mainFile: mainFile, + manifestFile: metaFile, + checksumFile: checksumFile, + expectedChecksum: 'some-checksum' + ) + + when: + def integrity = installed.getIntegrity() + + then: + integrity == ModuleIntegrity.CORRUPTED + } + + def 'should report MISSING_CHECKSUM when checksum file absent'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + def mainFile = moduleDir.resolve('main.nf') + mainFile.text = 'process TEST { }' + + def metaFile = moduleDir.resolve('meta.yml') + metaFile.text = 'name: test/module\nversion: 0.0.1' + + def checksumFile = moduleDir.resolve('.checksum') + // Don't create checksum file + + def installed = new InstalledModule( + reference: new ModuleReference('test', 'module'), + directory: moduleDir, + mainFile: mainFile, + manifestFile: metaFile, + checksumFile: checksumFile, + expectedChecksum: null + ) + + when: + def integrity = installed.getIntegrity() + + then: + integrity == ModuleIntegrity.MISSING_CHECKSUM + } + + def 'should handle checksum computation failure gracefully'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + def mainFile = moduleDir.resolve('main.nf') + mainFile.text = 'process TEST { }' + + def metaFile = moduleDir.resolve('meta.yml') + metaFile.text = 'name: test/module\nversion: 0.0.1' + + // Create checksum file with a value that won't match the computed checksum + def checksumFile = moduleDir.resolve('.checksum') + checksumFile.text = 'expected-checksum-that-will-not-match' + + def installed = new InstalledModule( + reference: new ModuleReference('test', 'module'), + directory: moduleDir, + mainFile: mainFile, + manifestFile: metaFile, + checksumFile: checksumFile, + expectedChecksum: 'expected-checksum-that-will-not-match' + ) + + when: + def integrity = installed.getIntegrity() + + then: + // Should handle gracefully - since checksum won't match, it should report as MODIFIED + integrity in [ModuleIntegrity.VALID, ModuleIntegrity.MODIFIED, ModuleIntegrity.CORRUPTED] + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleChecksumTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleChecksumTest.groovy new file mode 100644 index 0000000000..31c6600373 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleChecksumTest.groovy @@ -0,0 +1,415 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import java.nio.file.Files +import java.nio.file.Path + +import spock.lang.Specification + +/** + * Test suite for ModuleChecksum + * + * @author Jorge Ejarque + */ +class ModuleChecksumTest extends Specification { + + Path tempDir + + def setup() { + tempDir = Files.createTempDirectory('nf-checksum-test-') + } + + def cleanup() { + tempDir?.deleteDir() + } + + def 'should compute checksum for directory'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + // Create test files + moduleDir.resolve('main.nf').text = 'process TEST { }' + moduleDir.resolve('meta.yml').text = 'name: test\nversion: 1.0.0' + moduleDir.resolve('README.md').text = '# Test Module' + + when: + def checksum = ModuleChecksum.compute(moduleDir) + + then: + checksum != null + checksum.length() == 64 // SHA-256 produces 64 hex characters + checksum ==~ /^[a-f0-9]{64}$/ + } + + def 'should produce consistent checksums for same content'() { + given: + def moduleDir1 = tempDir.resolve('module1') + def moduleDir2 = tempDir.resolve('module2') + Files.createDirectories(moduleDir1) + Files.createDirectories(moduleDir2) + + // Create identical content in both directories + ['main.nf', 'meta.yml', 'README.md'].each { filename -> + moduleDir1.resolve(filename).text = "content of ${filename}" + moduleDir2.resolve(filename).text = "content of ${filename}" + } + + when: + def checksum1 = ModuleChecksum.compute(moduleDir1) + def checksum2 = ModuleChecksum.compute(moduleDir2) + + then: + checksum1 == checksum2 + } + + def 'should produce different checksums for different content'() { + given: + def moduleDir1 = tempDir.resolve('module1') + def moduleDir2 = tempDir.resolve('module2') + Files.createDirectories(moduleDir1) + Files.createDirectories(moduleDir2) + + // Create different content + moduleDir1.resolve('main.nf').text = 'process TEST1 { }' + moduleDir2.resolve('main.nf').text = 'process TEST2 { }' + + when: + def checksum1 = ModuleChecksum.compute(moduleDir1) + def checksum2 = ModuleChecksum.compute(moduleDir2) + + then: + checksum1 != checksum2 + } + + def 'should exclude .checksum file from computation'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + moduleDir.resolve('main.nf').text = 'process TEST { }' + + // Compute initial checksum + def checksum1 = ModuleChecksum.compute(moduleDir) + + // Add .checksum file + moduleDir.resolve('.checksum').text = 'some-checksum-value' + + // Compute checksum again + def checksum2 = ModuleChecksum.compute(moduleDir) + + expect: + checksum1 == checksum2 // Should be the same, .checksum is ignored + } + + def 'should include subdirectories in checksum'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + moduleDir.resolve('main.nf').text = 'process TEST { }' + + // Compute checksum without subdirectory + def checksum1 = ModuleChecksum.compute(moduleDir) + + // Add subdirectory with file + def subDir = moduleDir.resolve('templates') + Files.createDirectories(subDir) + subDir.resolve('script.sh').text = '#!/bin/bash\necho "test"' + + // Compute checksum with subdirectory + def checksum2 = ModuleChecksum.compute(moduleDir) + + expect: + checksum1 != checksum2 // Checksums should differ + } + + def 'should save checksum to .checksum file'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + def checksumValue = 'abc123def456' + + when: + ModuleChecksum.save(moduleDir, checksumValue) + + then: + def checksumFile = moduleDir.resolve('.checksum') + Files.exists(checksumFile) + checksumFile.text.trim() == checksumValue + } + + def 'should load checksum from .checksum file'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + def checksumFile = moduleDir.resolve('.checksum') + checksumFile.text = 'abc123def456' + + when: + def checksum = ModuleChecksum.load(moduleDir) + + then: + checksum == 'abc123def456' + } + + def 'should return null when loading non-existent checksum file'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + when: + def checksum = ModuleChecksum.load(moduleDir) + + then: + checksum == null + } + + def 'should handle empty directory'() { + given: + def moduleDir = tempDir.resolve('empty-module') + Files.createDirectories(moduleDir) + + when: + def checksum = ModuleChecksum.compute(moduleDir) + + then: + checksum != null + checksum.length() == 64 + } + + def 'should handle files with special characters in names'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + // Create files with special characters (but valid on filesystem) + moduleDir.resolve('main.nf').text = 'process TEST { }' + moduleDir.resolve('file with spaces.txt').text = 'content' + moduleDir.resolve('file-with-dashes.txt').text = 'content' + + when: + def checksum = ModuleChecksum.compute(moduleDir) + + then: + checksum != null + checksum.length() == 64 + } + + def 'should handle binary files'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + // Create text and binary files + moduleDir.resolve('main.nf').text = 'process TEST { }' + def binaryFile = moduleDir.resolve('data.bin') + binaryFile.bytes = [0x00, 0x01, 0x02, 0xFF] as byte[] + + when: + def checksum = ModuleChecksum.compute(moduleDir) + + then: + checksum != null + checksum.length() == 64 + } + + def 'should sort files consistently for checksum computation'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + // Create files in arbitrary order + moduleDir.resolve('zzz.nf').text = 'content' + moduleDir.resolve('aaa.nf').text = 'content' + moduleDir.resolve('mmm.nf').text = 'content' + + def checksum1 = ModuleChecksum.compute(moduleDir) + + // Create another directory with files in different order + def moduleDir2 = tempDir.resolve('module2') + Files.createDirectories(moduleDir2) + moduleDir2.resolve('aaa.nf').text = 'content' + moduleDir2.resolve('mmm.nf').text = 'content' + moduleDir2.resolve('zzz.nf').text = 'content' + + def checksum2 = ModuleChecksum.compute(moduleDir2) + + expect: + checksum1 == checksum2 // Order shouldn't matter + } + + // Tests for computeFile() method + + def 'should compute checksum for a single file with default algorithm'() { + given: + def testFile = tempDir.resolve('test.txt') + testFile.text = 'Hello, World!' + + when: + def checksum = ModuleChecksum.computeFile(testFile) + + then: + checksum != null + checksum.length() == 64 // SHA-256 produces 64 hex characters + checksum ==~ /^[a-f0-9]{64}$/ + } + + def 'should produce consistent checksums for same file content'() { + given: + def file1 = tempDir.resolve('file1.txt') + def file2 = tempDir.resolve('file2.txt') + def content = 'Same content in both files' + file1.text = content + file2.text = content + + when: + def checksum1 = ModuleChecksum.computeFile(file1) + def checksum2 = ModuleChecksum.computeFile(file2) + + then: + checksum1 == checksum2 + } + + def 'should produce different checksums for different file content'() { + given: + def file1 = tempDir.resolve('file1.txt') + def file2 = tempDir.resolve('file2.txt') + file1.text = 'Content A' + file2.text = 'Content B' + + when: + def checksum1 = ModuleChecksum.computeFile(file1) + def checksum2 = ModuleChecksum.computeFile(file2) + + then: + checksum1 != checksum2 + } + + def 'should compute checksum for empty file'() { + given: + def emptyFile = tempDir.resolve('empty.txt') + emptyFile.text = '' + + when: + def checksum = ModuleChecksum.computeFile(emptyFile) + + then: + checksum != null + checksum.length() == 64 + // SHA-256 of empty string + checksum == 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' + } + + def 'should compute checksum for binary file'() { + given: + def binaryFile = tempDir.resolve('binary.dat') + binaryFile.bytes = [0x00, 0xFF, 0x42, 0xAB, 0xCD, 0xEF] as byte[] + + when: + def checksum = ModuleChecksum.computeFile(binaryFile) + + then: + checksum != null + checksum.length() == 64 + checksum ==~ /^[a-f0-9]{64}$/ + } + + def 'should support different hash algorithms'() { + given: + def testFile = tempDir.resolve('test.txt') + testFile.text = 'Test content for different algorithms' + + when: + def sha256 = ModuleChecksum.computeFile(testFile, 'SHA-256') + def sha512 = ModuleChecksum.computeFile(testFile, 'SHA-512') + + then: + sha256 != null + sha512 != null + sha256.length() == 64 // SHA-256: 256 bits = 64 hex chars + sha512.length() == 128 // SHA-512: 512 bits = 128 hex chars + sha256 != sha512 + } + + def 'should handle case-insensitive algorithm names'() { + given: + def testFile = tempDir.resolve('test.txt') + testFile.text = 'Test content' + + when: + def checksum1 = ModuleChecksum.computeFile(testFile, 'sha-256') + def checksum2 = ModuleChecksum.computeFile(testFile, 'SHA-256') + def checksum3 = ModuleChecksum.computeFile(testFile, 'Sha-256') + + then: + checksum1 == checksum2 + checksum2 == checksum3 + } + + def 'should throw exception for non-existent file'() { + given: + def nonExistentFile = tempDir.resolve('does-not-exist.txt') + + when: + ModuleChecksum.computeFile(nonExistentFile) + + then: + thrown(IllegalArgumentException) + } + + def 'should throw exception for directory instead of file'() { + given: + def directory = tempDir.resolve('subdir') + Files.createDirectories(directory) + + when: + ModuleChecksum.computeFile(directory) + + then: + thrown(IllegalArgumentException) + } + + def 'should compute known SHA-256 checksum correctly'() { + given: + def testFile = tempDir.resolve('known.txt') + testFile.text = 'abc' + + when: + def checksum = ModuleChecksum.computeFile(testFile) + + then: + // Known SHA-256 hash of "abc" + checksum == 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad' + } + + def 'should handle large file checksum computation'() { + given: + def largeFile = tempDir.resolve('large.txt') + // Create a file with ~1MB of data + def content = 'x' * 1024 * 1024 + largeFile.text = content + + when: + def checksum = ModuleChecksum.computeFile(largeFile) + + then: + checksum != null + checksum.length() == 64 + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleManifestTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleManifestTest.groovy new file mode 100644 index 0000000000..6fbc15da65 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleManifestTest.groovy @@ -0,0 +1,161 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import nextflow.exception.AbortOperationException +import spock.lang.Specification +import spock.lang.TempDir + +import java.nio.file.Files +import java.nio.file.Path + +/** + * Tests for ModuleManifest + * + * @author Paolo Di Tommaso + */ +class ModuleManifestTest extends Specification { + + @TempDir + Path tempDir + + def 'should load valid manifest' () { + given: + def metaYaml = tempDir.resolve('meta.yaml') + metaYaml.text = ''' +name: nf-core/fastqc +version: 1.0.0 +description: FastQC quality control +authors: + - John Doe +license: MIT +keywords: + - quality-control + - fastq +requires: + nextflow: ">=24.04.0" +''' + + when: + def manifest = ModuleManifest.load(metaYaml) + + then: + manifest.name == 'nf-core/fastqc' + manifest.version == '1.0.0' + manifest.description == 'FastQC quality control' + manifest.authors == ['John Doe'] + manifest.license == 'MIT' + manifest.keywords == ['quality-control', 'fastq'] + manifest.requires == ['nextflow': '>=24.04.0'] + } + + def 'should fail to load non-existent manifest' () { + given: + def metaYaml = tempDir.resolve('meta.yaml') + + when: + ModuleManifest.load(metaYaml) + + then: + thrown(AbortOperationException) + } + + def 'should validate complete manifest' () { + given: + def manifest = new ModuleManifest( + name: 'nf-core/fastqc', + version: '1.0.0', + description: 'FastQC quality control', + license: 'MIT' + ) + + when: + def errors = manifest.validate() + + then: + errors.isEmpty() + manifest.isValid() + } + + def 'should detect missing required fields' () { + given: + def manifest = new ModuleManifest( + name: 'nf-core/fastqc' + // missing version, description, license + ) + + when: + def errors = manifest.validate() + + then: + errors.size() == 3 + errors.any { it.contains('version') } + errors.any { it.contains('description') } + errors.any { it.contains('license') } + !manifest.isValid() + } + + def 'should validate version format' () { + given: + def manifest = new ModuleManifest( + name: 'nf-core/fastqc', + version: version, + description: 'Test', + license: 'MIT' + ) + + when: + def errors = manifest.validate() + + then: + errors.isEmpty() == valid + + where: + version | valid + '1.0.0' | true + '1.0.0-alpha' | true + '1.0.0-beta.1' | true + '1.0' | false + 'v1.0.0' | false + '1.0.0.0' | false + } + + def 'should validate module name format' () { + given: + def manifest = new ModuleManifest( + name: name, + version: '1.0.0', + description: 'Test', + license: 'MIT' + ) + + when: + def errors = manifest.validate() + + then: + errors.isEmpty() == valid + + where: + name | valid + 'nf-core/fastqc' | true + 'myorg/my-module' | true + 'org_1/tool_2' | true + 'fastqc' | false + '@nf-core/fastqc' | false + 'nf-core/fast qc' | false + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleReferenceTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleReferenceTest.groovy new file mode 100644 index 0000000000..64b0613be8 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleReferenceTest.groovy @@ -0,0 +1,231 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import nextflow.exception.AbortOperationException +import spock.lang.Specification + +/** + * Test suite for ModuleReference + * + * @author Jorge Ejarque + */ +class ModuleReferenceTest extends Specification { + + def 'should parse valid module reference with @'() { + when: + def ref = ModuleReference.parse('@nf-core/fastqc') + + then: + ref.scope == 'nf-core' + ref.name == 'fastqc' + ref.fullName == '@nf-core/fastqc' + } + + def 'should parse valid module reference without @'() { + when: + def ref = ModuleReference.parse('nf-core/fastqc') + + then: + ref.scope == 'nf-core' + ref.name == 'fastqc' + ref.fullName == '@nf-core/fastqc' + } + + def 'should parse module reference with multiple slashes'() { + when: + def ref = ModuleReference.parse('@myorg/samtools/view') + + then: + ref.scope == 'myorg' + ref.name == 'samtools/view' + ref.fullName == '@myorg/samtools/view' + } + + def 'should reject invalid module reference without scope'() { + when: + ModuleReference.parse('fastqc') + + then: + thrown(AbortOperationException) + } + + def 'should reject empty module reference'() { + when: + ModuleReference.parse('') + + then: + thrown(AbortOperationException) + } + + def 'should reject null module reference'() { + when: + ModuleReference.parse(null) + + then: + thrown(AbortOperationException) + } + + def 'should reject module reference with only @'() { + when: + ModuleReference.parse('@') + + then: + thrown(AbortOperationException) + } + + def 'should reject module reference with only scope'() { + when: + ModuleReference.parse('@nf-core/') + + then: + thrown(AbortOperationException) + } + + def 'should handle module reference with trailing slash'() { + when: + ModuleReference.parse('@nf-core/fastqc/') + + then: + thrown(AbortOperationException) + } + + def 'should create module reference from components'() { + when: + def ref = new ModuleReference('nf-core', 'fastqc') + + then: + ref.scope == 'nf-core' + ref.name == 'fastqc' + ref.fullName == '@nf-core/fastqc' + } + + def 'should handle scope names with hyphens'() { + when: + def ref = ModuleReference.parse('@my-org/my-module') + + then: + ref.scope == 'my-org' + ref.name == 'my-module' + } + + def 'should handle scope names with underscores'() { + when: + def ref = ModuleReference.parse('@my_org/my_module') + + then: + ref.scope == 'my_org' + ref.name == 'my_module' + } + + def 'should handle module names with numbers'() { + when: + def ref = ModuleReference.parse('@nf-core/bwa-mem2') + + then: + ref.scope == 'nf-core' + ref.name == 'bwa-mem2' + } + + def 'should implement equals correctly'() { + given: + def ref1 = ModuleReference.parse('@nf-core/fastqc') + def ref2 = ModuleReference.parse('@nf-core/fastqc') + def ref3 = ModuleReference.parse('@nf-core/multiqc') + + expect: + ref1 == ref2 + ref1 != ref3 + } + + def 'should implement hashCode correctly'() { + given: + def ref1 = ModuleReference.parse('@nf-core/fastqc') + def ref2 = ModuleReference.parse('@nf-core/fastqc') + + expect: + ref1.hashCode() == ref2.hashCode() + } + + def 'should implement toString correctly'() { + given: + def ref = ModuleReference.parse('@nf-core/fastqc') + + expect: + ref.toString() == '@nf-core/fastqc' + } + + def 'should be usable as map key'() { + given: + def ref1 = ModuleReference.parse('@nf-core/fastqc') + def ref2 = ModuleReference.parse('@nf-core/fastqc') + def ref3 = ModuleReference.parse('@nf-core/multiqc') + + def map = [:] + map[ref1] = 'value1' + map[ref3] = 'value3' + + expect: + map[ref2] == 'value1' // ref2 equals ref1, should get same value + map[ref3] == 'value3' + map.size() == 2 + } + + def 'should handle org-style scopes'() { + when: + def ref = ModuleReference.parse('@mycompany.io/custom-module') + + then: + ref.scope == 'mycompany.io' + ref.name == 'custom-module' + } + + def 'should reject module reference with spaces'() { + when: + ModuleReference.parse('@nf-core/fast qc') + + then: + thrown(AbortOperationException) + } + + def 'should reject module reference with special characters'() { + when: + ModuleReference.parse('@nf-core/fastqc!') + + then: + thrown(AbortOperationException) + } + + def 'should handle deeply nested module names'() { + when: + def ref = ModuleReference.parse('@nf-core/samtools/sort/parallel') + + then: + ref.scope == 'nf-core' + ref.name == 'samtools/sort/parallel' + ref.fullName == '@nf-core/samtools/sort/parallel' + } + + def 'should parse from string with leading/trailing whitespace'() { + when: + def ref = ModuleReference.parse(' @nf-core/fastqc ') + + then: + ref.scope == 'nf-core' + ref.name == 'fastqc' + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy new file mode 100644 index 0000000000..6c41c40c41 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy @@ -0,0 +1,361 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.GZIPOutputStream +import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream + +import spock.lang.Specification + +/** + * Test suite for ModuleStorage + * + * @author Jorge Ejarque + */ +class ModuleStorageTest extends Specification { + + Path tempDir + + def setup() { + tempDir = Files.createTempDirectory('nf-module-storage-test-') + } + + def cleanup() { + tempDir?.deleteDir() + } + + def 'should get module directory path'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + + when: + def moduleDir = storage.getModuleDir(reference) + + then: + moduleDir == tempDir.resolve('modules/@nf-core/fastqc') + } + + def 'should check if module is installed'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def moduleDir = storage.getModuleDir(reference) + + when: + def installed = storage.isInstalled(reference) + + then: + !installed + + when: + Files.createDirectories(moduleDir) + installed = storage.isInstalled(reference) + + then: + installed + } + + def 'should return null for non-existent module'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'nonexistent') + + when: + def installed = storage.getInstalledModule(reference) + + then: + installed == null + } + + def 'should get installed module with metadata'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def moduleDir = storage.getModuleDir(reference) + Files.createDirectories(moduleDir) + + // Create main.nf + def mainFile = moduleDir.resolve('main.nf') + mainFile.text = 'process FASTQC { }' + + // Create meta.yml with version + def metaFile = moduleDir.resolve('meta.yml') + metaFile.text = ''' + name: nf-core/fastqc + version: 1.0.0 + description: FastQC quality control + keywords: + - quality-control + - fastqc + '''.stripIndent() + + // Create .checksum file + def checksumFile = moduleDir.resolve('.checksum') + checksumFile.text = 'abc123def456' + + when: + def installed = storage.getInstalledModule(reference) + + then: + installed != null + installed.reference == reference + installed.directory == moduleDir + installed.mainFile == mainFile + installed.manifestFile == moduleDir.resolve('meta.yml') + installed.checksumFile == checksumFile + installed.expectedChecksum == 'abc123def456' + installed.installedVersion == '1.0.0' + } + + def 'should list all installed modules'() { + given: + def storage = new ModuleStorage(tempDir) + + // Create multiple modules + def modules = [ + new ModuleReference('nf-core', 'fastqc'), + new ModuleReference('nf-core', 'multiqc'), + new ModuleReference('myorg', 'custom') + ] + + modules.each { ref -> + def moduleDir = storage.getModuleDir(ref) + Files.createDirectories(moduleDir) + + // Create main.nf + moduleDir.resolve('main.nf').text = 'process TEST { }' + + // Create meta.yml with version + moduleDir.resolve('meta.yml').text = """ + name: ${ref.nameWithoutPrefix} + version: 1.0.0 + """.stripIndent() + + // Create .checksum + moduleDir.resolve('.checksum').text = 'checksum' + } + + when: + def installed = storage.listInstalled() + + then: + installed.size() == 3 + installed*.reference.fullName.sort() == ['@myorg/custom', '@nf-core/fastqc', '@nf-core/multiqc'] + } + + def 'should return empty list when no modules installed'() { + given: + def storage = new ModuleStorage(tempDir) + + when: + def installed = storage.listInstalled() + + then: + installed.isEmpty() + } + + def 'should install module from gzip package'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def version = '1.0.0' + + // Create a gzip package file + def packageFile = Files.createTempFile('module-', '.tgz') + createTestPackage(packageFile) + + when: + def installed = storage.installModule(reference, version, packageFile) + + then: + installed != null + installed.reference == reference + installed.installedVersion == '1.0.0' + Files.exists(installed.mainFile) + Files.exists(installed.checksumFile) + + cleanup: + packageFile?.delete() + } + + def 'should replace existing module on install'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def moduleDir = storage.getModuleDir(reference) + + // Create existing installation + Files.createDirectories(moduleDir) + def oldFile = moduleDir.resolve('old-file.txt') + oldFile.text = 'old content' + + // Create package + def packageFile = Files.createTempFile('module-', '.tgz') + createTestPackage(packageFile) + + when: + def installed = storage.installModule(reference, '2.0.0', packageFile) + + then: + installed != null + !Files.exists(oldFile) // Old file should be removed + Files.exists(installed.mainFile) + + cleanup: + packageFile?.delete() + } + + def 'should remove installed module'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def moduleDir = storage.getModuleDir(reference) + + // Create module + Files.createDirectories(moduleDir) + moduleDir.resolve('main.nf').text = 'process TEST { }' + + expect: + Files.exists(moduleDir) + + when: + def removed = storage.removeModule(reference) + + then: + removed + !Files.exists(moduleDir) + } + + def 'should return false when removing non-existent module'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'nonexistent') + + when: + def removed = storage.removeModule(reference) + + then: + !removed + } + + def 'should compute and save checksum on install'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def packageFile = Files.createTempFile('module-', '.tgz') + createTestPackage(packageFile) + + when: + def installed = storage.installModule(reference, '1.0.0', packageFile) + + then: + installed.expectedChecksum != null + installed.expectedChecksum.length() > 0 + Files.exists(installed.checksumFile) + + cleanup: + packageFile?.delete() + } + + def 'should handle installation failure gracefully'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def invalidPackage = Files.createTempFile('invalid-', '.tgz') + invalidPackage.text = 'not a valid gzip file' + + when: + storage.installModule(reference, '1.0.0', invalidPackage) + + then: + thrown(Exception) + + and: + !storage.isInstalled(reference) // Should not leave partial installation + + cleanup: + invalidPackage?.delete() + } + + /** + * Helper method to create a test package file + */ + private void createTestPackage(Path packageFile) { + // Create a temporary directory with module content + def tempModuleDir = Files.createTempDirectory('temp-module-') + + // Create main.nf + tempModuleDir.resolve('main.nf').text = ''' + process FASTQC { + input: + path reads + + output: + path "*.html" + + script: + """ + fastqc ${reads} + """ + } + '''.stripIndent() + + // Create meta.yml + tempModuleDir.resolve('meta.yml').text = ''' + name: nf-core/fastqc + version: 1.0.0 + description: FastQC quality control + keywords: + - quality-control + - fastqc + '''.stripIndent() + + // Create README + tempModuleDir.resolve('README.md').text = '# FastQC Module' + + // Create tar.gz archive using Java libraries + Files.newOutputStream(packageFile).withCloseable { fos -> + new GZIPOutputStream(fos).withCloseable { gzos -> + new TarArchiveOutputStream(gzos).withCloseable { tos -> + // Add all files from tempModuleDir + Files.walk(tempModuleDir).each { Path path -> + if (Files.isRegularFile(path)) { + // Get relative path + def relativePath = tempModuleDir.relativize(path).toString() + + // Create tar entry + def entry = new TarArchiveEntry(path.toFile(), relativePath) + tos.putArchiveEntry(entry) + + // Write file content + Files.copy(path, tos) + + tos.closeArchiveEntry() + } + } + } + } + } + + // Cleanup temp directory + tempModuleDir.deleteDir() + } +} From f5822a52aabf773c130c8d237a40102a46e807f7 Mon Sep 17 00:00:00 2001 From: jorgee Date: Fri, 30 Jan 2026 14:05:52 +0100 Subject: [PATCH 02/23] add print of process outputs Signed-off-by: jorgee --- .../script/ProcessEntryHandler.groovy | 77 ++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessEntryHandler.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessEntryHandler.groovy index 0d80e0333c..38ee87be61 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessEntryHandler.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessEntryHandler.groovy @@ -16,6 +16,13 @@ package nextflow.script +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel +import nextflow.exception.AbortOperationException +import nextflow.extension.CH +import nextflow.extension.DataflowHelper +import nextflow.extension.DumpHelper + import java.nio.file.Path import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -29,6 +36,8 @@ import nextflow.script.params.TupleInParam import nextflow.script.params.v2.ProcessInput import nextflow.script.params.v2.ProcessTupleInput +import java.util.concurrent.atomic.AtomicInteger + /** * Helper class for process entry execution feature. * @@ -47,6 +56,8 @@ class ProcessEntryHandler { private final BaseScript script private final Session session private final ScriptMeta meta + // Map to store process outputs + private Map processOutputs ProcessEntryHandler(BaseScript script, Session session, ScriptMeta meta) { this.script = script @@ -86,7 +97,7 @@ class ProcessEntryHandler { // Get input parameter values and execute the process final inputArgs = getProcessArguments(processDef) final processResult = script.invokeMethod(processName, inputArgs as Object[]) - + printOutput(processName, processResult) return processResult } @@ -97,6 +108,70 @@ class ProcessEntryHandler { return new WorkflowDef(script, workflowBody) } + /** + * Prints the process outputs. + * ChannelOut has two structures containing the outputs and anonymous channels. + * @param processName + * @param processResult ChannelOut containing the process outputs + */ + private printOutput(String processName, def processResult){ + if( ! processResult instanceof ChannelOut ) { + throw new AbortOperationException("Not a valid process output ($processResult.class)") + } + final results = processResult as ChannelOut + if (results.isEmpty()){ + log.debug("No outputs found for $processName") + return + } + + final named = new HashMap(results.size()) + + //Compute reverse index for named outputs + for (String name : results.getNames()){ + named.put(results.getProperty(name) as DataflowWriteChannel, name) + } + // Create the processOutputs map to keep the outputs order, and create the subcriber per output channel to collect the output values + int unnamedIndex = 1 + processOutputs = new LinkedHashMap<>(results.size()) + results.each { + String name = named.get(it) + if (!name) { + name = "anonymous-$unnamedIndex".toString() + unnamedIndex++ + } + processOutputs.put(name, []) + createProcessOutputSubscriber(name, CH.getReadChannel(it)) + } + //Add workflow + session.workflowMetadata.onComplete { + if( processOutputs) { + println "" + println "Process $processName Outputs:" + println DumpHelper.prettyPrintJson(processOutputs) + println "" + } + } + } + + /** + * Create a subscriber operator to inspect the output channels and build a map. + * At `onNext` event, add the output value in the `processOutputs` map with the output name as key. + * At `onComplete` event, convert single element list to single value. + * + * @param name Process output name + * @param channel Process output channel + */ + private void createProcessOutputSubscriber(String name, DataflowReadChannel channel ){ + def onNextClosure = { it -> + (processOutputs[name] as List).add(it) } + def onCompleteClosure = { + final list = processOutputs[name] as List + if( list.size() == 1) { + processOutputs[name] = list[0] + } + } + DataflowHelper.subscribeImpl(channel, [onNext: onNextClosure, onComplete: onCompleteClosure]) + } /** * Gets the input arguments for a process by parsing input parameter structures From 24a741fa70aac492b7b91aded8193c6ef70f4b73 Mon Sep 17 00:00:00 2001 From: jorgee Date: Fri, 30 Jan 2026 20:28:22 +0100 Subject: [PATCH 03/23] rename ModuleSpec and PipelineSpec Signed-off-by: jorgee --- .../nextflow/cli/module/ModuleInstall.groovy | 4 +-- .../nextflow/cli/module/ModulePublish.groovy | 8 ++--- .../nextflow/cli/module/ModuleRemove.groovy | 4 +-- .../nextflow/cli/module/ModuleRun.groovy | 29 +++---------------- ...oduleManifest.groovy => ModuleSpec.groovy} | 8 ++--- .../nextflow/module/ModuleStorage.groovy | 2 +- .../PipelineSpec.groovy} | 6 ++-- ...ifestTest.groovy => ModuleSpecTest.groovy} | 17 +++++------ 8 files changed, 28 insertions(+), 50 deletions(-) rename modules/nextflow/src/main/groovy/nextflow/module/{ModuleManifest.groovy => ModuleSpec.groovy} (95%) rename modules/nextflow/src/main/groovy/nextflow/{util/NextflowSpecFile.groovy => pipeline/PipelineSpec.groovy} (98%) rename modules/nextflow/src/test/groovy/nextflow/module/{ModuleManifestTest.groovy => ModuleSpecTest.groovy} (90%) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy index 55d7b9826c..adbb926aeb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy @@ -27,7 +27,7 @@ import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException import nextflow.module.ModuleReference import nextflow.module.ModuleResolver -import nextflow.util.NextflowSpecFile +import nextflow.pipeline.PipelineSpec import java.nio.file.Paths @@ -76,7 +76,7 @@ class ModuleInstall extends CmdBase { //TODO: Decide final location of modules currently in nextflow_spec.json. // Alternative: Use nextflow config. It requires to implement nextflow.config updater features // def modulesConfig = config.navigate('modules') as ModulesConfig - def specFile = new NextflowSpecFile(baseDir) + def specFile = new PipelineSpec(baseDir) def modulesConfig = new ModulesConfig(specFile.getModules()) // Create resolver and install diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy index ec53b9d88b..d0fbfca6e2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy @@ -25,7 +25,7 @@ import nextflow.cli.CmdBase import nextflow.config.ConfigBuilder import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException -import nextflow.module.ModuleManifest +import nextflow.module.ModuleSpec import nextflow.module.ModuleReference import nextflow.module.ModuleRegistryClient import nextflow.module.ModuleStorage @@ -78,7 +78,7 @@ class ModulePublish extends CmdBase { // Step 2: Load and validate manifest def manifestPath = moduleDir.resolve(ModuleStorage.MODULE_MANIFEST_FILE) - def manifest = ModuleManifest.load(manifestPath) + def manifest = ModuleSpec.load(manifestPath) def manifestErrors = manifest.validate() if (!manifestErrors.isEmpty()) { @@ -106,7 +106,7 @@ class ModulePublish extends CmdBase { } - private void publishModule(Path moduleDir, RegistryConfig registryConfig, ModuleManifest manifest){ + private void publishModule(Path moduleDir, RegistryConfig registryConfig, ModuleSpec manifest){ log.info "Creating module bundle..." def storage = new ModuleStorage(moduleDir.parent) def tempBundleFile = Files.createTempFile("nf-module-publish-", ".tar.gz") @@ -155,7 +155,7 @@ class ModulePublish extends CmdBase { } } - private void printDryRunInfo(ModuleManifest manifest) { + private void printDryRunInfo(ModuleSpec manifest) { println "✓ Module structure is valid" println "" println "Module details:" diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRemove.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRemove.groovy index 8eb7f4cb25..820c3839bf 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRemove.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRemove.groovy @@ -24,7 +24,7 @@ import nextflow.cli.CmdBase import nextflow.exception.AbortOperationException import nextflow.module.ModuleReference import nextflow.module.ModuleStorage -import nextflow.util.NextflowSpecFile +import nextflow.pipeline.PipelineSpec import java.nio.file.Paths @@ -71,7 +71,7 @@ class ModuleRemove extends CmdBase { def baseDir = Paths.get('.').toAbsolutePath().normalize() //TODO: Decide final location of modules currently in nextflow_spec.json. - def specFile = new NextflowSpecFile(baseDir) + def specFile = new PipelineSpec(baseDir) // Create resolver and spec file manager def storage = new ModuleStorage(baseDir) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy index 11ae422a0c..29a8ed13ee 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy @@ -25,9 +25,10 @@ import nextflow.config.ConfigBuilder import nextflow.config.ModulesConfig import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException +import nextflow.file.FileHelper import nextflow.module.ModuleReference import nextflow.module.ModuleResolver -import nextflow.util.NextflowSpecFile +import nextflow.pipeline.PipelineSpec import java.nio.file.Files import java.nio.file.Path @@ -79,7 +80,7 @@ class ModuleRun extends CmdRun { //TODO: Decide final location of modules currently in nextflow_spec.json. // Alternative: Use nextflow config. It requires to implement nextflow.config updater features // def modulesConfig = config.navigate('modules') as ModulesConfig - def specFile = new NextflowSpecFile(baseDir) + def specFile = new PipelineSpec(baseDir) def modulesConfig = new ModulesConfig(specFile.getModules()) //TODO: Decide if create resolver with a temporarily storage or use current ./modules @@ -102,29 +103,7 @@ class ModuleRun extends CmdRun { } finally { // Clean up temporary directory - if (tempDir && Files.exists(tempDir)) { - try { - deleteDirectory(tempDir) - log.debug "Cleaned up temporary directory: ${tempDir}" - } catch (Exception e) { - log.warn "Failed to clean up temporary directory: ${tempDir}", e - } - } + FileHelper.deletePath(tempDir) } } - - /** - * Delete a directory recursively - * - * @param dir The directory to delete - */ - private void deleteDirectory(Path dir) { - if (!Files.exists(dir)) { - return - } - - Files.walk(dir) - .sorted(Comparator.reverseOrder()) - .each { Path path -> Files.delete(path) } - } } diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleManifest.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpec.groovy similarity index 95% rename from modules/nextflow/src/main/groovy/nextflow/module/ModuleManifest.groovy rename to modules/nextflow/src/main/groovy/nextflow/module/ModuleSpec.groovy index d78cea126f..6996681b32 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleManifest.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpec.groovy @@ -31,7 +31,7 @@ import java.nio.file.Path */ @Slf4j @CompileStatic -class ModuleManifest { +class ModuleSpec { String name String version @@ -45,9 +45,9 @@ class ModuleManifest { * Load a module manifest from a meta.yaml file * * @param metaYamlPath Path to meta.yaml - * @return ModuleManifest instance + * @return ModuleSpec instance */ - static ModuleManifest load(Path metaYamlPath) { + static ModuleSpec load(Path metaYamlPath) { if (!Files.exists(metaYamlPath)) { throw new AbortOperationException("Module manifest not found: ${metaYamlPath}") } @@ -56,7 +56,7 @@ class ModuleManifest { def yaml = new Yaml() def data = yaml.load(Files.newInputStream(metaYamlPath)) as Map - def manifest = new ModuleManifest() + def manifest = new ModuleSpec() manifest.name = data.name as String manifest.version = data.version as String manifest.description = data.description as String diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy index df4a360599..8f08d808b6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy @@ -106,7 +106,7 @@ class ModuleStorage { // Load checksum if available installed.expectedChecksum = ModuleChecksum.load(moduleDir) - installed.installedVersion = ModuleManifest.load(installed.manifestFile).version + installed.installedVersion = ModuleSpec.load(installed.manifestFile).version return installed } diff --git a/modules/nextflow/src/main/groovy/nextflow/util/NextflowSpecFile.groovy b/modules/nextflow/src/main/groovy/nextflow/pipeline/PipelineSpec.groovy similarity index 98% rename from modules/nextflow/src/main/groovy/nextflow/util/NextflowSpecFile.groovy rename to modules/nextflow/src/main/groovy/nextflow/pipeline/PipelineSpec.groovy index 07d34eab44..a333b720de 100644 --- a/modules/nextflow/src/main/groovy/nextflow/util/NextflowSpecFile.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/pipeline/PipelineSpec.groovy @@ -14,7 +14,7 @@ * limitations under the License. */ -package nextflow.util +package nextflow.pipeline import groovy.json.JsonOutput import groovy.json.JsonSlurper @@ -32,14 +32,14 @@ import java.nio.file.StandardOpenOption */ @Slf4j @CompileStatic -class NextflowSpecFile { +class PipelineSpec { private static final String SPEC_FILE_NAME = 'nextflow_spec.json' private final Path baseDir private final Path specFile - NextflowSpecFile(Path baseDir) { + PipelineSpec(Path baseDir) { this.baseDir = baseDir this.specFile = baseDir.resolve(SPEC_FILE_NAME) } diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleManifestTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleSpecTest.groovy similarity index 90% rename from modules/nextflow/src/test/groovy/nextflow/module/ModuleManifestTest.groovy rename to modules/nextflow/src/test/groovy/nextflow/module/ModuleSpecTest.groovy index 6fbc15da65..aa53e38364 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleManifestTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleSpecTest.groovy @@ -20,15 +20,14 @@ import nextflow.exception.AbortOperationException import spock.lang.Specification import spock.lang.TempDir -import java.nio.file.Files import java.nio.file.Path /** - * Tests for ModuleManifest + * Tests for ModuleSpec * * @author Paolo Di Tommaso */ -class ModuleManifestTest extends Specification { +class ModuleSpecTest extends Specification { @TempDir Path tempDir @@ -51,7 +50,7 @@ requires: ''' when: - def manifest = ModuleManifest.load(metaYaml) + def manifest = ModuleSpec.load(metaYaml) then: manifest.name == 'nf-core/fastqc' @@ -68,7 +67,7 @@ requires: def metaYaml = tempDir.resolve('meta.yaml') when: - ModuleManifest.load(metaYaml) + ModuleSpec.load(metaYaml) then: thrown(AbortOperationException) @@ -76,7 +75,7 @@ requires: def 'should validate complete manifest' () { given: - def manifest = new ModuleManifest( + def manifest = new ModuleSpec( name: 'nf-core/fastqc', version: '1.0.0', description: 'FastQC quality control', @@ -93,7 +92,7 @@ requires: def 'should detect missing required fields' () { given: - def manifest = new ModuleManifest( + def manifest = new ModuleSpec( name: 'nf-core/fastqc' // missing version, description, license ) @@ -111,7 +110,7 @@ requires: def 'should validate version format' () { given: - def manifest = new ModuleManifest( + def manifest = new ModuleSpec( name: 'nf-core/fastqc', version: version, description: 'Test', @@ -136,7 +135,7 @@ requires: def 'should validate module name format' () { given: - def manifest = new ModuleManifest( + def manifest = new ModuleSpec( name: name, version: '1.0.0', description: 'Test', From 2432e905c8e1ec5a418e0f89b2806b6a4534347d Mon Sep 17 00:00:00 2001 From: jorgee Date: Fri, 6 Feb 2026 20:51:50 +0100 Subject: [PATCH 04/23] add documentation and tests Signed-off-by: jorgee --- docs/cli.md | 115 ++++- docs/reference/cli.md | 181 ++++++++ modules/nextflow/build.gradle | 2 + .../nextflow/cli/module/ModuleInstall.groovy | 23 +- .../nextflow/cli/module/ModuleList.groovy | 7 +- .../nextflow/cli/module/ModulePublish.groovy | 9 +- .../nextflow/cli/module/ModuleRemove.groovy | 9 +- .../nextflow/cli/module/ModuleRun.groovy | 22 +- .../nextflow/cli/module/ModuleSearch.groovy | 10 +- .../nextflow/config/ModulesConfig.groovy | 10 +- .../nextflow/module/ModuleResolver.groovy | 11 +- .../groovy/nextflow/module/ModuleSpec.groovy | 6 +- .../nextflow/module/ModuleStorage.groovy | 41 +- .../cli/module/ModuleInstallTest.groovy | 415 +++++++++++++++++ .../nextflow/cli/module/ModuleListTest.groovy | 186 ++++++++ .../cli/module/ModuleRemoveTest.groovy | 223 +++++++++ .../nextflow/cli/module/ModuleRunTest.groovy | 248 ++++++++++ .../cli/module/ModuleSearchTest.groovy | 225 +++++++++ .../nextflow/config/ModulesConfigTest.groovy | 197 ++++++++ .../nextflow/config/RegistryConfigTest.groovy | 356 ++++++++++++++ .../module/ModuleRegistryClientTest.groovy | 433 ++++++++++++++++++ .../nextflow/module/ModuleResolverTest.groovy | 194 ++++++++ .../nextflow/module/ModuleSpecTest.groovy | 19 +- .../nextflow/module/ModuleStorageTest.groovy | 44 ++ modules/nf-commons/build.gradle | 2 +- 25 files changed, 2937 insertions(+), 51 deletions(-) create mode 100644 modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInstallTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleListTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleRemoveTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleRunTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleSearchTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/config/ModulesConfigTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/config/RegistryConfigTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/module/ModuleRegistryClientTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverTest.groovy diff --git a/docs/cli.md b/docs/cli.md index 607a041756..a4283d47de 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -262,7 +262,120 @@ $ nextflow secrets set AWS_ACCESS_KEY_ID $ nextflow secrets delete AWS_ACCESS_KEY_ID ``` -See {ref}`cli-secrets` for more information. +See {ref}`cli-secrets` for more information. + +## Module management + +:::{versionadded} 26.04.0 +::: + +Module management commands enable working with reusable, registry-based modules. The Nextflow module system allows you to install, run, search, and publish standardized modules from registries, eliminating duplicate work and spreading improvements throughout the community. + +Use these commands to discover modules in registries, install them into your project, run them directly without creating a workflow, and publish your own modules for others to use. + +### Installing modules + +The `module install` command downloads modules from a registry and makes them available in your workflow. Modules are stored locally in the `modules/` directory and version information is tracked in `nextflow_spec.json`. + +Use this to add reusable modules to your pipeline, manage module versions, or update modules to newer versions. + +```console +$ nextflow module install nf-core/fastqc +$ nextflow module install nf-core/fastqc -version 1.0.0 +``` + +After installation, module will be available in `modules/@nf-core/fastqc` and included in `nextflow_spec.json` + +Use the `-force` flag to reinstall a module even if local modifications exist. + +See {ref}`cli-module-install` for more information. + +### Running modules directly + +The `module run` command executes a module directly from the registry without requiring a wrapper workflow. This provides immediate access to module functionality for ad-hoc tasks or testing. + +Use this to quickly run a module, test module functionality, or execute one-off data processing tasks. + +```console +$ nextflow module run nf-core/fastqc --input 'data/*.fastq.gz' +$ nextflow module run nf-core/fastqc --input 'data/*.fastq.gz' -version 1.0.0 +``` + +The command accepts all standard Nextflow execution options (`-profile`, `-resume`, etc.): + +```console +$ nextflow module run nf-core/salmon \ + --reads reads.fq \ + --index salmon_index \ + -profile docker \ + -resume +``` + +See {ref}`cli-module-run` for more information. + +### Listing modules + +The `module list` command displays all modules currently installed in your project, showing their versions and integrity status. + +Use this to review installed modules, check module versions, or detect local modifications. + +```console +$ nextflow module list +$ nextflow module list -json +``` + +The output shows each module's name, installed version, and whether it has been modified locally. Use `-json` for machine-readable output suitable for scripting. + +See {ref}`cli-module-list` for more information. + +### Searching for modules + +The `module search` command queries the module registry to discover available modules by keyword or name. + +Use this to find modules for specific tasks, explore available tools, or discover community contributions. + +```console +$ nextflow module search alignment +$ nextflow module search "quality control" -limit 10 +$ nextflow module search bwa -json +``` + +Results include module names, versions, descriptions, and download statistics. Use `-limit` to control the number of results and `-json` for programmatic access. + +See {ref}`cli-module-search` for more information. + +### Removing modules + +The `module remove` command deletes modules from your project, removing local files and configuration entries. + +Use this to clean up unused modules, free disk space, or remove deprecated modules from your pipeline. + +```console +$ nextflow module remove nf-core/fastqc +$ nextflow module remove nf-core/fastqc -keep-config +$ nextflow module remove nf-core/fastqc -keep-files +``` + +By default, both local files and configuration entries are removed. Use `-keep-config` to preserve version information in `nextflow_spec.json`, or `-keep-files` to remove only the configuration entry while keeping local files. + +See {ref}`cli-module-remove` for more information. + +### Publishing modules + +The `module publish` command uploads modules to a registry, making them available for others to install and use. + +Use this to share your modules with the community, contribute to module libraries, or distribute modules within your organization. + +```console +$ nextflow module publish myorg/my-module +$ nextflow module publish myorg/my-module -dry-run +``` + +Publishing requires authentication via the `NXF_REGISTRY_TOKEN` environment variable or `registry.auth` in the Nextflow configuration. The module must include `main.nf`, `meta.yaml`, and `README.md` files. + +Use `-dry-run` to validate your module structure without uploading. + +See {ref}`cli-module-publish` for more information. ## Configuration and validation diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 23f7ed1855..832af687ed 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1125,6 +1125,187 @@ $ nextflow log tiny_leavitt -F 'process =~ /split_letters/' work/1f/f1ea9158fb23b53d5083953121d6b6 ``` +(cli-module)= + +### `module` + +:::{versionadded} 26.04.0 +::: + +Manage Nextflow modules from registries. + +**Usage** + +```console +$ nextflow module [options] +``` + +**Description** + +The `module` command provides a comprehensive system for managing reusable, registry-based modules. It enables installing modules from registries, running them directly, searching for available modules, and publishing your own modules for community use. + +**Subcommands** + +(cli-module-install)= + +`install [options] [scope/name]` + +: Install a module from the registry into your project. +: Downloaded modules are stored in the `modules/` directory and version information is tracked in `nextflow_spec.json`. +: The following options are available: + + `-version` + : Specify the module version to install (e.g., `1.0.0`). If not specified, installs the latest version. + + `-force` + : Force reinstall even if the module exists locally with modifications. Without this flag, Nextflow prevents overwriting locally modified modules. + +: **Examples:** + + ```console + # Install latest version + $ nextflow module install nf-core/fastqc + + # Install specific version + $ nextflow module install nf-core/fastqc -version 1.0.0 + + # Force reinstall over local modifications + $ nextflow module install nf-core/fastqc -force + ``` + +(cli-module-run)= + +`run [options] [scope/name] [-- ]` + +: Execute a module directly from the registry without creating a wrapper workflow. +: Automatically downloads the module if not already installed. Accepts all standard Nextflow run options. +: The following options are available: + + `-version` + : Specify the module version to run (e.g., `1.0.0`). If not specified, uses the latest version. + + All standard `run` command options + : The `module run` command extends the `run` command and accepts all its options, including `-profile`, `-resume`, `-c`, etc. + +: **Examples:** + + ```console + # Run module with inputs + $ nextflow module run nf-core/fastqc --input 'data/*.fastq.gz' + + # Run specific version with Nextflow options + $ nextflow module run nf-core/fastqc \ + --input 'data/*.fastq.gz' \ + -version 1.0.0 \ + -profile docker \ + -resume + ``` + +(cli-module-list)= + +`list [options]` + +: List all modules currently installed in your project. +: Shows module names, versions, and integrity status (whether they've been modified locally). +: The following options are available: + + `-json` + : Output results in JSON format for programmatic processing. + +: **Examples:** + + ```console + # Display installed modules in formatted table + $ nextflow module list + + # Output as JSON + $ nextflow module list -json + ``` + +(cli-module-search)= + +`search [options] [query]` + +: Search for modules in the registry by keyword or name. +: Returns modules matching the query with their names, versions, descriptions, and download statistics. +: The following options are available: + + `-limit` + : Maximum number of results to return (default: varies by registry). + + `-json` + : Output results in JSON format for programmatic processing. + +: **Examples:** + + ```console + # Search for alignment-related modules + $ nextflow module search alignment + + # Search with limited results + $ nextflow module search "quality control" -limit 10 + + # Get results as JSON + $ nextflow module search bwa -json + ``` + +(cli-module-remove)= + +`remove [options] [scope/name]` + +: Remove a module from your project. +: By default, removes both local files and configuration entries. Use options to control what gets removed. +: The following options are available: + + `-keep-config` + : Keep the version entry in `nextflow_spec.json` but delete local files from the `modules/` directory. + + `-keep-files` + : Remove the version entry from `nextflow_spec.json` but keep local files in the `modules/` directory. + +: **Examples:** + + ```console + # Remove module completely + $ nextflow module remove nf-core/fastqc + + # Delete files but keep version config + $ nextflow module remove nf-core/fastqc -keep-config + + # Remove from config but keep local files + $ nextflow module remove nf-core/fastqc -keep-files + ``` + +(cli-module-publish)= + +`publish [options] [scope/name]` + +: Publish a module to the registry, making it available for others to install. +: Requires authentication via `NXF_REGISTRY_TOKEN` environment variable or `registry.auth` configuration. +: The module directory must contain `main.nf`, `meta.yaml`, and `README.md`. +: The following options are available: + + `-dry-run` + : Validate the module structure and metadata without uploading to the registry. Useful for testing before publishing. + + `-registry` + : Specify the registry to publish the module (default: `https://registry.nextflow.io`) + +: **Examples:** + + ```console + # Validate module structure without publishing + $ nextflow module publish myorg/my-module -dry-run + + # Publish to nextflow registry + $ export NXF_REGISTRY_TOKEN=your-token + $ nextflow module publish myorg/my-module + + # Publish to a custom registry + $ export NXF_REGISTRY_TOKEN=your-token + $ nextflow module publish myorg/my-module -registry 'https://custom.registry.com' + ``` + (cli-plugin)= ### `plugin` diff --git a/modules/nextflow/build.gradle b/modules/nextflow/build.gradle index dd46d41e36..b0596807cc 100644 --- a/modules/nextflow/build.gradle +++ b/modules/nextflow/build.gradle @@ -55,9 +55,11 @@ dependencies { api 'io.seqera:lib-trace:0.1.0' api 'com.fasterxml.woodstox:woodstox-core:7.1.1' api 'org.apache.commons:commons-compress:1.27.1' // For tar.gz extraction + api 'io.seqera:npr-api:0.20.1' testImplementation 'org.subethamail:subethasmtp:3.1.7' testImplementation (project(':nf-lineage')) + testImplementation 'org.wiremock:wiremock:3.13.1' // test configuration testFixturesApi ("org.apache.groovy:groovy-test:4.0.30") { exclude group: 'org.apache.groovy' } testFixturesApi ("org.objenesis:objenesis:3.4") diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy index adbb926aeb..915ab6afc2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy @@ -26,9 +26,12 @@ import nextflow.config.ModulesConfig import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException import nextflow.module.ModuleReference +import nextflow.module.ModuleRegistryClient import nextflow.module.ModuleResolver import nextflow.pipeline.PipelineSpec +import nextflow.util.TestOnly +import java.nio.file.Path import java.nio.file.Paths /** @@ -50,6 +53,12 @@ class ModuleInstall extends CmdBase { @Parameter(description = "[scope/name]", required = true) List args + @TestOnly + protected Path root + + @TestOnly + protected ModuleRegistryClient client + @Override String getName() { return 'install' @@ -66,21 +75,19 @@ class ModuleInstall extends CmdBase { def reference = ModuleReference.parse(moduleRef) // Get config - def baseDir = Paths.get('.').toAbsolutePath().normalize() + def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize() def config = new ConfigBuilder() .setOptions(launcher.options) .setBaseDir(baseDir) .build() - def registryConfig = config.navigate('registry') as RegistryConfig + final registryConfig = config.navigate('registry') as RegistryConfig - //TODO: Decide final location of modules currently in nextflow_spec.json. - // Alternative: Use nextflow config. It requires to implement nextflow.config updater features - // def modulesConfig = config.navigate('modules') as ModulesConfig - def specFile = new PipelineSpec(baseDir) - def modulesConfig = new ModulesConfig(specFile.getModules()) + // Get modules versions from nextflow_spec.json. + final specFile = new PipelineSpec(baseDir) + final modulesConfig = new ModulesConfig(specFile.getModules()) // Create resolver and install - def resolver = new ModuleResolver(baseDir, modulesConfig, registryConfig) + def resolver = new ModuleResolver(baseDir, client ?: new ModuleRegistryClient(registryConfig), modulesConfig) try { def installedMainFile = resolver.installModule(reference, version, force) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleList.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleList.groovy index d2fa1dec62..2f40d7a32e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleList.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleList.groovy @@ -26,7 +26,9 @@ import nextflow.exception.AbortOperationException import nextflow.module.InstalledModule import nextflow.module.ModuleIntegrity import nextflow.module.ModuleStorage +import nextflow.util.TestOnly +import java.nio.file.Path import java.nio.file.Paths /** @@ -42,6 +44,9 @@ class ModuleList extends CmdBase { @Parameter(names = ["-json"], description = "Output in JSON format", arity=0) boolean jsonOutput = false + @TestOnly + protected Path root + @Override String getName() { return 'list' @@ -51,7 +56,7 @@ class ModuleList extends CmdBase { void run() { // Get config - def baseDir = Paths.get('.').toAbsolutePath().normalize() + def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize() // Create resolver and list modules diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy index d0fbfca6e2..f17395c6ca 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy @@ -29,6 +29,7 @@ import nextflow.module.ModuleSpec import nextflow.module.ModuleReference import nextflow.module.ModuleRegistryClient import nextflow.module.ModuleStorage +import nextflow.util.TestOnly import java.nio.file.Files import java.nio.file.Path @@ -53,6 +54,12 @@ class ModulePublish extends CmdBase { @Parameter(description = "Module directory path or scope/name") List args + @TestOnly + protected Path root + + @TestOnly + protected ModuleRegistryClient client + @Override String getName() { return 'publish' @@ -240,7 +247,7 @@ class ModulePublish extends CmdBase { } final ref = ModuleReference.parse('@' + module) - final localStorage = new ModuleStorage(Paths.get('.').toAbsolutePath().normalize()) + final localStorage = new ModuleStorage(root ?: Paths.get('.').toAbsolutePath().normalize()) if (!localStorage.isInstalled(ref)){ throw new AbortOperationException("No module diretory found for $module") diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRemove.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRemove.groovy index 820c3839bf..ee207bb3ad 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRemove.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRemove.groovy @@ -25,7 +25,9 @@ import nextflow.exception.AbortOperationException import nextflow.module.ModuleReference import nextflow.module.ModuleStorage import nextflow.pipeline.PipelineSpec +import nextflow.util.TestOnly +import java.nio.file.Path import java.nio.file.Paths /** @@ -47,6 +49,9 @@ class ModuleRemove extends CmdBase { @Parameter(names = ["-keep-files"], description = "Remove from config but keep local files", arity = 0) boolean keepFiles = false + @TestOnly + protected Path root + @Override String getName() { return 'remove' @@ -68,9 +73,9 @@ class ModuleRemove extends CmdBase { def reference = ModuleReference.parse(moduleRef) // Get config - def baseDir = Paths.get('.').toAbsolutePath().normalize() + def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize() - //TODO: Decide final location of modules currently in nextflow_spec.json. + //Get module versions from nextflow_spec.json. def specFile = new PipelineSpec(baseDir) // Create resolver and spec file manager diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy index 29a8ed13ee..3a547295cb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy @@ -27,8 +27,10 @@ import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException import nextflow.file.FileHelper import nextflow.module.ModuleReference +import nextflow.module.ModuleRegistryClient import nextflow.module.ModuleResolver import nextflow.pipeline.PipelineSpec +import nextflow.util.TestOnly import java.nio.file.Files import java.nio.file.Path @@ -46,6 +48,12 @@ class ModuleRun extends CmdRun { @Parameter(names = ["-version"], description = "Module version") String version + @TestOnly + protected Path root + + @TestOnly + protected ModuleRegistryClient client + @Override String getName() { return 'run' @@ -69,7 +77,7 @@ class ModuleRun extends CmdRun { } // Get config - def baseDir = Paths.get('.').toAbsolutePath().normalize() + def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize() def config = new ConfigBuilder() .setOptions(launcher.options) .setBaseDir(baseDir) @@ -77,15 +85,11 @@ class ModuleRun extends CmdRun { def registryConfig = config.navigate('registry') as RegistryConfig - //TODO: Decide final location of modules currently in nextflow_spec.json. - // Alternative: Use nextflow config. It requires to implement nextflow.config updater features - // def modulesConfig = config.navigate('modules') as ModulesConfig + //Get module version from nextflow_spec.json. def specFile = new PipelineSpec(baseDir) def modulesConfig = new ModulesConfig(specFile.getModules()) - //TODO: Decide if create resolver with a temporarily storage or use current ./modules - def tempDir = Files.createTempDirectory("nf-module-run-") - def resolver = new ModuleResolver(tempDir, modulesConfig, registryConfig) + def resolver = new ModuleResolver(baseDir, client ?: new ModuleRegistryClient(registryConfig), modulesConfig) try{ Path moduleFile = resolver.installModule(reference, version) if( moduleFile ) { @@ -101,9 +105,5 @@ class ModuleRun extends CmdRun { log.error("Failed to run module", e) throw new AbortOperationException("Module run failed: ${e.message}", e) } - finally { - // Clean up temporary directory - FileHelper.deletePath(tempDir) - } } } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy index 9027e9a64d..af92dc52fd 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy @@ -28,6 +28,7 @@ import nextflow.config.ConfigBuilder import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException import nextflow.module.ModuleRegistryClient +import nextflow.util.TestOnly import java.nio.file.Paths @@ -50,6 +51,9 @@ class ModuleSearch extends CmdBase { @Parameter(description = "", required = true) List args + @TestOnly + protected ModuleRegistryClient client + @Override String getName() { return 'search' @@ -71,14 +75,14 @@ class ModuleSearch extends CmdBase { final registryConfig = config.navigate('registry') as RegistryConfig - // Create client to seach - final client = new ModuleRegistryClient(registryConfig) + // Create client to search + final client = this.client ?: new ModuleRegistryClient(registryConfig) try { println "Searching for '${query}'..." final results = client.search(query, limit) - if (results.totalResults == 0 || !results.results || results.results.isEmpty()) { + if (!results || results.totalResults == 0 || !results.results || results.results.isEmpty()) { println "No modules found" return } diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ModulesConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ModulesConfig.groovy index 558adec552..c1968c62ea 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ModulesConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ModulesConfig.groovy @@ -22,6 +22,7 @@ import nextflow.config.spec.ConfigOption import nextflow.config.spec.ConfigScope import nextflow.config.spec.ScopeName import nextflow.script.dsl.Description +import nextflow.util.TestOnly /** * Configuration scope for module version declarations @@ -47,7 +48,7 @@ class ModulesConfig implements ConfigScope { ModulesConfig(Map opts) { if (opts) { opts.each { key, value -> - modules[key.toString()] = value.toString() + this.modules.put(key.toString(), value.toString()) } } } @@ -59,7 +60,7 @@ class ModulesConfig implements ConfigScope { * @return The configured version, or null if not configured */ String getVersion(String moduleName) { - return modules.get(moduleName) + return this.modules.get(moduleName) } /** @@ -67,7 +68,8 @@ class ModulesConfig implements ConfigScope { * * @return Map of module name to version */ - Map getModules() { + @TestOnly + Map getAllModules() { return Collections.unmodifiableMap(modules) } @@ -78,7 +80,7 @@ class ModulesConfig implements ConfigScope { * @param version The version to set */ void setVersion(String moduleName, String version) { - modules[moduleName] = version + this.modules.put(moduleName, version) } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy index 1720382127..dc96c7a3b6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy @@ -37,15 +37,18 @@ class ModuleResolver { private final ModuleRegistryClient registryClient private final ModuleStorage storage private final ModulesConfig modulesConfig - private final RegistryConfig registryConfig - ModuleResolver(Path baseDir, ModulesConfig modulesConfig = null, RegistryConfig registryConfig = null) { - this.registryConfig = registryConfig ?: new RegistryConfig() - this.registryClient = new ModuleRegistryClient(this.registryConfig) + ModuleResolver (Path baseDir, ModuleRegistryClient registryClient, ModulesConfig modulesConfig = null) { + this.registryClient = registryClient this.storage = new ModuleStorage(baseDir) this.modulesConfig = modulesConfig ?: new ModulesConfig() } + ModuleResolver(Path baseDir, ModulesConfig modulesConfig = null, RegistryConfig registryConfig = null) { + this(baseDir, new ModuleRegistryClient(registryConfig ?: new RegistryConfig()), modulesConfig) + + } + /** * Resolve a module reference to an installed module path * diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpec.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpec.groovy index 6996681b32..613a9fe959 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpec.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpec.groovy @@ -98,9 +98,9 @@ class ModuleSpec { errors << "Invalid version format: ${version} (expected semantic versioning, e.g., 1.0.0)".toString() } - // Validate name format (scope/name) - if (name && !name.matches(/^[a-zA-Z0-9_-]+\/[a-zA-Z0-9_-]+$/)) { - errors << "Invalid module name format: ${name} (expected scope/name, e.g., nf-core/fastqc)".toString() + // Validate name format (scope/name or scope/path/to/name for nested modules) + if (name && !name.matches(/^[a-zA-Z0-9._-]+\/[a-zA-Z0-9_-]+(?:\/[a-zA-Z0-9_-]+)*$/)) { + errors << "Invalid module name format: ${name} (expected scope/name or scope/path/to/name, e.g., nf-core/fastqc or nf-core/gfatools/gfa2fa)".toString() } return errors diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy index 8f08d808b6..e169df789f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy @@ -130,11 +130,33 @@ class ModuleStorage { // Remove @ prefix from directory name to get scope def scope = scopeDirName.startsWith('@') ? scopeDirName.substring(1) : scopeDirName - // Iterate over module directories within scope - Files.list(scopeDir).each { Path moduleDir -> - if (!Files.isDirectory(moduleDir)) return + // Recursively find all directories containing meta.yml under this scope + findModulesRecursive(scopeDir, scope, modules) + } + + return modules + } - def name = moduleDir.fileName.toString() + /** + * Recursively find modules in subdirectories + * @param dir Current directory to search + * @param scope Module scope + * @param modules List to accumulate found modules + */ + private void findModulesRecursive(Path dir, String scope, List modules) { + if (!Files.isDirectory(dir)) return + + // Check if current directory contains meta.yml (is a module) + if (Files.exists(dir.resolve(MODULE_MANIFEST_FILE))) { + // Calculate the module name from the path relative to scope directory + def scopeDir = dir.getParent() + while (scopeDir != null && !scopeDir.fileName.toString().equals('@' + scope)) { + scopeDir = scopeDir.getParent() + } + + if (scopeDir != null) { + def relativePath = scopeDir.relativize(dir).toString() + def name = relativePath.replace('\\', '/') // Normalize path separators def reference = new ModuleReference(scope, name) def installed = getInstalledModule(reference) @@ -144,7 +166,16 @@ class ModuleStorage { } } - return modules + // Recursively search subdirectories + try { + Files.list(dir).each { Path subDir -> + if (Files.isDirectory(subDir)) { + findModulesRecursive(subDir, scope, modules) + } + } + } catch (IOException e) { + log.warn "Failed to list directory ${dir}: ${e.message}" + } } /** diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInstallTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInstallTest.groovy new file mode 100644 index 0000000000..11761d719a --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInstallTest.groovy @@ -0,0 +1,415 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli.module + +import io.seqera.npr.api.schema.v1.Module +import io.seqera.npr.api.schema.v1.ModuleRelease +import nextflow.cli.Launcher +import nextflow.exception.AbortOperationException +import nextflow.module.ModuleRegistryClient +import nextflow.pipeline.PipelineSpec +import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream +import org.junit.Rule +import spock.lang.Specification +import spock.lang.TempDir +import test.OutputCapture + +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.GZIPOutputStream + +/** + * Tests for ModuleInstall command + * + * @author Jorge Ejarque + */ +class ModuleInstallTest extends Specification { + + @Rule + OutputCapture capture = new OutputCapture() + + @TempDir + Path tempDir + + def 'should install module with latest version'() { + given: + def cmd = new ModuleInstall() + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.args = ['nf-core/fastqc'] + cmd.root = tempDir + + and: + // Create mock module package + def modulePackage = createModulePackage('nf-core', 'fastqc', '1.0.0') + + // Mock registry client + def mockClient = Mock(ModuleRegistryClient) + mockClient.fetchModule('@nf-core/fastqc') >> new Module( + name: '@nf-core/fastqc', + latest: new ModuleRelease(version: '1.0.0') + ) + mockClient.downloadModule('@nf-core/fastqc', '1.0.0', _) >> { String name, String version, Path dest -> + Files.write(dest, modulePackage) + return dest + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Installing') + output.contains('nf-core/fastqc') + output.contains('1.0.0') + + and: + def moduleDir = tempDir.resolve('modules/@nf-core/fastqc') + Files.exists(moduleDir) + Files.exists(moduleDir.resolve('main.nf')) + Files.exists(moduleDir.resolve('meta.yml')) + + and: + def spec = new PipelineSpec(tempDir) + spec.getModules().get('@nf-core/fastqc') == '1.0.0' + } + + def 'should install module with specific version'() { + given: + def cmd = new ModuleInstall() + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.args = ['nf-core/fastqc'] + cmd.version = '2.0.0' + cmd.root = tempDir + + and: + def modulePackage = createModulePackage('nf-core', 'fastqc', '2.0.0') + + def mockClient = Mock(ModuleRegistryClient) + mockClient.downloadModule(_, _, _) >> { String name, String version, Path dest -> + Files.write(dest, modulePackage) + return dest + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Installing') + output.contains('nf-core/fastqc') + output.contains('2.0.0') + + and: + def spec = new PipelineSpec(tempDir) + spec.getModules().get('@nf-core/fastqc') == '2.0.0' + + } + + def 'should update existing module with force flag'() { + given: + // Pre-install version 1.0.0 + def moduleDir = tempDir.resolve('modules/@nf-core/fastqc') + Files.createDirectories(moduleDir) + moduleDir.resolve('main.nf').text = 'process OLD { }' + moduleDir.resolve('meta.yml').text = """ + name: nf-core/fastqc + version: '1.0.0' + description: Test module + """.stripIndent() + + def spec = new PipelineSpec(tempDir) + spec.addModuleEntry('@nf-core/fastqc', '1.0.0') + + and: + def cmd = new ModuleInstall() + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.args = ['nf-core/fastqc'] + cmd.version = '2.0.0' + cmd.force = true + cmd.root = tempDir + + and: + def modulePackage = createModulePackage('nf-core', 'fastqc', '2.0.0') + + def mockClient = Mock(ModuleRegistryClient) + mockClient.downloadModule('@nf-core/fastqc', '2.0.0', _) >> { String name, String version, Path dest -> + Files.write(dest, modulePackage) + return dest + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Installing') + output.contains('2.0.0') + + and: + def updatedSpec = new PipelineSpec(tempDir) + updatedSpec.getModules().get('@nf-core/fastqc') == '2.0.0' + + and: + moduleDir.resolve('main.nf').text.contains('FASTQC') // New content + } + + def 'should fail when module already installed without force'() { + given: + // Pre-install the module + def moduleDir = tempDir.resolve('modules/@nf-core/fastqc') + Files.createDirectories(moduleDir) + moduleDir.resolve('main.nf').text = 'process FASTQC { }' + moduleDir.resolve('meta.yml').text = """ + name: nf-core/fastqc + version: '1.0.0' + description: Test module + """.stripIndent() + moduleDir.resolve('.checksum').text = 'wrong-checksum' + def spec = new PipelineSpec(tempDir) + spec.addModuleEntry('@nf-core/fastqc', '1.0.0') + + and: + def cmd = new ModuleInstall() + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.args = ['nf-core/fastqc'] + cmd.version = '2.0.0' + cmd.root = tempDir + + def mockClient = Mock(ModuleRegistryClient) + mockClient.fetchModule('@nf-core/fastqc') >> new Module( + name: '@nf-core/fastqc', + latest: new ModuleRelease(version: '2.0.0') + ) + cmd.client = mockClient + + when: + cmd.run() + + then: + def e = thrown(AbortOperationException) + e.message.contains('already installed') || e.message.contains('-force') + } + + def 'should handle module with scope in name'() { + given: + def cmd = new ModuleInstall() + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.args = ['myorg/custom-module'] + cmd.root = tempDir + + and: + def modulePackage = createModulePackage('myorg', 'custom-module', '1.0.0') + + def mockClient = Mock(ModuleRegistryClient) + mockClient.fetchModule('@myorg/custom-module') >> new Module( + name: '@myorg/custom-module', + latest: new ModuleRelease(version: '1.0.0') + ) + mockClient.downloadModule('@myorg/custom-module', '1.0.0', _) >> { String name, String version, Path dest -> + Files.write(dest, modulePackage) + return dest + } + cmd.client = mockClient + + when: + cmd.run() + + then: + def moduleDir = tempDir.resolve('modules/@myorg/custom-module') + Files.exists(moduleDir) + Files.exists(moduleDir.resolve('main.nf')) + + and: + def spec = new PipelineSpec(tempDir) + spec.getModules().get('@myorg/custom-module') == '1.0.0' + } + + def 'should create modules directory if it does not exist'() { + given: + def cmd = new ModuleInstall() + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.args = ['nf-core/fastqc'] + cmd.root = tempDir + + and: + def modulePackage = createModulePackage('nf-core', 'fastqc', '1.0.0') + + def mockClient = Mock(ModuleRegistryClient) + mockClient.fetchModule('@nf-core/fastqc') >> new Module( + name: 'nf-core/fastqc', + latest: new ModuleRelease(version: '1.0.0') + ) + mockClient.downloadModule('@nf-core/fastqc', '1.0.0', _) >> { String name, String version, Path dest -> + Files.write(dest, modulePackage) + return dest + } + cmd.client = mockClient + + when: + cmd.run() + + then: + Files.exists(tempDir.resolve('modules')) + Files.exists(tempDir.resolve('modules/@nf-core')) + Files.exists(tempDir.resolve('modules/@nf-core/fastqc')) + } + + def 'should create checksum file after installation'() { + given: + def cmd = new ModuleInstall() + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.args = ['nf-core/fastqc'] + cmd.root = tempDir + + and: + def modulePackage = createModulePackage('nf-core', 'fastqc', '1.0.0') + + def mockClient = Mock(ModuleRegistryClient) + mockClient.fetchModule('@nf-core/fastqc') >> new Module( + name: 'nf-core/fastqc', + latest: new ModuleRelease(version: '1.0.0') + ) + mockClient.downloadModule('@nf-core/fastqc', '1.0.0', _) >> { String name, String version, Path dest -> + Files.write(dest, modulePackage) + return dest + } + cmd.client = mockClient + + when: + cmd.run() + + then: + def moduleDir = tempDir.resolve('modules/@nf-core/fastqc') + Files.exists(moduleDir.resolve('.checksum')) + + and: + def checksum = moduleDir.resolve('.checksum').text + checksum != null + !checksum.isEmpty() + } + + def 'should fail with no arguments'() { + given: + def cmd = new ModuleInstall() + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.args = [] + cmd.root = tempDir + + when: + cmd.run() + + then: + thrown(AbortOperationException) + } + + def 'should fail with too many arguments'() { + given: + def cmd = new ModuleInstall() + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.args = ['nf-core/fastqc', 'extra-arg'] + cmd.root = tempDir + + when: + cmd.run() + + then: + thrown(AbortOperationException) + } + + def 'should fail with invalid module reference'() { + given: + def cmd = new ModuleInstall() + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.args = ['invalid-module-name'] // Missing scope + cmd.root = tempDir + + when: + cmd.run() + + then: + thrown(AbortOperationException) + } + + // Helper method to create a module package (tar.gz) + private byte[] createModulePackage(String scope, String name, String version) { + def baos = new ByteArrayOutputStream() + + // Create tar.gz with module files + def mainNfContent = """ + process ${name.toUpperCase().replaceAll('-', '_')} { + input: + path reads + + output: + path "*.html" + + script: + \"\"\" + echo "Running ${name}" + \"\"\" + } + """.stripIndent() + new GZIPOutputStream(baos).withCloseable { gzos -> + new TarArchiveOutputStream(gzos).withCloseable { tos -> + // Add main.nf + addTarEntry(tos, 'main.nf', mainNfContent.bytes) + + // Add meta.yml + def metaContent = """ + name: ${scope}/${name} + version: ${version} + description: Test module + """.stripIndent() + addTarEntry(tos, 'meta.yml', metaContent.bytes) + } + } + + return baos.toByteArray() + } + + private void addTarEntry(TarArchiveOutputStream tos, String name, byte[] content) { + def entry = new TarArchiveEntry(name) + entry.setSize(content.length) + tos.putArchiveEntry(entry) + tos.write(content) + tos.closeArchiveEntry() + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleListTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleListTest.groovy new file mode 100644 index 0000000000..0ed761c20c --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleListTest.groovy @@ -0,0 +1,186 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli.module + +import groovy.json.JsonSlurper +import nextflow.module.ModuleChecksum +import nextflow.module.ModuleReference +import nextflow.module.ModuleStorage +import org.junit.Rule +import spock.lang.Specification +import spock.lang.TempDir +import test.OutputCapture + +import java.nio.file.Files +import java.nio.file.Path + +/** + * Tests for ModuleList command + * + * @author Jorge Ejarque + */ +class ModuleListTest extends Specification { + + @Rule + OutputCapture capture = new OutputCapture() + + @TempDir + Path tempDir + + // No setup needed - using root field directly + + def 'should list installed modules with formatted output'() { + given: + def storage = new ModuleStorage(tempDir) + + // Create test modules + createTestModule(storage, 'nf-core', 'fastqc', '1.0.0') + createTestModule(storage, 'nf-core', 'multiqc', '2.1.0') + + and: + def cmd = new ModuleList() + cmd.root = tempDir + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Installed modules:') + output.contains('nf-core/fastqc') + output.contains('1.0.0') + output.contains('nf-core/multiqc') + output.contains('2.1.0') + output.contains('OK') || output.contains('NO CHECKSUM') + } + + def 'should list installed modules with JSON output'() { + given: + def storage = new ModuleStorage(tempDir) + + // Create test module + createTestModule(storage, 'nf-core', 'fastqc', '1.5.0') + + and: + def cmd = new ModuleList() + cmd.jsonOutput = true + cmd.root = tempDir + + when: + cmd.run() + def output = capture.toString() + def json = new JsonSlurper().parseText(output) + + then: + json.modules != null + json.modules.size() == 1 + json.modules[0].name == 'nf-core/fastqc' + json.modules[0].version == '1.5.0' + json.modules[0].integrity != null + } + + def 'should handle no installed modules'() { + given: + def cmd = new ModuleList() + cmd.root = tempDir // Use test directory with no modules + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('No modules installed') + } + + def 'should show modified status for locally modified modules'() { + given: + def storage = new ModuleStorage(tempDir) + def moduleDir = createTestModule(storage, 'nf-core', 'fastqc', '1.0.0') + + // Modify the module to trigger checksum mismatch + moduleDir.resolve('main.nf').text = 'process MODIFIED { }' + + and: + def cmd = new ModuleList() + cmd.root = tempDir + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('nf-core/fastqc') + output.contains('MODIFIED') + } + + def 'should list multiple modules sorted by name'() { + given: + def storage = new ModuleStorage(tempDir) + + // Create modules in random order + createTestModule(storage, 'nf-core', 'samtools', '1.0.0') + createTestModule(storage, 'nf-core', 'fastqc', '1.0.0') + createTestModule(storage, 'myorg', 'custom', '2.0.0') + + and: + def cmd = new ModuleList() + cmd.root = tempDir + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('fastqc') + output.contains('samtools') + output.contains('myorg/custom') + } + + private Path createTestModule(ModuleStorage storage, String scope, String name, String version) { + def moduleDir = storage.getModuleDir(new ModuleReference(scope, name)) + Files.createDirectories(moduleDir) + + // Create main.nf + moduleDir.resolve('main.nf').text = """ + process ${name.toUpperCase()} { + input: + path reads + + output: + path "*.html" + + script: + \"\"\" + echo "test" + \"\"\" + } + """.stripIndent() + + // Create meta.yml + moduleDir.resolve('meta.yml').text = """ + name: ${scope}/${name} + version: ${version} + description: Test module + """.stripIndent() + + // Create checksum + def checksum = ModuleChecksum.compute(moduleDir) + moduleDir.resolve('.checksum').text = checksum + + return moduleDir + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleRemoveTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleRemoveTest.groovy new file mode 100644 index 0000000000..3266b33106 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleRemoveTest.groovy @@ -0,0 +1,223 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli.module + +import nextflow.exception.AbortOperationException +import nextflow.module.ModuleReference +import nextflow.module.ModuleStorage +import nextflow.pipeline.PipelineSpec +import org.junit.Rule +import spock.lang.Specification +import spock.lang.TempDir +import test.OutputCapture + +import java.nio.file.Files +import java.nio.file.Path + +/** + * Tests for ModuleRemove command + * + * @author Jorge Ejarque + */ +class ModuleRemoveTest extends Specification { + + @Rule + OutputCapture capture = new OutputCapture() + + @TempDir + Path tempDir + + // No setup needed - using root field directly + + def 'should remove module files and config entry'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def moduleDir = createTestModule(storage, reference) + + // Create spec file with module entry + def specFile = new PipelineSpec(tempDir) + specFile.addModuleEntry('@nf-core/fastqc', '1.0.0') + + and: + def cmd = new ModuleRemove() + cmd.args = ['nf-core/fastqc'] + cmd.root = tempDir + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Removing module files') + output.contains('Module files removed successfully') + output.contains('Removing module entry from nextflow_spec.json') + output.contains('Module entry removed from configuration') + !Files.exists(moduleDir) + + and: + def spec = new PipelineSpec(tempDir) + spec.getModules().get('@nf-core/fastqc') == null + } + + def 'should keep config with -keep-config flag'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def moduleDir = createTestModule(storage, reference) + + // Create spec file + def specFile = new PipelineSpec(tempDir) + specFile.addModuleEntry('@nf-core/fastqc', '1.0.0') + + and: + def cmd = new ModuleRemove() + cmd.args = ['nf-core/fastqc'] + cmd.keepConfig = true + cmd.root = tempDir + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Removing module files') + output.contains('Keeping module entry in nextflow_spec.json') + !Files.exists(moduleDir) + + and: + def spec = new PipelineSpec(tempDir) + spec.getModules().get('@nf-core/fastqc') == '1.0.0' + } + + def 'should keep files with -keep-files flag'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def moduleDir = createTestModule(storage, reference) + + // Create spec file + def specFile = new PipelineSpec(tempDir) + specFile.addModuleEntry('@nf-core/fastqc', '1.0.0') + + and: + def cmd = new ModuleRemove() + cmd.args = ['nf-core/fastqc'] + cmd.keepFiles = true + cmd.root = tempDir + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Keeping module files') + output.contains('Removing module entry from nextflow_spec.json') + Files.exists(moduleDir) + Files.exists(moduleDir.resolve('main.nf')) + + and: + def spec = new PipelineSpec(tempDir) + spec.getModules().get('@nf-core/fastqc') == null + } + + def 'should fail when both keep flags are set'() { + given: + def cmd = new ModuleRemove() + cmd.args = ['nf-core/fastqc'] + cmd.keepConfig = true + cmd.keepFiles = true + cmd.root = tempDir + + when: + cmd.run() + + then: + def e = thrown(AbortOperationException) + e.message.contains('Cannot use both -keep-config and -keep-files') + } + + def 'should handle removing non-existent module'() { + given: + def cmd = new ModuleRemove() + cmd.args = ['nf-core/nonexistent'] + cmd.root = tempDir + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('was not installed locally') || output.contains('was not found') + } + + def 'should fail with no arguments'() { + given: + def cmd = new ModuleRemove() + cmd.args = [] + cmd.root = tempDir + + when: + cmd.run() + + then: + thrown(AbortOperationException) + } + + def 'should fail with too many arguments'() { + given: + def cmd = new ModuleRemove() + cmd.args = ['nf-core/fastqc', 'extra-arg'] + cmd.root = tempDir + + when: + cmd.run() + + then: + thrown(AbortOperationException) + } + + private Path createTestModule(ModuleStorage storage, ModuleReference reference) { + def moduleDir = storage.getModuleDir(reference) + Files.createDirectories(moduleDir) + + // Create main.nf + moduleDir.resolve('main.nf').text = ''' + process FASTQC { + input: + path reads + + output: + path "*.html" + + script: + """ + fastqc ${reads} + """ + } + '''.stripIndent() + + // Create meta.yml + moduleDir.resolve('meta.yml').text = ''' + name: nf-core/fastqc + version: 1.0.0 + description: FastQC quality control + '''.stripIndent() + + return moduleDir + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleRunTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleRunTest.groovy new file mode 100644 index 0000000000..cebbc81518 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleRunTest.groovy @@ -0,0 +1,248 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli.module + +import io.seqera.npr.api.schema.v1.Module +import io.seqera.npr.api.schema.v1.ModuleRelease +import nextflow.cli.CliOptions +import nextflow.cli.Launcher +import nextflow.exception.AbortOperationException +import nextflow.module.ModuleReference +import nextflow.module.ModuleRegistryClient +import nextflow.module.ModuleStorage +import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream +import org.junit.Rule +import spock.lang.Specification +import spock.lang.TempDir +import test.OutputCapture + +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.GZIPOutputStream + +/** + * Tests for ModuleRun command + * + * @author Jorge Ejarque + */ +class ModuleRunTest extends Specification { + + @Rule + OutputCapture capture = new OutputCapture() + + @TempDir + Path tempDir + + def 'should run module and create output file'() { + given: + // Create a simple module script that creates a file + def moduleScript = ''' + process CREATE_FILE { + output: + path "test_output.txt" + + script: + """ + echo "Module executed successfully" > test_output.txt + """ + } + '''.stripIndent() + + and: + // Create module directory structure + def storage = new ModuleStorage(tempDir) + def moduleRef = new ModuleReference('nf-core', 'test-module') + def moduleDir = storage.getModuleDir(moduleRef) + Files.createDirectories(moduleDir) + + // Write main.nf + moduleDir.resolve('main.nf').text = moduleScript + + // Write meta.yml + moduleDir.resolve('meta.yml').text = ''' + name: nf-core/test-module + version: 1.0.0 + description: Test module that creates a file + '''.stripIndent() + + and: + // Create mock module package + def modulePackage = createModulePackage(moduleScript) + + // Mock registry client + def mockClient = Stub(ModuleRegistryClient) + def moduleRelease = new ModuleRelease() + moduleRelease.version = '1.0.0' + def module = new Module() + module.name = '@nf-core/test-module' + module.latest = moduleRelease + mockClient.fetchModule(_) >> module // Use wildcard to match any argument + mockClient.downloadModule(_, _, _) >> { String name, String version, Path dest -> + Files.write(dest, modulePackage) + return dest + } + + and: + def cmd = new ModuleRun() + cmd.launcher = Mock(Launcher) { + getOptions() >> new CliOptions() + getCliString() >> "nextflow module run nf-core/test-module" + } + cmd.args = ['nf-core/test-module'] + cmd.root = tempDir + cmd.client = mockClient + + when: + cmd.run() + def stdout = capture + .toString() + .readLines()// remove the log part + .findResults { line -> !line.contains('DEBUG') ? line : null } + .findResults { line -> !line.contains('INFO') ? line : null }.join(" ") + + then: + stdout.contains('Executing module...') + stdout.contains('Process CREATE_FILE Outputs:') + stdout.contains("test_output.txt") + and: + // Verify module was installed + Files.exists(moduleDir) + Files.exists(moduleDir.resolve('main.nf')) + + } + + def 'should run module with specific version'() { + given: + def moduleScript = ''' + process CREATE_FILE_V2 { + output: + path "test_output_v2.txt" + + script: + """ + echo "Module version 2.0.0 executed successfully" > test_output_v2.txt + """ + } + ''' + + and: + def storage = new ModuleStorage(tempDir) + def moduleRef = new ModuleReference('nf-core', 'test-module') + def moduleDir = storage.getModuleDir(moduleRef) + Files.createDirectories(moduleDir) + moduleDir.resolve('main.nf').text = moduleScript + moduleDir.resolve('meta.yml').text = 'name: nf-core/test-module\nversion: 2.0.0' + + and: + def modulePackage = createModulePackage(moduleScript) + + def mockClient = Mock(ModuleRegistryClient) + mockClient.downloadModule('@nf-core/test-module', '2.0.0', _) >> { String name, String version, Path dest -> + Files.write(dest, modulePackage) + return dest + } + + and: + def cmd = new ModuleRun() + cmd.launcher = Mock(Launcher) { + getOptions() >> new CliOptions() + getCliString() >> "nextflow module run nf-core/test-module" + } + cmd.args = ['nf-core/test-module'] + cmd.version = '2.0.0' + cmd.root = tempDir + cmd.client = mockClient + + when: + cmd.run() + + then: + def stdout = capture + .toString() + .readLines()// remove the log part + .findResults { line -> !line.contains('DEBUG') ? line : null } + .findResults { line -> !line.contains('INFO') ? line : null } + .findResults { line -> !line.contains('plugin') ? line : null }.join(" ") + stdout.contains('Executing module...') + stdout.contains('Process CREATE_FILE_V2 Outputs:') + stdout.contains("test_output_v2.txt") + + } + + def 'should fail with no arguments'() { + given: + def cmd = new ModuleRun() + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.args = [] + cmd.root = tempDir + + when: + cmd.run() + + then: + thrown(AbortOperationException) + } + + def 'should fail with invalid module reference'() { + given: + def cmd = new ModuleRun() + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.args = ['invalid-module'] // Missing scope + cmd.root = tempDir + + when: + cmd.run() + + then: + thrown(AbortOperationException) + } + + // Helper method to create a module package (tar.gz) + private byte[] createModulePackage(String mainNfContent) { + def baos = new ByteArrayOutputStream() + + new GZIPOutputStream(baos).withCloseable { gzos -> + new TarArchiveOutputStream(gzos).withCloseable { tos -> + // Add main.nf + addTarEntry(tos, 'main.nf', mainNfContent.bytes) + + // Add meta.yml + def metaContent = ''' + name: test-module + version: 1.0.0 + description: Test module + '''.stripIndent() + addTarEntry(tos, 'meta.yml', metaContent.bytes) + } + } + + return baos.toByteArray() + } + + private void addTarEntry(TarArchiveOutputStream tos, String name, byte[] content) { + def entry = new TarArchiveEntry(name) + entry.setSize(content.length) + tos.putArchiveEntry(entry) + tos.write(content) + tos.closeArchiveEntry() + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleSearchTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleSearchTest.groovy new file mode 100644 index 0000000000..e62be758c9 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleSearchTest.groovy @@ -0,0 +1,225 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli.module + +import groovy.json.JsonSlurper +import io.seqera.npr.api.schema.v1.ModuleSearchResult +import io.seqera.npr.api.schema.v1.SearchModulesResponse +import nextflow.exception.AbortOperationException +import nextflow.module.ModuleRegistryClient +import nextflow.cli.Launcher +import org.junit.Rule +import spock.lang.Specification +import test.OutputCapture + + +/** + * Tests for ModuleSearch command + * + * @author Jorge Ejarque + */ +class ModuleSearchTest extends Specification { + + @Rule + OutputCapture capture = new OutputCapture() + + // No setup needed - using client field for mocking + + def 'should search and display results in formatted output'() { + given: + def result1 = new ModuleSearchResult( + name: 'nf-core/fastqc', + repositoryPath: 'nf-core/modules', + description: 'FastQC quality control', + relevanceScore: 0.95, + keywords: ['quality-control', 'fastqc'], + tools: ['fastqc'], + revoked: false + ) + def result2 = new ModuleSearchResult( + name: 'nf-core/multiqc', + repositoryPath: 'nf-core/modules', + description: 'MultiQC reporting', + relevanceScore: 0.85, + keywords: ['quality-control', 'reporting'], + tools: ['multiqc'], + revoked: false + ) + + and: + def cmd = new ModuleSearch() + cmd.args = ['quality'] + cmd.launcher = Mock(Launcher){ + getOptions() >> null + } + cmd.limit = 20 + + and: + def response = new SearchModulesResponse( + query: 'quality', + totalResults: 2, + results: [result1, result2] + ) + assert response.results + // Mock the registry client directly using the test field + def mockClient = Mock(ModuleRegistryClient) { + search(_, _) >> response + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Searching for') + output.contains('quality') + output.contains('nf-core/fastqc') + output.contains('FastQC quality control') + output.contains('nf-core/multiqc') + output.contains('MultiQC reporting') + + } + + def 'should search and display results in JSON output'() { + given: + def result1 = new ModuleSearchResult( + name: 'nf-core/fastqc', + description: 'FastQC quality control', + relevanceScore: 0.95, + keywords: ['quality-control'], + tools: ['fastqc'], + revoked: false + ) + + and: + def cmd = new ModuleSearch() + cmd.launcher = Mock(Launcher){ + getOptions() >> null + } + cmd.args = ['fastqc'] + cmd.limit = 10 + cmd.jsonOutput = true + + and: + // Mock the registry client + def mockClient = Mock(ModuleRegistryClient) + mockClient.search('fastqc', 10) >> new SearchModulesResponse( + query: 'fastqc', + totalResults: 1, + results: [result1] + ) + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString().readLines().last + def json = new JsonSlurper().parseText(output) + + then: + json.query == 'fastqc' + json.totalResults == 1 + json.results.size() == 1 + json.results[0].name == 'nf-core/fastqc' + json.results[0].description == 'FastQC quality control' + + } + + def 'should handle no search results'() { + given: + def cmd = new ModuleSearch() + cmd.launcher = Mock(Launcher){ + getOptions() >> null + } + cmd.args = ['nonexistent-module'] + cmd.limit = 20 + + and: + // Mock empty results + def mockClient = Mock(ModuleRegistryClient) + mockClient.search('nonexistent-module', 20) >> new SearchModulesResponse( + query: 'nonexistent-module', + totalResults: 0, + results: [] + ) + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('No modules found') + + } + + def 'should fail with no arguments'() { + given: + def cmd = new ModuleSearch() + cmd.launcher = Mock(Launcher){ + getOptions() >> null + } + cmd.args = [] + + when: + cmd.run() + + then: + thrown(AbortOperationException) + } + + def 'should handle search with custom limit'() { + given: + def results = (1..5).collect { i -> + new ModuleSearchResult( + name: "nf-core/module${i}", + description: "Module ${i}", + relevanceScore: 0.9 - (i * 0.1), + keywords: ['test'], + tools: ["tool${i}"], + revoked: false + ) + } + + and: + def cmd = new ModuleSearch() + cmd.launcher = Mock(Launcher){ + getOptions() >> null + } + cmd.args = ['test'] + cmd.limit = 5 + + and: + // Mock the client with 5 results + def mockClient = Mock(ModuleRegistryClient) + mockClient.search('test', 5) >> new SearchModulesResponse( + query: 'test', + totalResults: 5, + results: results + ) + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('nf-core/module1') + output.contains('nf-core/module5') + (1..5).every { i -> output.contains("Module ${i}") } + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/config/ModulesConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/config/ModulesConfigTest.groovy new file mode 100644 index 0000000000..d4c28cbf6f --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/config/ModulesConfigTest.groovy @@ -0,0 +1,197 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.config + +import spock.lang.Specification + +/** + * Tests for ModulesConfig + * + * @author Jorge Ejarque + */ +class ModulesConfigTest extends Specification { + + def 'should create empty config'() { + when: + def config = new ModulesConfig() + + then: + config.getAllModules().isEmpty() + } + + def 'should set and get module version'() { + given: + def config = new ModulesConfig() + + when: + config.setVersion('@nf-core/fastqc', '1.0.0') + + then: + config.getVersion('@nf-core/fastqc') == '1.0.0' + config.hasVersion('@nf-core/fastqc') + } + + def 'should return null for unconfigured module'() { + given: + def config = new ModulesConfig() + + when: + def version = config.getVersion('@nf-core/bwa') + + then: + version == null + !config.hasVersion('@nf-core/bwa') + } + + def 'should override existing version'() { + given: + def config = new ModulesConfig() + config.setVersion('@nf-core/fastqc', '1.0.0') + + when: + config.setVersion('@nf-core/fastqc', '2.0.0') + + then: + config.getVersion('@nf-core/fastqc') == '2.0.0' + } + + def 'should return unmodifiable map from getModules'() { + given: + def config = new ModulesConfig() + config.setVersion('@nf-core/fastqc', '1.0.0') + + when: + def modules = config.getAllModules() + modules.put('@nf-core/bwa', '2.0.0') + + then: + thrown(UnsupportedOperationException) + } + + def 'should return all configured modules'() { + given: + def config = new ModulesConfig() + config.setVersion('@nf-core/fastqc', '1.0.0') + config.setVersion('@nf-core/bwa', '2.0.0') + config.setVersion('@myorg/custom', '0.5.0') + + when: + def modules = config.getAllModules() + + then: + modules.size() == 3 + modules['@nf-core/fastqc'] == '1.0.0' + modules['@nf-core/bwa'] == '2.0.0' + modules['@myorg/custom'] == '0.5.0' + } + + def 'should handle empty initialization'() { + when: + def config = new ModulesConfig(null) + + then: + config.getAllModules().isEmpty() + !config.hasVersion('@nf-core/fastqc') + } + + def 'should store multiple versions independently'() { + given: + def config = new ModulesConfig() + + when: + config.setVersion('@nf-core/fastqc', '1.0.0') + config.setVersion('@nf-core/bwa', '2.0.0') + config.setVersion('@myorg/custom', '0.5.0') + + then: + config.getVersion('@nf-core/fastqc') == '1.0.0' + config.getVersion('@nf-core/bwa') == '2.0.0' + config.getVersion('@myorg/custom') == '0.5.0' + config.allModules.size() == 3 + } + + def 'should handle module names with special characters'() { + given: + def config = new ModulesConfig() + + when: + config.setVersion('@org-name/module-name', '1.0.0') + config.setVersion('@org_name/module_name', '2.0.0') + config.setVersion('simple-module', '3.0.0') + + then: + config.getVersion('@org-name/module-name') == '1.0.0' + config.getVersion('@org_name/module_name') == '2.0.0' + config.getVersion('simple-module') == '3.0.0' + } + + def 'should handle version strings with various formats'() { + given: + def config = new ModulesConfig() + + when: + config.setVersion('@nf-core/fastqc', '1.0.0') + config.setVersion('@nf-core/bwa', 'v2.0.0') + config.setVersion('@nf-core/samtools', '1.0.0-beta') + config.setVersion('@nf-core/bowtie', '1.0.0-rc.1') + + then: + config.getVersion('@nf-core/fastqc') == '1.0.0' + config.getVersion('@nf-core/bwa') == 'v2.0.0' + config.getVersion('@nf-core/samtools') == '1.0.0-beta' + config.getVersion('@nf-core/bowtie') == '1.0.0-rc.1' + } + + def 'should check if multiple modules have versions'() { + given: + def config = new ModulesConfig() + config.setVersion('@nf-core/fastqc', '1.0.0') + config.setVersion('@nf-core/bwa', '2.0.0') + + expect: + config.hasVersion('@nf-core/fastqc') + config.hasVersion('@nf-core/bwa') + !config.hasVersion('@nf-core/samtools') + } + + def 'should handle version updates'() { + given: + def config = new ModulesConfig() + config.setVersion('@nf-core/fastqc', '1.0.0') + + when: + config.setVersion('@nf-core/fastqc', '1.1.0') + config.setVersion('@nf-core/fastqc', '2.0.0') + + then: + config.getVersion('@nf-core/fastqc') == '2.0.0' + } + + def 'should maintain separate versions for different modules'() { + given: + def config = new ModulesConfig() + + when: + config.setVersion('@nf-core/fastqc', '1.0.0') + config.setVersion('@nf-core/bwa', '2.0.0') + + then: + config.getVersion('@nf-core/fastqc') == '1.0.0' + config.getVersion('@nf-core/bwa') == '2.0.0' + config.getVersion('@nf-core/fastqc') != config.getVersion('@nf-core/bwa') + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/config/RegistryConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/config/RegistryConfigTest.groovy new file mode 100644 index 0000000000..02d18f6a6c --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/config/RegistryConfigTest.groovy @@ -0,0 +1,356 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.config + +import spock.lang.Specification + +/** + * Tests for RegistryConfig + * + * @author Jorge Ejarque + */ +class RegistryConfigTest extends Specification { + + def cleanup() { + // Clean up any environment variables set during tests + System.clearProperty('NXF_REGISTRY_TOKEN') + } + + def 'should create config with default values'() { + when: + def config = new RegistryConfig() + + then: + config.url == RegistryConfig.DEFAULT_REGISTRY_URL + config.allUrls == [RegistryConfig.DEFAULT_REGISTRY_URL] + config.getAuthToken(RegistryConfig.DEFAULT_REGISTRY_URL) == null + } + + def 'should initialize with custom URL'() { + given: + def opts = [url: 'https://custom.registry.com'] + + when: + def config = new RegistryConfig(opts) + + then: + config.url == 'https://custom.registry.com' + config.allUrls == ['https://custom.registry.com'] + } + + def 'should initialize with multiple URLs'() { + given: + def opts = [ + urls: [ + 'https://primary.registry.com', + 'https://fallback.registry.com' + ] + ] + + when: + def config = new RegistryConfig(opts) + + then: + config.allUrls == [ + 'https://primary.registry.com', + 'https://fallback.registry.com' + ] + } + + def 'should prefer urls list over single url'() { + given: + def opts = [ + url: 'https://single.registry.com', + urls: [ + 'https://primary.registry.com', + 'https://fallback.registry.com' + ] + ] + + when: + def config = new RegistryConfig(opts) + + then: + config.allUrls == [ + 'https://primary.registry.com', + 'https://fallback.registry.com' + ] + } + + def 'should fall back to default URL when no URL provided'() { + given: + def opts = [:] + + when: + def config = new RegistryConfig(opts) + + then: + config.url == RegistryConfig.DEFAULT_REGISTRY_URL + config.allUrls == [RegistryConfig.DEFAULT_REGISTRY_URL] + } + + def 'should initialize with authentication'() { + given: + def opts = [ + url: 'https://registry.com', + auth: [ + 'https://registry.com': 'token123' + ] + ] + + when: + def config = new RegistryConfig(opts) + + then: + config.getAuthToken('https://registry.com') == 'token123' + } + + def 'should return null for unconfigured auth'() { + given: + def config = new RegistryConfig([url: 'https://registry.com']) + + when: + def token = config.getAuthToken('https://registry.com') + + then: + token == null + } + + def 'should detect environment variable reference format'() { + given: + def opts = [ + url: 'https://registry.com', + auth: [ + 'https://registry.com': '${TEST_TOKEN_VAR}' + ] + ] + def config = new RegistryConfig(opts) + + when: + def token = config.getAuthToken('https://registry.com') + + then: + token == '${TEST_TOKEN_VAR}' + token.startsWith('${') && token.endsWith('}') + } + + def 'should return literal token when not environment variable reference'() { + given: + def opts = [ + url: 'https://registry.com', + auth: [ + 'https://registry.com': 'literal-token' + ] + ] + def config = new RegistryConfig(opts) + + when: + def token = config.getAuthTokenResolved('https://registry.com') + + then: + token == 'literal-token' + } + + def 'should prefer configured auth over environment variable'() { + given: + def opts = [ + url: 'https://registry.com', + auth: [ + 'https://registry.com': 'config-token' + ] + ] + def config = new RegistryConfig(opts) + + when: + def token = config.getAuthTokenResolved('https://registry.com') + + then: + // If NXF_REGISTRY_TOKEN env var exists, configured token takes precedence + token == 'config-token' + } + + def 'should check if auth is configured'() { + given: + def config = new RegistryConfig([ + url: 'https://registry.com', + auth: [ + 'https://registry.com': 'token123' + ] + ]) + + expect: + config.hasAuth('https://registry.com') + !config.hasAuth('https://other-registry.com') + } + + def 'should support multiple registry auths'() { + given: + def opts = [ + urls: [ + 'https://primary.registry.com', + 'https://fallback.registry.com' + ], + auth: [ + 'https://primary.registry.com': 'primary-token', + 'https://fallback.registry.com': 'fallback-token' + ] + ] + def config = new RegistryConfig(opts) + + expect: + config.getAuthToken('https://primary.registry.com') == 'primary-token' + config.getAuthToken('https://fallback.registry.com') == 'fallback-token' + config.hasAuth('https://primary.registry.com') + config.hasAuth('https://fallback.registry.com') + } + + def 'should handle null auth map'() { + given: + def config = new RegistryConfig([url: 'https://registry.com', auth: null]) + + expect: + config.getAuthToken('https://registry.com') == null + !config.hasAuth('https://registry.com') + } + + def 'should handle empty auth map'() { + given: + def config = new RegistryConfig([url: 'https://registry.com', auth: [:]]) + + expect: + config.getAuthToken('https://registry.com') == null + !config.hasAuth('https://registry.com') + } + + def 'should return null when environment variable is not set'() { + given: + def opts = [ + url: 'https://registry.com', + auth: [ + 'https://registry.com': '${NONEXISTENT_VAR}' + ] + ] + def config = new RegistryConfig(opts) + + when: + def token = config.getAuthTokenResolved('https://registry.com') + + then: + token == null + } + + def 'should handle complex URL patterns in auth keys'() { + given: + def opts = [ + auth: [ + 'https://registry.com': 'token1', + 'https://registry.com:8080': 'token2', + 'http://localhost:3000': 'token3' + ] + ] + def config = new RegistryConfig(opts) + + expect: + config.getAuthToken('https://registry.com') == 'token1' + config.getAuthToken('https://registry.com:8080') == 'token2' + config.getAuthToken('http://localhost:3000') == 'token3' + } + + def 'should handle empty urls list gracefully'() { + given: + def opts = [ + url: 'https://registry.com', + urls: [] + ] + + when: + def config = new RegistryConfig(opts) + + then: + config.allUrls == ['https://registry.com'] + } + + def 'should use default URL when both url and urls are empty'() { + given: + def opts = [ + url: null, + urls: [] + ] + + when: + def config = new RegistryConfig(opts) + + then: + config.allUrls == [RegistryConfig.DEFAULT_REGISTRY_URL] + } + + def 'should correctly identify auth when configured'() { + given: + def config = new RegistryConfig([ + url: 'https://registry.com', + auth: ['https://registry.com': 'token'] + ]) + + expect: + config.hasAuth('https://registry.com') + } + + def 'should handle registry URL without trailing slash'() { + given: + def opts = [ + url: 'https://registry.com', + auth: ['https://registry.com': 'token'] + ] + def config = new RegistryConfig(opts) + + expect: + config.getAuthToken('https://registry.com') == 'token' + } + + def 'should handle registry URL with trailing slash'() { + given: + def opts = [ + url: 'https://registry.com/', + auth: ['https://registry.com/': 'token'] + ] + def config = new RegistryConfig(opts) + + expect: + config.getAuthToken('https://registry.com/') == 'token' + } + + def 'should preserve order of URLs in list'() { + given: + def opts = [ + urls: [ + 'https://first.registry.com', + 'https://second.registry.com', + 'https://third.registry.com' + ] + ] + + when: + def config = new RegistryConfig(opts) + + then: + config.allUrls == [ + 'https://first.registry.com', + 'https://second.registry.com', + 'https://third.registry.com' + ] + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleRegistryClientTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleRegistryClientTest.groovy new file mode 100644 index 0000000000..ac4e97c1f4 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleRegistryClientTest.groovy @@ -0,0 +1,433 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import com.github.tomakehurst.wiremock.WireMockServer +import com.github.tomakehurst.wiremock.client.WireMock +import groovy.json.JsonOutput +import nextflow.config.RegistryConfig +import nextflow.exception.AbortOperationException +import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream +import spock.lang.Specification +import spock.lang.TempDir + +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.GZIPOutputStream + +import static com.github.tomakehurst.wiremock.client.WireMock.* +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig + +/** + * Integration tests for ModuleRegistryClient using WireMock + * + * @author Jorge Ejarque + */ +class ModuleRegistryClientTest extends Specification { + + @TempDir + Path tempDir + + WireMockServer wireMock + + def setup() { + wireMock = new WireMockServer(wireMockConfig().dynamicPort()) + wireMock.start() + WireMock.configureFor("localhost", wireMock.port()) + } + + def cleanup() { + wireMock?.stop() + } + + def 'should fetch module metadata from registry'() { + given: + def moduleResponse = [ + name: 'nf-core/fastqc', + description: 'FastQC quality control', + latest: [ + version: '1.1.0', + createdAt: '2024-02-01T00:00:00Z' + ] + ] + + // Note: nf-core/fastqc is URL-encoded as nf-core%2Ffastqc + stubFor(get(urlEqualTo('/api/modules/nf-core%2Ffastqc')) + .willReturn(aResponse() + .withStatus(200) + .withHeader('Content-Type', 'application/json') + .withBody(JsonOutput.toJson(moduleResponse)))) + + and: + def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def client = new ModuleRegistryClient(config) + + when: + def result = client.fetchModule('nf-core/fastqc') + + then: + result != null + result.name == 'nf-core/fastqc' + result.latest != null + result.latest.version == '1.1.0' + + and: 'verify request was made' + verify(getRequestedFor(urlEqualTo('/api/modules/nf-core%2Ffastqc'))) + } + + def 'should search modules in registry'() { + given: + def searchResponse = [ + query: 'fastqc', + totalResults: 2, + results: [ + [ + name: 'nf-core/fastqc', + description: 'FastQC quality control', + relevanceScore: 0.95, + keywords: ['quality-control'], + tools: ['fastqc'], + revoked: false + ], + [ + name: 'other/fastqc', + description: 'Another FastQC module', + relevanceScore: 0.75, + keywords: ['qc'], + tools: ['fastqc'], + revoked: false + ] + ] + ] + + stubFor(get(urlPathEqualTo('/api/modules')) + .withQueryParam('query', equalTo('fastqc')) + .withQueryParam('limit', equalTo('10')) + .willReturn(aResponse() + .withStatus(200) + .withHeader('Content-Type', 'application/json') + .withBody(JsonOutput.toJson(searchResponse)))) + + and: + def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def client = new ModuleRegistryClient(config) + + when: + def result = client.search('fastqc', 10) + + then: + result != null + result.query == 'fastqc' + result.totalResults == 2 + result.results.size() == 2 + result.results[0].name == 'nf-core/fastqc' + // Use closeTo for Float/BigDecimal comparison + Math.abs(result.results[0].relevanceScore - 0.95) < 0.001 + + and: 'verify query parameters' + verify(getRequestedFor(urlPathEqualTo('/api/modules')) + .withQueryParam('query', equalTo('fastqc')) + .withQueryParam('limit', equalTo('10'))) + } + + def 'should download module package from registry'() { + given: + def modulePackage = createTestModulePackage() + def expectedChecksum = "${computeSha256(modulePackage)}" + + // Note: URL-encoded path + stubFor(get(urlEqualTo('/api/modules/nf-core%2Ffastqc/1.0.0/download')) + .willReturn(aResponse() + .withStatus(200) + .withHeader('Content-Type', 'application/gzip') + .withHeader('X-Checksum', "sha256:${expectedChecksum}") + .withBody(modulePackage))) + + and: + def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def client = new ModuleRegistryClient(config) + def destFile = tempDir.resolve('module.tgz') + + when: + def result = client.downloadModule('nf-core/fastqc', '1.0.0', destFile) + + then: + result == destFile + Files.exists(destFile) + Files.size(destFile) == modulePackage.length + + and: + verify(getRequestedFor(urlEqualTo('/api/modules/nf-core%2Ffastqc/1.0.0/download'))) + } + + def 'should successfully fetch module without authentication'() { + given: + stubFor(get(urlEqualTo('/api/modules/nf-core%2Ffastqc')) + .willReturn(aResponse() + .withStatus(200) + .withHeader('Content-Type', 'application/json') + .withBody(JsonOutput.toJson([name: 'nf-core/fastqc', latest: [version: '1.0.0']])))) + + and: + def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def client = new ModuleRegistryClient(config) + + when: + def result = client.fetchModule('nf-core/fastqc') + + then: + result != null + result.name == 'nf-core/fastqc' + + and: 'verify request was made' + verify(getRequestedFor(urlEqualTo('/api/modules/nf-core%2Ffastqc'))) + } + + def 'should handle 404 not found error'() { + given: + stubFor(get(urlEqualTo('/api/modules/nf-core%2Fnonexistent')) + .willReturn(aResponse() + .withStatus(404) + .withBody('Module not found'))) + + and: + def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def client = new ModuleRegistryClient(config) + + when: + client.fetchModule('nf-core/nonexistent') + + then: + def ex = thrown(AbortOperationException) + ex.message.contains('Unable to fetch module') || ex.message.contains('Module not found') + + and: + verify(getRequestedFor(urlEqualTo('/api/modules/nf-core%2Fnonexistent'))) + } + + def 'should handle 500 server error'() { + given: + stubFor(get(urlEqualTo('/api/modules/nf-core%2Ffastqc')) + .willReturn(aResponse() + .withStatus(500) + .withBody('Internal Server Error'))) + + and: + def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def client = new ModuleRegistryClient(config) + + when: + client.fetchModule('nf-core/fastqc') + + then: + thrown(AbortOperationException) + } + + def 'should send user agent header'() { + given: + stubFor(get(urlEqualTo('/api/modules/nf-core%2Ffastqc')) + .willReturn(aResponse() + .withStatus(200) + .withHeader('Content-Type', 'application/json') + .withBody(JsonOutput.toJson([name: 'nf-core/fastqc', latest: [version: '1.0.0'], releases: []])))) + + and: + def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def client = new ModuleRegistryClient(config) + + when: + client.fetchModule('nf-core/fastqc') + + then: + verify(getRequestedFor(urlEqualTo('/api/modules/nf-core%2Ffastqc')) + .withHeader('User-Agent', matching('.*'))) + } + + def 'should handle empty search results'() { + given: + stubFor(get(urlPathEqualTo('/api/modules')) + .withQueryParam('query', equalTo('nonexistent')) + .willReturn(aResponse() + .withStatus(200) + .withHeader('Content-Type', 'application/json') + .withBody(JsonOutput.toJson([ + query: 'nonexistent', + totalResults: 0, + results: [] + ])))) + + and: + def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def client = new ModuleRegistryClient(config) + + when: + def result = client.search('nonexistent', 10) + + then: + result != null + result.totalResults == 0 + result.results.isEmpty() + } + + def 'should respect custom search limit'() { + given: + stubFor(get(urlPathEqualTo('/api/modules')) + .willReturn(aResponse() + .withStatus(200) + .withHeader('Content-Type', 'application/json') + .withBody(JsonOutput.toJson([ + query: 'test', + totalResults: 0, + results: [] + ])))) + + and: + def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def client = new ModuleRegistryClient(config) + + when: + client.search('test', 25) + + then: + verify(getRequestedFor(urlPathEqualTo('/api/modules')) + .withQueryParam('limit', equalTo('25'))) + } + + def 'should handle malformed JSON response'() { + given: + stubFor(get(urlEqualTo('/api/modules/nf-core%2Ffastqc')) + .willReturn(aResponse() + .withStatus(200) + .withHeader('Content-Type', 'application/json') + .withBody('not valid json {]'))) + + and: + def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def client = new ModuleRegistryClient(config) + + when: + client.fetchModule('nf-core/fastqc') + + then: + thrown(AbortOperationException) + } + + def 'should verify download includes checksum header'() { + given: + def modulePackage = createTestModulePackage() + def checksum = "sha256:${computeSha256(modulePackage)}" + + stubFor(get(urlEqualTo('/api/modules/nf-core%2Ffastqc/1.0.0/download')) + .willReturn(aResponse() + .withStatus(200) + .withHeader('Content-Type', 'application/gzip') + .withHeader('X-Checksum', checksum) + .withHeader('Docker-Content-Digest', checksum) + .withBody(modulePackage))) + + and: + def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def client = new ModuleRegistryClient(config) + def destFile = tempDir.resolve('module.tgz') + + when: + def result = client.downloadModule('nf-core/fastqc', '1.0.0', destFile) + + then: + result == destFile + Files.exists(destFile) + + and: 'verify checksum header was present' + verify(getRequestedFor(urlEqualTo('/api/modules/nf-core%2Ffastqc/1.0.0/download'))) + } + + def 'should handle network errors gracefully'() { + given: + // Stub will not be set up, causing connection refused + def config = new RegistryConfig([url: "http://localhost:9999"]) // Invalid port + def client = new ModuleRegistryClient(config) + + when: + client.fetchModule('nf-core/fastqc') + + then: + thrown(AbortOperationException) + } + + def 'should require authentication for publish'() { + given: + def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def client = new ModuleRegistryClient(config) + def publishRequest = [name: 'nf-core/mymodule', version: '1.0.0'] + + when: + client.publishModule('nf-core/mymodule', publishRequest) + + then: + def ex = thrown(AbortOperationException) + ex.message.contains('Authentication required') + } + + def 'should handle publish failure with no auth token'() { + given: + def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def client = new ModuleRegistryClient(config) + def publishRequest = [name: 'nf-core/mymodule', version: '1.0.0'] + + when: + client.publishModule('nf-core/mymodule', publishRequest) + + then: + def ex = thrown(AbortOperationException) + ex.message.contains('Authentication required') + } + + // Helper methods + + private byte[] createTestModulePackage() { + def baos = new ByteArrayOutputStream() + + new GZIPOutputStream(baos).withCloseable { gzos -> + new TarArchiveOutputStream(gzos).withCloseable { tos -> + // Add main.nf + def mainContent = 'process TEST { script: "echo test" }' + addTarEntry(tos, 'main.nf', mainContent.bytes) + + // Add meta.yml + def metaContent = 'name: test\nversion: 1.0.0' + addTarEntry(tos, 'meta.yml', metaContent.bytes) + } + } + + return baos.toByteArray() + } + + private void addTarEntry(TarArchiveOutputStream tos, String name, byte[] content) { + def entry = new TarArchiveEntry(name) + entry.setSize(content.length) + tos.putArchiveEntry(entry) + tos.write(content) + tos.closeArchiveEntry() + } + + private String computeSha256(byte[] data) { + def digest = java.security.MessageDigest.getInstance('SHA-256') + def hash = digest.digest(data) + return hash.encodeHex().toString() + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverTest.groovy new file mode 100644 index 0000000000..8c145bb89c --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverTest.groovy @@ -0,0 +1,194 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import nextflow.config.ModulesConfig +import nextflow.exception.AbortOperationException +import nextflow.file.FileHelper +import spock.lang.Specification +import spock.lang.TempDir + +import java.nio.file.Files +import java.nio.file.Path + +/** + * Tests for ModuleResolver + * + * @author Jorge Ejarque + */ +class ModuleResolverTest extends Specification { + + @TempDir + Path tempDir + + def 'should throw exception when resolving non-installed module without auto-install'() { + given: + def resolver = new ModuleResolver(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + + when: + resolver.resolve(reference, null, false) + + then: + def e = thrown(AbortOperationException) + e.message.contains('not installed') + e.message.contains('nextflow module install') + } + + def 'should throw exception when installed module is corrupted'() { + given: + def resolver = new ModuleResolver(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def storage = new ModuleStorage(tempDir) + def moduleDir = storage.getModuleDir(reference) + + // Create corrupted module (directory exists but no main.nf) + Files.createDirectories(moduleDir) + moduleDir.resolve('meta.yml').text = ''' + name: nf-core/fastqc + version: 1.0.0 + ''' + + when: + resolver.resolve(reference, null, false) + + then: + def e = thrown(AbortOperationException) + e.message.contains('corrupted') + + cleanup: + FileHelper.deletePath(moduleDir) + } + + def 'should warn about locally modified module'() { + given: + def resolver = new ModuleResolver(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def storage = new ModuleStorage(tempDir) + def moduleDir = storage.getModuleDir(reference) + + // Create module with mismatched checksum + Files.createDirectories(moduleDir) + moduleDir.resolve('main.nf').text = 'process TEST { }' + moduleDir.resolve('meta.yml').text = ''' + name: nf-core/fastqc + version: 1.0.0 + ''' + moduleDir.resolve('.checksum').text = 'wrong-checksum' + + when: + def result = resolver.resolve(reference, null, false) + + then: + result != null + result == moduleDir.resolve('main.nf') + + cleanup: + FileHelper.deletePath(moduleDir) + } + + def 'should throw exception when version mismatch without auto-install'() { + given: + def modulesConfig = new ModulesConfig(['@nf-core/fastqc': '2.0.0']) + def resolver = new ModuleResolver(tempDir, modulesConfig, null) + def reference = new ModuleReference('nf-core', 'fastqc') + def storage = new ModuleStorage(tempDir) + def moduleDir = storage.getModuleDir(reference) + + // Create module with different version + Files.createDirectories(moduleDir) + moduleDir.resolve('main.nf').text = 'process TEST { }' + moduleDir.resolve('meta.yml').text = ''' + name: nf-core/fastqc + version: 1.0.0 + ''' + + // Compute and save correct checksum + def checksum = ModuleChecksum.compute(moduleDir) + moduleDir.resolve('.checksum').text = checksum + + when: + resolver.resolve(reference, null, false) + + then: + def e = thrown(AbortOperationException) + e.message.contains('version mismatch') + e.message.contains('installed=1.0.0') + e.message.contains('required=2.0.0') + + cleanup: + FileHelper.deletePath(moduleDir) + } + + def 'should resolve installed module with matching version'() { + given: + def resolver = new ModuleResolver(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def storage = new ModuleStorage(tempDir) + def moduleDir = storage.getModuleDir(reference) + + // Create valid module + Files.createDirectories(moduleDir) + def mainFile = moduleDir.resolve('main.nf') + mainFile.text = 'process TEST { }' + moduleDir.resolve('meta.yml').text = ''' + name: nf-core/fastqc + version: 1.0.0 + ''' + + // Compute and save correct checksum + def checksum = ModuleChecksum.compute(moduleDir) + moduleDir.resolve('.checksum').text = checksum + + when: + def result = resolver.resolve(reference, '1.0.0', false) + + then: + result == mainFile + + cleanup: + FileHelper.deletePath(moduleDir) + } + + def 'should throw exception when trying to update a module with local modifications without force'() { + given: + def resolver = new ModuleResolver(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def storage = new ModuleStorage(tempDir) + def moduleDir = storage.getModuleDir(reference) + + // Create module with wrong checksum (simulating local modifications) + Files.createDirectories(moduleDir) + moduleDir.resolve('main.nf').text = 'process TEST { }' + moduleDir.resolve('meta.yml').text = ''' + name: nf-core/fastqc + version: 1.0.0 + ''' + moduleDir.resolve('.checksum').text = 'wrong-checksum' + + when: + resolver.installModule(reference, '2.0.0', false) + + then: + def e = thrown(AbortOperationException) + e.message.contains('local modifications') + e.message.contains('--force') + + cleanup: + FileHelper.deletePath(moduleDir) + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleSpecTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleSpecTest.groovy index aa53e38364..cf90a07c7c 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleSpecTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleSpecTest.groovy @@ -149,12 +149,17 @@ requires: errors.isEmpty() == valid where: - name | valid - 'nf-core/fastqc' | true - 'myorg/my-module' | true - 'org_1/tool_2' | true - 'fastqc' | false - '@nf-core/fastqc' | false - 'nf-core/fast qc' | false + name | valid + 'nf-core/fastqc' | true + 'myorg/my-module' | true + 'org_1/tool_2' | true + 'nf-core/gfatools/gfa2fa' | true // nested module path + 'myorg/tools/sub/module' | true // deeply nested + 'org.name/tool/sub' | true // dot in scope + 'fastqc' | false + '@nf-core/fastqc' | false + 'nf-core/fast qc' | false + 'nf-core/' | false // trailing slash + '/nf-core/fastqc' | false // leading slash } } diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy index 6c41c40c41..dab9517ddc 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy @@ -161,6 +161,50 @@ class ModuleStorageTest extends Specification { installed*.reference.fullName.sort() == ['@myorg/custom', '@nf-core/fastqc', '@nf-core/multiqc'] } + def 'should list nested modules recursively'() { + given: + def storage = new ModuleStorage(tempDir) + + // Create modules with nested paths + def modules = [ + new ModuleReference('nf-core', 'fastqc'), + new ModuleReference('nf-core', 'gfatools/gfa2fa'), + new ModuleReference('nf-core', 'gfatools/gfa2gfa'), + new ModuleReference('myorg', 'tools/subtools/module') + ] + + modules.each { ref -> + def moduleDir = storage.getModuleDir(ref) + Files.createDirectories(moduleDir) + + // Create main.nf + moduleDir.resolve('main.nf').text = 'process TEST { }' + + // Create meta.yml with version + moduleDir.resolve('meta.yml').text = """ + name: ${ref.nameWithoutPrefix} + version: 1.0.0 + description: Test module + license: MIT + """.stripIndent() + + // Create .checksum + moduleDir.resolve('.checksum').text = 'checksum' + } + + when: + def installed = storage.listInstalled() + + then: + installed.size() == 4 + installed*.reference.fullName.sort() == [ + '@myorg/tools/subtools/module', + '@nf-core/fastqc', + '@nf-core/gfatools/gfa2fa', + '@nf-core/gfatools/gfa2gfa' + ] + } + def 'should return empty list when no modules installed'() { given: def storage = new ModuleStorage(tempDir) diff --git a/modules/nf-commons/build.gradle b/modules/nf-commons/build.gradle index 6be82a655c..308b2ac1ee 100644 --- a/modules/nf-commons/build.gradle +++ b/modules/nf-commons/build.gradle @@ -38,7 +38,7 @@ dependencies { api 'io.seqera:lib-retry:2.0.0' // patch gson dependency required by pf4j api 'com.google.code.gson:gson:2.13.1' - api 'io.seqera:npr-api:0.6.1' + api 'io.seqera:npr-api:0.20.1' /* testImplementation inherited from top gradle build file */ testImplementation(testFixtures(project(":nextflow"))) From 31bf49c542c691e0edbb7143b8350f45fe29155c Mon Sep 17 00:00:00 2001 From: jorgee Date: Tue, 10 Feb 2026 11:37:32 +0100 Subject: [PATCH 05/23] add registry config as config extension Signed-off-by: jorgee --- .../src/main/groovy/nextflow/config/RegistryConfig.groovy | 6 +++--- modules/nextflow/src/main/resources/META-INF/extensions.idx | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/config/RegistryConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/config/RegistryConfig.groovy index c87f3021a1..24ee57063f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/RegistryConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/RegistryConfig.groovy @@ -39,15 +39,15 @@ class RegistryConfig implements ConfigScope { @ConfigOption @Description("Primary registry URL") - private String url + final private String url @ConfigOption @Description("List of registry URLs to try in order") - private List urls + final private List urls @ConfigOption @Description("Authentication configuration per registry (registry URL -> token)") - private Map auth + final private Map auth /* required by extension point -- do not remove */ RegistryConfig() { diff --git a/modules/nextflow/src/main/resources/META-INF/extensions.idx b/modules/nextflow/src/main/resources/META-INF/extensions.idx index 7250b10d76..f9075fb3d9 100644 --- a/modules/nextflow/src/main/resources/META-INF/extensions.idx +++ b/modules/nextflow/src/main/resources/META-INF/extensions.idx @@ -19,6 +19,7 @@ nextflow.conda.CondaConfig nextflow.config.ConfigMap nextflow.config.Manifest nextflow.config.WorkflowConfig +nextflow.config.RegistryConfig nextflow.container.ApptainerConfig nextflow.container.CharliecloudConfig nextflow.container.DockerConfig From b091d734154226d8a2ede7b6b77a606acd6cc138 Mon Sep 17 00:00:00 2001 From: jorgee Date: Wed, 11 Feb 2026 17:03:04 +0100 Subject: [PATCH 06/23] fix NPE in nextflow CLI help Signed-off-by: jorgee --- .../nextflow/src/main/groovy/nextflow/cli/Launcher.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy index 9b3db289fb..a170e04892 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy @@ -159,8 +159,10 @@ class Launcher { fullVersion = '-version' in normalizedArgs command = allCommands.find { it.name == jcommander.getParsedCommand() } //Attach unknown options to command in case of needed - final unknownOptions = jcommander.commands.get(jcommander.getParsedCommand()).getUnknownOptions() - command.setUnknownOptions(unknownOptions) + if (command) { + final unknownOptions = jcommander.commands.get(jcommander.getParsedCommand())?.getUnknownOptions() ?: [] + command.setUnknownOptions(unknownOptions) + } // whether is running a daemon daemonMode = command instanceof CmdNode // set the log file name From a03f6303621be8e1e5940376d34e22cc85bd2e5e Mon Sep 17 00:00:00 2001 From: jorgee Date: Thu, 19 Feb 2026 11:59:15 +0100 Subject: [PATCH 07/23] add module info and expect /api path in the config registry url, update to v1 registry and other required fixes Signed-off-by: jorgee --- docs/cli.md | 16 + docs/reference/cli.md | 27 + .../main/groovy/nextflow/cli/CmdModule.groovy | 28 +- .../nextflow/cli/module/ModuleInfo.groovy | 283 ++++++ .../nextflow/cli/module/ModuleSearch.groovy | 9 - .../module/ModuleRegistryClient.groovy | 10 +- .../nextflow/cli/module/ModuleInfoTest.groovy | 819 ++++++++++++++++++ .../module/ModuleRegistryClientTest.groovy | 68 +- 8 files changed, 1201 insertions(+), 59 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInfo.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInfoTest.groovy diff --git a/docs/cli.md b/docs/cli.md index a4283d47de..075ffa9a8c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -344,6 +344,22 @@ Results include module names, versions, descriptions, and download statistics. U See {ref}`cli-module-search` for more information. +### Viewing module information + +The `module info` command displays detailed metadata and usage information for a specific module from the registry. + +Use this to understand module requirements, view input/output specifications, see available tools, or generate usage templates before installing or running a module. + +```console +$ nextflow module info nf-core/fastqc +$ nextflow module info nf-core/fastqc -version 1.0.0 +$ nextflow module info nf-core/fastqc -json +``` + +The output includes the module's version, description, authors, keywords, tools, input/output channels, and a generated usage template showing how to run the module. Use `-json` for machine-readable output suitable for programmatic access. + +See {ref}`cli-module-info` for more information. + ### Removing modules The `module remove` command deletes modules from your project, removing local files and configuration entries. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 832af687ed..1e43be63c8 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1249,6 +1249,33 @@ The `module` command provides a comprehensive system for managing reusable, regi $ nextflow module search bwa -json ``` +(cli-module-info)= + +`info [options] [scope/name]` + +: Display detailed information about a module from the registry. +: Shows module metadata, version, description, authors, keywords, tools, input/output specifications, and generates a usage template. +: The following options are available: + + `-version` + : Specify the module version to query (e.g., `1.0.0`). If not specified, displays information for the latest version. + + `-json` + : Output results in JSON format for programmatic processing. + +: **Examples:** + + ```console + # Display information for latest version + $ nextflow module info nf-core/fastqc + + # Display information for specific version + $ nextflow module info nf-core/fastqc -version 1.0.0 + + # Get results as JSON + $ nextflow module info nf-core/fastqc -json + ``` + (cli-module-remove)= `remove [options] [scope/name]` diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdModule.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdModule.groovy index d43cbf3ddc..e5312f7d56 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdModule.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdModule.groovy @@ -22,6 +22,7 @@ import com.beust.jcommander.ParameterException import com.beust.jcommander.Parameters import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import nextflow.cli.module.ModuleInfo import nextflow.cli.module.ModuleInstall import nextflow.cli.module.ModuleList import nextflow.cli.module.ModulePublish @@ -52,11 +53,12 @@ class CmdModule extends CmdBase implements UsageAware { commands << new ModuleList() commands << new ModuleRemove() commands << new ModuleSearch() + commands << new ModuleInfo() commands << new ModulePublish() } - protected JCommander commander(){ - if (!this.jCommander) { + protected JCommander commander() { + if( !this.jCommander ) { this.jCommander = new JCommander(this) this.jCommander.setProgramName('nextflow module') // Register all subcommands @@ -81,25 +83,29 @@ class CmdModule extends CmdBase implements UsageAware { try { + if( !args ) { + usage() + return + } final jc = commander() final moduleArgs = args + unknownOptions jc.parse(moduleArgs as String[]) final parsedCommand = jc.getParsedCommand() - if (!parsedCommand) { + if( !parsedCommand ) { jc.usage() return } // Get the parsed subcommand instance final subcommand = jc.getCommands() - .get(parsedCommand) - .getObjects()[0] as CmdBase + .get(parsedCommand) + .getObjects()[0] as CmdBase // Execute with fields already populated by JCommander subcommand.run() - } catch ( ParameterException e) { + } catch( ParameterException e ) { throw new AbortOperationException("${e.getMessage()} -- Check the available commands and options and syntax with 'nextflow module -h'") } } @@ -124,7 +130,7 @@ class CmdModule extends CmdBase implements UsageAware { @Override void usage(List args) { def result = [] - if (!args) { + if( !args ) { result << 'Usage: nextflow module [options]' result << '' result << 'Commands:' @@ -134,13 +140,11 @@ class CmdModule extends CmdBase implements UsageAware { } result << '' println result.join('\n').toString() - } - else { + } else { final sub = findCmd(args[0]) - if (sub) { + if( sub ) { commander().usage(args[0]) - } - else { + } else { throw new AbortOperationException("Unknown module sub-command: ${args[0]}") } } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInfo.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInfo.groovy new file mode 100644 index 0000000000..ebf4a03897 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInfo.groovy @@ -0,0 +1,283 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli.module + +import com.beust.jcommander.Parameter +import com.beust.jcommander.Parameters +import groovy.json.JsonOutput +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.seqera.npr.api.schema.v1.ModuleChannel +import io.seqera.npr.api.schema.v1.ModuleChannelItem +import io.seqera.npr.api.schema.v1.ModuleMetadata +import io.seqera.npr.api.schema.v1.ModuleRelease +import io.seqera.npr.api.schema.v1.ModuleTool +import nextflow.cli.CmdBase +import nextflow.config.ConfigBuilder +import nextflow.config.RegistryConfig +import nextflow.exception.AbortOperationException +import nextflow.module.InstalledModule +import nextflow.module.ModuleReference +import nextflow.module.ModuleRegistryClient +import nextflow.module.ModuleSpec +import nextflow.module.ModuleStorage +import nextflow.util.TestOnly + +import java.nio.file.Path +import java.nio.file.Paths + +/** + * Module info subcommand - displays module metadata and usage template + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +@Parameters(commandDescription = "Show module information and usage template") +class ModuleInfo extends CmdBase { + + @Parameter(names = ["-version"], description = "Module version") + String version + + @Parameter(names = ["-json"], description = "Output in JSON format", arity = 0) + boolean jsonOutput = false + + @Parameter(description = "[scope/name]", required = true) + List args + + @TestOnly + protected Path root + + @TestOnly + protected ModuleRegistryClient client + + @Override + String getName() { + return 'info' + } + + @Override + void run() { + if( !args || args.size() != 1 ) { + throw new AbortOperationException("Incorrect number of arguments") + } + + def moduleRef = '@' + args[0] + def reference = ModuleReference.parse(moduleRef) + + // Get config + def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize() + def config = new ConfigBuilder() + .setOptions(launcher.options) + .setBaseDir(baseDir) + .build() + final registryConfig = config.navigate('registry') as RegistryConfig + + + // Fetch full metadata from registry to get input/output parameters + def registryClient = this.client ?: new ModuleRegistryClient(registryConfig) + ModuleRelease release = null + + try { + if( version ) { + release = registryClient.fetchRelease(reference.fullName, version) + + } else { + release = registryClient.fetchModule(reference.fullName).latest + } + } catch( Exception e ) { + log.warn "Failed to fetch metadata from registry: ${e.message}" + } + if( release?.metadata ) { + log.info("No metadata found for $reference.nameWithoutPrefix ${release?.version ? "($release.version)" : ''}") + } + if( jsonOutput ) { + printJsonInfo(reference, release) + } else { + printFormattedInfo(reference, release) + } + } + + private void printFormattedInfo(ModuleReference reference, ModuleRelease release) { + ModuleMetadata metadata = release.metadata + println "" + println "Module: ${reference.nameWithoutPrefix}" + println "Version: ${release.version}" + println "Description: ${metadata.description ?: release.description ?: 'N/A'}" + + if( metadata.authors ) { + println "Authors: ${metadata.authors.join(', ')}" + } + + if( metadata.maintainers ) { + println "Maintainers: ${metadata.maintainers.join(', ')}" + } + + if( metadata.keywords ) { + println "Keywords: ${metadata.keywords.join(', ')}" + } + + printToolsInfo(metadata?.tools ?: []) + + printInputsInfo(metadata.input ?: []) + + printOutputsInfo(metadata.output ?: [:]) + + // Generate and display usage template + println "" + println "Usage Template:" + println "-" * 80 + println generateUsageTemplate(reference, metadata) + println "" + } + + private void printOutputsInfo(Map outputs) { + if( outputs ) { + println "" + println "Output:" + outputs.each { name, output -> + println "- ${name} ${output.tuple ? '(tuple)' : ''}" + displayChannel("\t", output) + } + } + } + + private void printInputsInfo(List inputs) { + if( inputs ) { + println "" + println "Input:" + inputs.each { input -> + if( input.tuple ) { + println "- (tuple)" + displayChannel("\t", input) + } else + displayChannel("", input) + } + } + } + + private void printToolsInfo(List toolsList) { + if( toolsList ) { + println "" + println "Tools:" + toolsList.each { tool -> + println " - ${tool.name}${tool.version ? ' v' + tool.version : ''}" + if( tool.homepage ) { + println " Homepage: ${tool.homepage}" + } + } + } + } + + private void displayChannel(String prefix, ModuleChannel channel) { + channel.items.each { ModuleChannelItem item -> + println "${prefix}- ${item.name}${item.type ? ' (' + item.type + ')' : ''}" + if( item.description ) { + println "${prefix}\t${item.description.replaceAll(/\R/, ' ')}" + } + if( item.pattern ) { + println "${prefix}\tPattern: ${item.pattern}" + } + } + } + + private String generateUsageTemplate(ModuleReference reference, ModuleMetadata metadata) { + def template = new StringBuilder() + template.append("nextflow module run ${reference.nameWithoutPrefix}") + if( version ) + template.append(" -version $version") + + // Use metadata from registry if available, otherwise use spec from meta.yml + def inputs = metadata?.input ?: [] + + if( inputs ) { + inputs.each { input -> + input.items.each { ModuleChannelItem item -> + def placeholder = item.name.toUpperCase().replaceAll(/[^A-Z0-9_]/, '_') + if( item.type.equalsIgnoreCase("map") ) { + template.append(" --${item.name}. <${placeholder}_KEY>") + } else { + template.append(" --${item.name} <${placeholder}>") + } + } + } + } + return template.toString() + } + + private void printJsonInfo(ModuleReference reference, ModuleRelease release) { + def metadata = release?.metadata + def info = [ + name : reference.nameWithoutPrefix, + fullName : reference.fullName, + version : release.version, + description: metadata.description ?: release.description, + authors : metadata.authors, + keywords : metadata.keywords, + ] + + def toolsList = metadata.tools ?: [] + if( toolsList ) { + info.tools = toolsList.collect { tool -> + return [ + name : tool.name, + version : tool.version, + homepage : tool.homepage, + documentation: tool.documentation + ] + } + } + + def inputs = metadata.input ?: [] + if( inputs ) { + info.input = inputs.collect { input -> + return [ + tuple: input.tuple, + items: input.items?.collect { item -> + [ + name : item.name, + type : item.type, + description: item.description, + pattern : item.pattern + ] + } + ] + } + } + + def outputs = metadata.output ?: [:] + if( outputs ) { + info.output = outputs.collectEntries { name, output -> + return [name, [ + tuple: output.tuple, + items: output.items?.collect { item -> + [ + name : item.name, + type : item.type, + description: item.description, + pattern : item.pattern + ] + } + ]] + } + } + + info.usageTemplate = generateUsageTemplate(reference, metadata) + + println JsonOutput.prettyPrint(JsonOutput.toJson(info)) + } +} \ No newline at end of file diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy index af92dc52fd..3698067319 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy @@ -109,18 +109,9 @@ class ModuleSearch extends CmdBase { response.results.each { ModuleSearchResult result -> println " ${result.name}" - if (result.relevanceScore != null) { - println " Relevance: ${String.format('%.2f', result.relevanceScore)}" - } if (result.description) { println " Description: ${result.description}" } - if (result.keywords && !result.keywords.isEmpty()) { - println " Keywords: ${result.keywords.join(', ')}" - } - if (result.tools && !result.tools.isEmpty()) { - println " Tools: ${result.tools.join(', ')}" - } println "" } } diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy index 960ee9b03e..7c54d8c73a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy @@ -92,7 +92,7 @@ class ModuleRegistryClient { * Fetch module from a specific registry URL */ private Module fetchModuleFromRegistry(String registryUrl, String name) { - def endpoint = "${registryUrl}/api/modules/${encodeName(name)}" + def endpoint = "${registryUrl}/v1/modules/${encodeName(name)}" def uri = URI.create(endpoint) def requestBuilder = HttpRequest.newBuilder() @@ -170,7 +170,7 @@ class ModuleRegistryClient { * Fetch release from a specific registry URL */ private ModuleRelease fetchReleaseFromRegistry(String registryUrl, String name, String version) { - def endpoint = "${registryUrl}/api/modules/${encodeName(name)}/${version}" + def endpoint = "${registryUrl}/v1/modules/${encodeName(name)}/${version}" def uri = URI.create(endpoint) def requestBuilder = HttpRequest.newBuilder() @@ -245,7 +245,7 @@ class ModuleRegistryClient { * Download module from a specific registry URL */ private Path downloadModuleFromRegistry(String registryUrl, String name, String version, Path targetPath) { - def endpoint = "${registryUrl}/api/modules/${encodeName(name)}/${version}/download" + def endpoint = "${registryUrl}/v1/modules/${encodeName(name)}/${version}/download" def uri = URI.create(endpoint) def requestBuilder = HttpRequest.newBuilder() @@ -361,7 +361,7 @@ class ModuleRegistryClient { * Search in a specific registry */ private SearchModulesResponse searchInRegistry(String registryUrl, String query, int limit) { - def endpoint = "${registryUrl}/api/modules?query=${URLEncoder.encode(query, 'UTF-8')}&limit=${limit}" + def endpoint = "${registryUrl}/v1/modules?query=${URLEncoder.encode(query, 'UTF-8')}&limit=${limit}" def uri = URI.create(endpoint) def requestBuilder = HttpRequest.newBuilder() @@ -439,7 +439,7 @@ class ModuleRegistryClient { def request, String authToken) { - String endpoint = "${registryUrl}/api/modules/${encodeName(name)}".toString() + String endpoint = "${registryUrl}/v1/modules/${encodeName(name)}".toString() URI uri = URI.create(endpoint) // Serialize request to JSON diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInfoTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInfoTest.groovy new file mode 100644 index 0000000000..f0d721e62b --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInfoTest.groovy @@ -0,0 +1,819 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli.module + +import groovy.json.JsonSlurper +import io.seqera.npr.api.schema.v1.Module +import io.seqera.npr.api.schema.v1.ModuleChannel +import io.seqera.npr.api.schema.v1.ModuleChannelItem +import io.seqera.npr.api.schema.v1.ModuleMetadata +import io.seqera.npr.api.schema.v1.ModuleRelease +import io.seqera.npr.api.schema.v1.ModuleTool +import nextflow.cli.Launcher +import nextflow.exception.AbortOperationException +import nextflow.module.ModuleRegistryClient +import org.junit.Rule +import spock.lang.Specification +import spock.lang.TempDir +import test.OutputCapture + +import java.nio.file.Path + +/** + * Tests for ModuleInfo command + * + * @author Jorge Ejarque + */ +class ModuleInfoTest extends Specification { + + @Rule + OutputCapture capture = new OutputCapture() + + @TempDir + Path tempDir + + def 'should display module info in formatted output'() { + given: + def metadata = new ModuleMetadata( + description: 'FastQC quality control analysis', + authors: ['nf-core', 'community'], + keywords: ['quality-control', 'fastqc', 'reads'] + ) + + and: + def release = new ModuleRelease( + version: '1.0.0', + description: 'FastQC module', + metadata: metadata + ) + + and: + def cmd = new ModuleInfo() + cmd.args = ['nf-core/fastqc'] + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.root = tempDir + + and: + def mockModule = Stub(Module) { + getLatest() >> release + } + def mockClient = Mock(ModuleRegistryClient) { + fetchModule(_) >> mockModule + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Module:') + output.contains('nf-core/fastqc') + output.contains('Version:') + output.contains('1.0.0') + output.contains('Description:') + output.contains('FastQC quality control analysis') + output.contains('Authors:') + output.contains('nf-core, community') + output.contains('Keywords:') + output.contains('quality-control, fastqc, reads') + output.contains('Usage Template:') + } + + def 'should display module info with specific version'() { + given: + def metadata = new ModuleMetadata( + description: 'FastQC quality control' + ) + + and: + def release = new ModuleRelease( + version: '0.9.0', + description: 'FastQC module', + metadata: metadata + ) + + and: + def cmd = new ModuleInfo() + cmd.args = ['nf-core/fastqc'] + cmd.version = '0.9.0' + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.root = tempDir + + and: + def mockClient = Mock(ModuleRegistryClient) { + fetchRelease(_, _) >> release + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Version:') + output.contains('0.9.0') + } + + def 'should display module info in JSON format'() { + given: + def metadata = new ModuleMetadata( + description: 'FastQC quality control', + authors: ['nf-core'], + keywords: ['quality-control'] + ) + + and: + def release = new ModuleRelease( + version: '1.0.0', + description: 'FastQC module', + metadata: metadata + ) + + and: + def cmd = new ModuleInfo() + cmd.args = ['nf-core/fastqc'] + cmd.jsonOutput = true + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.root = tempDir + + and: + def mockModule = Stub(Module) { + getLatest() >> release + } + def mockClient = Mock(ModuleRegistryClient) { + fetchModule(_) >> mockModule + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + // Extract JSON part (skip debug/log lines) + def lines = output.readLines() + def jsonStart = lines.findIndexOf { it.trim().startsWith('{') } + def jsonText = lines[jsonStart..-1].join('\n') + def json = new JsonSlurper().parseText(jsonText) + + then: + json.name == 'nf-core/fastqc' + json.fullName == '@nf-core/fastqc' + json.version == '1.0.0' + json.description == 'FastQC quality control' + json.authors == ['nf-core'] + json.keywords == ['quality-control'] + json.usageTemplate != null + } + + def 'should display module info with tools'() { + given: + def tool = new ModuleTool( + name: 'fastqc', + version: '0.12.1', + homepage: URI.create('https://www.bioinformatics.babraham.ac.uk/projects/fastqc/'), + documentation: URI.create('https://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/') + ) + + and: + def metadata = new ModuleMetadata( + description: 'FastQC quality control', + tools: [tool] + ) + + and: + def release = new ModuleRelease( + version: '1.0.0', + metadata: metadata + ) + + and: + def cmd = new ModuleInfo() + cmd.args = ['nf-core/fastqc'] + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.root = tempDir + + and: + def mockModule = Stub(Module) { + getLatest() >> release + } + def mockClient = Mock(ModuleRegistryClient) { + fetchModule(_) >> mockModule + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Tools:') + output.contains('fastqc v0.12.1') + output.contains('Homepage: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/') + } + + def 'should display module info with inputs'() { + given: + def inputItem1 = new ModuleChannelItem( + name: 'reads', + type: 'file', + description: 'Input FASTQ files', + pattern: '*.fastq.gz' + ) + def inputItem2 = new ModuleChannelItem( + name: 'meta', + type: 'map', + description: 'Sample metadata' + ) + + and: + def input = new ModuleChannel( + tuple: true, + items: [inputItem1, inputItem2] + ) + + and: + def metadata = new ModuleMetadata( + description: 'FastQC quality control', + input: [input] + ) + + and: + def release = new ModuleRelease( + version: '1.0.0', + metadata: metadata + ) + + and: + def cmd = new ModuleInfo() + cmd.args = ['nf-core/fastqc'] + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.root = tempDir + + and: + def mockModule = Stub(Module) { + getLatest() >> release + } + def mockClient = Mock(ModuleRegistryClient) { + fetchModule(_) >> mockModule + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Input:') + output.contains('(tuple)') + output.contains('reads (file)') + output.contains('Input FASTQ files') + output.contains('Pattern: *.fastq.gz') + output.contains('meta (map)') + output.contains('Sample metadata') + } + + def 'should display module info with outputs'() { + given: + def outputItem = new ModuleChannelItem( + name: 'html', + type: 'file', + description: 'FastQC HTML report', + pattern: '*_fastqc.html' + ) + + and: + def outputChannel = new ModuleChannel( + tuple: false, + items: [outputItem] + ) + + and: + def metadata = new ModuleMetadata( + description: 'FastQC quality control', + output: ['html': outputChannel] + ) + + and: + def release = new ModuleRelease( + version: '1.0.0', + metadata: metadata + ) + + and: + def cmd = new ModuleInfo() + cmd.args = ['nf-core/fastqc'] + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.root = tempDir + + and: + def mockModule = Stub(Module) { + getLatest() >> release + } + def mockClient = Mock(ModuleRegistryClient) { + fetchModule(_) >> mockModule + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Output:') + output.contains('html') + output.contains('html (file)') + output.contains('FastQC HTML report') + output.contains('Pattern: *_fastqc.html') + } + + def 'should generate usage template with inputs'() { + given: + def inputItem1 = new ModuleChannelItem( + name: 'reads', + type: 'file' + ) + def inputItem2 = new ModuleChannelItem( + name: 'sample-id', + type: 'val' + ) + + and: + def input1 = new ModuleChannel( + tuple: false, + items: [inputItem1] + ) + def input2 = new ModuleChannel( + tuple: false, + items: [inputItem2] + ) + + and: + def metadata = new ModuleMetadata( + description: 'FastQC quality control', + input: [input1, input2] + ) + + and: + def release = new ModuleRelease( + version: '1.0.0', + metadata: metadata + ) + + and: + def cmd = new ModuleInfo() + cmd.args = ['nf-core/fastqc'] + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.root = tempDir + + and: + def mockModule = Stub(Module) { + getLatest() >> release + } + def mockClient = Mock(ModuleRegistryClient) { + fetchModule(_) >> mockModule + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Usage Template:') + output.contains('nextflow module run nf-core/fastqc') + output.contains('--reads ') + output.contains('--sample-id ') + } + + def 'should generate usage template with version'() { + given: + def metadata = new ModuleMetadata( + description: 'FastQC quality control', + input: [] + ) + + and: + def release = new ModuleRelease( + version: '2.0.0', + metadata: metadata + ) + + and: + def cmd = new ModuleInfo() + cmd.args = ['nf-core/fastqc'] + cmd.version = '2.0.0' + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.root = tempDir + + and: + def mockClient = Mock(ModuleRegistryClient) { + fetchRelease(_, _) >> release + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Usage Template:') + output.contains('nextflow module run nf-core/fastqc -version 2.0.0') + } + + def 'should generate usage template with map inputs'() { + given: + def inputItem = new ModuleChannelItem( + name: 'params', + type: 'map' + ) + + and: + def input = new ModuleChannel( + tuple: false, + items: [inputItem] + ) + + and: + def metadata = new ModuleMetadata( + description: 'FastQC quality control', + input: [input] + ) + + and: + def release = new ModuleRelease( + version: '1.0.0', + metadata: metadata + ) + + and: + def cmd = new ModuleInfo() + cmd.args = ['nf-core/fastqc'] + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.root = tempDir + + and: + def mockModule = Stub(Module) { + getLatest() >> release + } + def mockClient = Mock(ModuleRegistryClient) { + fetchModule(_) >> mockModule + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Usage Template:') + output.contains('--params. ') + } + + def 'should fail with no arguments'() { + given: + def cmd = new ModuleInfo() + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.args = [] + cmd.root = tempDir + + when: + cmd.run() + + then: + thrown(AbortOperationException) + } + + def 'should fail with multiple arguments'() { + given: + def cmd = new ModuleInfo() + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.args = ['module1', 'module2'] + cmd.root = tempDir + + when: + cmd.run() + + then: + thrown(AbortOperationException) + } + + def 'should display minimal info when metadata is sparse'() { + given: + def metadata = new ModuleMetadata( + description: null, + authors: null, + keywords: null + ) + + and: + def release = new ModuleRelease( + version: '1.0.0', + description: 'Module description', + metadata: metadata + ) + + and: + def cmd = new ModuleInfo() + cmd.args = ['nf-core/fastqc'] + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.root = tempDir + + and: + def mockModule = Stub(Module) { + getLatest() >> release + } + def mockClient = Mock(ModuleRegistryClient) { + fetchModule(_) >> mockModule + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Module:') + output.contains('nf-core/fastqc') + output.contains('Version:') + output.contains('1.0.0') + output.contains('Description:') + output.contains('Module description') + } + + def 'should display N/A when no description is available'() { + given: + def metadata = new ModuleMetadata( + description: null + ) + + and: + def release = new ModuleRelease( + version: '1.0.0', + description: null, + metadata: metadata + ) + + and: + def cmd = new ModuleInfo() + cmd.args = ['nf-core/fastqc'] + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.root = tempDir + + and: + def mockModule = Stub(Module) { + getLatest() >> release + } + def mockClient = Mock(ModuleRegistryClient) { + fetchModule(_) >> mockModule + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Description: N/A') + } + + def 'should display module with tuple outputs'() { + given: + def outputItem1 = new ModuleChannelItem( + name: 'html', + type: 'file', + description: 'HTML report' + ) + def outputItem2 = new ModuleChannelItem( + name: 'zip', + type: 'file', + description: 'ZIP archive' + ) + + and: + def outputChannel = new ModuleChannel( + tuple: true, + items: [outputItem1, outputItem2] + ) + + and: + def metadata = new ModuleMetadata( + description: 'FastQC quality control', + output: ['qc': outputChannel] + ) + + and: + def release = new ModuleRelease( + version: '1.0.0', + metadata: metadata + ) + + and: + def cmd = new ModuleInfo() + cmd.args = ['nf-core/fastqc'] + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.root = tempDir + + and: + def mockModule = Stub(Module) { + getLatest() >> release + } + def mockClient = Mock(ModuleRegistryClient) { + fetchModule(_) >> mockModule + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + + then: + output.contains('Output:') + output.contains('qc (tuple)') + output.contains('html (file)') + output.contains('HTML report') + output.contains('zip (file)') + output.contains('ZIP archive') + output.contains(')') + } + + def 'should include all tool information in JSON output'() { + given: + def tool = new ModuleTool( + name: 'fastqc', + version: '0.12.1', + homepage: URI.create('https://example.com'), + documentation: URI.create('https://docs.example.com') + ) + + and: + def metadata = new ModuleMetadata( + description: 'FastQC quality control', + tools: [tool] + ) + + and: + def release = new ModuleRelease( + version: '1.0.0', + metadata: metadata + ) + + and: + def cmd = new ModuleInfo() + cmd.args = ['nf-core/fastqc'] + cmd.jsonOutput = true + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.root = tempDir + + and: + def mockModule = Stub(Module) { + getLatest() >> release + } + def mockClient = Mock(ModuleRegistryClient) { + fetchModule(_) >> mockModule + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + // Extract JSON part (skip debug/log lines) + def lines = output.readLines() + def jsonStart = lines.findIndexOf { it.trim().startsWith('{') } + def jsonText = lines[jsonStart..-1].join('\n') + def json = new JsonSlurper().parseText(jsonText) + + then: + json.tools.size() == 1 + json.tools[0].name == 'fastqc' + json.tools[0].version == '0.12.1' + json.tools[0].homepage.scheme == 'https' + json.tools[0].homepage.host == 'example.com' + json.tools[0].documentation.scheme == 'https' + json.tools[0].documentation.host == 'docs.example.com' + } + + def 'should include input/output information in JSON output'() { + given: + def inputItem = new ModuleChannelItem( + name: 'reads', + type: 'file', + description: 'Input reads', + pattern: '*.fastq.gz' + ) + def inputChannel = new ModuleChannel( + tuple: true, + items: [inputItem] + ) + + and: + def outputItem = new ModuleChannelItem( + name: 'html', + type: 'file', + description: 'HTML report', + pattern: '*.html' + ) + def outputChannel = new ModuleChannel( + tuple: false, + items: [outputItem] + ) + + and: + def metadata = new ModuleMetadata( + description: 'FastQC quality control', + input: [inputChannel], + output: ['html': outputChannel] + ) + + and: + def release = new ModuleRelease( + version: '1.0.0', + metadata: metadata + ) + + and: + def cmd = new ModuleInfo() + cmd.args = ['nf-core/fastqc'] + cmd.jsonOutput = true + cmd.launcher = Mock(Launcher) { + getOptions() >> null + } + cmd.root = tempDir + + and: + def mockModule = Stub(Module) { + getLatest() >> release + } + def mockClient = Mock(ModuleRegistryClient) { + fetchModule(_) >> mockModule + } + cmd.client = mockClient + + when: + cmd.run() + def output = capture.toString() + // Extract JSON part (skip debug/log lines) + def lines = output.readLines() + def jsonStart = lines.findIndexOf { it.trim().startsWith('{') } + def jsonText = lines[jsonStart..-1].join('\n') + def json = new JsonSlurper().parseText(jsonText) + + then: + json.input.size() == 1 + json.input[0].tuple == true + json.input[0].items[0].name == 'reads' + json.input[0].items[0].type == 'file' + json.input[0].items[0].description == 'Input reads' + json.input[0].items[0].pattern == '*.fastq.gz' + + and: + json.output.html.tuple == false + json.output.html.items[0].name == 'html' + json.output.html.items[0].type == 'file' + json.output.html.items[0].description == 'HTML report' + json.output.html.items[0].pattern == '*.html' + } +} \ No newline at end of file diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleRegistryClientTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleRegistryClientTest.groovy index ac4e97c1f4..57311f809f 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleRegistryClientTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleRegistryClientTest.groovy @@ -44,11 +44,13 @@ class ModuleRegistryClientTest extends Specification { Path tempDir WireMockServer wireMock - + String url + static final String MODULES_API_PATH = "/api/v1/modules" def setup() { wireMock = new WireMockServer(wireMockConfig().dynamicPort()) wireMock.start() WireMock.configureFor("localhost", wireMock.port()) + url = "http://localhost:${wireMock.port()}/api" } def cleanup() { @@ -67,14 +69,14 @@ class ModuleRegistryClientTest extends Specification { ] // Note: nf-core/fastqc is URL-encoded as nf-core%2Ffastqc - stubFor(get(urlEqualTo('/api/modules/nf-core%2Ffastqc')) + stubFor(get(urlEqualTo(MODULES_API_PATH + '/nf-core%2Ffastqc')) .willReturn(aResponse() .withStatus(200) .withHeader('Content-Type', 'application/json') .withBody(JsonOutput.toJson(moduleResponse)))) and: - def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def config = new RegistryConfig([url: url]) def client = new ModuleRegistryClient(config) when: @@ -87,7 +89,7 @@ class ModuleRegistryClientTest extends Specification { result.latest.version == '1.1.0' and: 'verify request was made' - verify(getRequestedFor(urlEqualTo('/api/modules/nf-core%2Ffastqc'))) + verify(getRequestedFor(urlEqualTo(MODULES_API_PATH + '/nf-core%2Ffastqc'))) } def 'should search modules in registry'() { @@ -115,7 +117,7 @@ class ModuleRegistryClientTest extends Specification { ] ] - stubFor(get(urlPathEqualTo('/api/modules')) + stubFor(get(urlPathEqualTo(MODULES_API_PATH)) .withQueryParam('query', equalTo('fastqc')) .withQueryParam('limit', equalTo('10')) .willReturn(aResponse() @@ -124,7 +126,7 @@ class ModuleRegistryClientTest extends Specification { .withBody(JsonOutput.toJson(searchResponse)))) and: - def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def config = new RegistryConfig([url: url]) def client = new ModuleRegistryClient(config) when: @@ -140,7 +142,7 @@ class ModuleRegistryClientTest extends Specification { Math.abs(result.results[0].relevanceScore - 0.95) < 0.001 and: 'verify query parameters' - verify(getRequestedFor(urlPathEqualTo('/api/modules')) + verify(getRequestedFor(urlPathEqualTo(MODULES_API_PATH)) .withQueryParam('query', equalTo('fastqc')) .withQueryParam('limit', equalTo('10'))) } @@ -151,7 +153,7 @@ class ModuleRegistryClientTest extends Specification { def expectedChecksum = "${computeSha256(modulePackage)}" // Note: URL-encoded path - stubFor(get(urlEqualTo('/api/modules/nf-core%2Ffastqc/1.0.0/download')) + stubFor(get(urlEqualTo(MODULES_API_PATH + '/nf-core%2Ffastqc/1.0.0/download')) .willReturn(aResponse() .withStatus(200) .withHeader('Content-Type', 'application/gzip') @@ -159,7 +161,7 @@ class ModuleRegistryClientTest extends Specification { .withBody(modulePackage))) and: - def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def config = new RegistryConfig([url: url]) def client = new ModuleRegistryClient(config) def destFile = tempDir.resolve('module.tgz') @@ -172,19 +174,19 @@ class ModuleRegistryClientTest extends Specification { Files.size(destFile) == modulePackage.length and: - verify(getRequestedFor(urlEqualTo('/api/modules/nf-core%2Ffastqc/1.0.0/download'))) + verify(getRequestedFor(urlEqualTo(MODULES_API_PATH +'/nf-core%2Ffastqc/1.0.0/download'))) } def 'should successfully fetch module without authentication'() { given: - stubFor(get(urlEqualTo('/api/modules/nf-core%2Ffastqc')) + stubFor(get(urlEqualTo(MODULES_API_PATH + '/nf-core%2Ffastqc')) .willReturn(aResponse() .withStatus(200) .withHeader('Content-Type', 'application/json') .withBody(JsonOutput.toJson([name: 'nf-core/fastqc', latest: [version: '1.0.0']])))) and: - def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def config = new RegistryConfig([url: url]) def client = new ModuleRegistryClient(config) when: @@ -195,18 +197,18 @@ class ModuleRegistryClientTest extends Specification { result.name == 'nf-core/fastqc' and: 'verify request was made' - verify(getRequestedFor(urlEqualTo('/api/modules/nf-core%2Ffastqc'))) + verify(getRequestedFor(urlEqualTo(MODULES_API_PATH + '/nf-core%2Ffastqc'))) } def 'should handle 404 not found error'() { given: - stubFor(get(urlEqualTo('/api/modules/nf-core%2Fnonexistent')) + stubFor(get(urlEqualTo(MODULES_API_PATH + '/nf-core%2Fnonexistent')) .willReturn(aResponse() .withStatus(404) .withBody('Module not found'))) and: - def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def config = new RegistryConfig([url: url]) def client = new ModuleRegistryClient(config) when: @@ -217,18 +219,18 @@ class ModuleRegistryClientTest extends Specification { ex.message.contains('Unable to fetch module') || ex.message.contains('Module not found') and: - verify(getRequestedFor(urlEqualTo('/api/modules/nf-core%2Fnonexistent'))) + verify(getRequestedFor(urlEqualTo(MODULES_API_PATH + '/nf-core%2Fnonexistent'))) } def 'should handle 500 server error'() { given: - stubFor(get(urlEqualTo('/api/modules/nf-core%2Ffastqc')) + stubFor(get(urlEqualTo(MODULES_API_PATH + '/nf-core%2Ffastqc')) .willReturn(aResponse() .withStatus(500) .withBody('Internal Server Error'))) and: - def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def config = new RegistryConfig([url: url]) def client = new ModuleRegistryClient(config) when: @@ -240,27 +242,27 @@ class ModuleRegistryClientTest extends Specification { def 'should send user agent header'() { given: - stubFor(get(urlEqualTo('/api/modules/nf-core%2Ffastqc')) + stubFor(get(urlEqualTo(MODULES_API_PATH + '/nf-core%2Ffastqc')) .willReturn(aResponse() .withStatus(200) .withHeader('Content-Type', 'application/json') .withBody(JsonOutput.toJson([name: 'nf-core/fastqc', latest: [version: '1.0.0'], releases: []])))) and: - def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def config = new RegistryConfig([url: url]) def client = new ModuleRegistryClient(config) when: client.fetchModule('nf-core/fastqc') then: - verify(getRequestedFor(urlEqualTo('/api/modules/nf-core%2Ffastqc')) + verify(getRequestedFor(urlEqualTo(MODULES_API_PATH + '/nf-core%2Ffastqc')) .withHeader('User-Agent', matching('.*'))) } def 'should handle empty search results'() { given: - stubFor(get(urlPathEqualTo('/api/modules')) + stubFor(get(urlPathEqualTo(MODULES_API_PATH)) .withQueryParam('query', equalTo('nonexistent')) .willReturn(aResponse() .withStatus(200) @@ -272,7 +274,7 @@ class ModuleRegistryClientTest extends Specification { ])))) and: - def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def config = new RegistryConfig([url: url]) def client = new ModuleRegistryClient(config) when: @@ -286,7 +288,7 @@ class ModuleRegistryClientTest extends Specification { def 'should respect custom search limit'() { given: - stubFor(get(urlPathEqualTo('/api/modules')) + stubFor(get(urlPathEqualTo(MODULES_API_PATH)) .willReturn(aResponse() .withStatus(200) .withHeader('Content-Type', 'application/json') @@ -297,27 +299,27 @@ class ModuleRegistryClientTest extends Specification { ])))) and: - def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def config = new RegistryConfig([url: url]) def client = new ModuleRegistryClient(config) when: client.search('test', 25) then: - verify(getRequestedFor(urlPathEqualTo('/api/modules')) + verify(getRequestedFor(urlPathEqualTo(MODULES_API_PATH )) .withQueryParam('limit', equalTo('25'))) } def 'should handle malformed JSON response'() { given: - stubFor(get(urlEqualTo('/api/modules/nf-core%2Ffastqc')) + stubFor(get(urlEqualTo(MODULES_API_PATH + '/nf-core%2Ffastqc')) .willReturn(aResponse() .withStatus(200) .withHeader('Content-Type', 'application/json') .withBody('not valid json {]'))) and: - def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def config = new RegistryConfig([url: url]) def client = new ModuleRegistryClient(config) when: @@ -332,7 +334,7 @@ class ModuleRegistryClientTest extends Specification { def modulePackage = createTestModulePackage() def checksum = "sha256:${computeSha256(modulePackage)}" - stubFor(get(urlEqualTo('/api/modules/nf-core%2Ffastqc/1.0.0/download')) + stubFor(get(urlEqualTo(MODULES_API_PATH + '/nf-core%2Ffastqc/1.0.0/download')) .willReturn(aResponse() .withStatus(200) .withHeader('Content-Type', 'application/gzip') @@ -341,7 +343,7 @@ class ModuleRegistryClientTest extends Specification { .withBody(modulePackage))) and: - def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def config = new RegistryConfig([url: url]) def client = new ModuleRegistryClient(config) def destFile = tempDir.resolve('module.tgz') @@ -353,7 +355,7 @@ class ModuleRegistryClientTest extends Specification { Files.exists(destFile) and: 'verify checksum header was present' - verify(getRequestedFor(urlEqualTo('/api/modules/nf-core%2Ffastqc/1.0.0/download'))) + verify(getRequestedFor(urlEqualTo(MODULES_API_PATH + '/nf-core%2Ffastqc/1.0.0/download'))) } def 'should handle network errors gracefully'() { @@ -371,7 +373,7 @@ class ModuleRegistryClientTest extends Specification { def 'should require authentication for publish'() { given: - def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def config = new RegistryConfig([url: url]) def client = new ModuleRegistryClient(config) def publishRequest = [name: 'nf-core/mymodule', version: '1.0.0'] @@ -385,7 +387,7 @@ class ModuleRegistryClientTest extends Specification { def 'should handle publish failure with no auth token'() { given: - def config = new RegistryConfig([url: "http://localhost:${wireMock.port()}"]) + def config = new RegistryConfig([url: url]) def client = new ModuleRegistryClient(config) def publishRequest = [name: 'nf-core/mymodule', version: '1.0.0'] From 36e9085d37a41bdb3814283b349839022fad7449 Mon Sep 17 00:00:00 2001 From: jorgee Date: Thu, 19 Feb 2026 12:06:10 +0100 Subject: [PATCH 08/23] update default registry to include /api Signed-off-by: jorgee --- .../src/main/groovy/nextflow/config/RegistryConfig.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/config/RegistryConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/config/RegistryConfig.groovy index 24ee57063f..a7c582537e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/RegistryConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/RegistryConfig.groovy @@ -35,7 +35,7 @@ import nextflow.script.dsl.Description @CompileStatic class RegistryConfig implements ConfigScope { - final static public String DEFAULT_REGISTRY_URL = 'https://registry.nextflow.io' + final static public String DEFAULT_REGISTRY_URL = 'https://registry.nextflow.io/api' @ConfigOption @Description("Primary registry URL") From fc327c4001c149d58cdf1bb6ccebbdb8d3497b5a Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 19 Feb 2026 16:40:21 +0100 Subject: [PATCH 09/23] Fix compilation issue [ci skip] Signed-off-by: Paolo Di Tommaso --- .../src/main/nextflow/plugin/HttpPluginRepository.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/nf-commons/src/main/nextflow/plugin/HttpPluginRepository.groovy b/modules/nf-commons/src/main/nextflow/plugin/HttpPluginRepository.groovy index b52dd323f8..48a81218ea 100644 --- a/modules/nf-commons/src/main/nextflow/plugin/HttpPluginRepository.groovy +++ b/modules/nf-commons/src/main/nextflow/plugin/HttpPluginRepository.groovy @@ -9,7 +9,7 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.seqera.http.HxClient import io.seqera.npr.api.schema.v1.ListDependenciesResponse -import io.seqera.npr.api.schema.v1.Plugin +import io.seqera.npr.api.schema.v1.PluginDependency import nextflow.BuildInfo import nextflow.util.RetryConfig import org.pf4j.PluginRuntimeException @@ -156,7 +156,7 @@ class HttpPluginRepository implements PrefetchUpdateRepository { throw new PluginRuntimeException("Failed to download plugin metadata: Failed to parse response body") } final result = new HashMap() - for( Plugin plugin : decoded.plugins ) { + for( PluginDependency plugin : decoded.plugins ) { if( plugin.releases ) { final pluginInfo = mapToPluginInfo(plugin) result.put(plugin.id, pluginInfo) @@ -179,7 +179,7 @@ class HttpPluginRepository implements PrefetchUpdateRepository { * @param plugin The Plugin object from the repository API response * @return A PluginInfo object compatible with pf4j's update repository interface */ - static protected PluginInfo mapToPluginInfo(Plugin plugin) { + static protected PluginInfo mapToPluginInfo(PluginDependency plugin) { assert plugin.releases, "Plugin releases cannot be empty" final pluginInfo = new PluginInfo() From c3b02a89c3143873444d47ecf458dd60c826b93a Mon Sep 17 00:00:00 2001 From: jorgee Date: Wed, 25 Feb 2026 16:13:50 +0100 Subject: [PATCH 10/23] add review comments and fix compilation and tests Signed-off-by: jorgee --- modules/nextflow/build.gradle | 2 +- .../nextflow/cli/module/ModuleInfo.groovy | 67 +++- .../nextflow/cli/module/ModuleRun.groovy | 23 +- .../nextflow/config/RegistryConfig.groovy | 132 ------- .../module/ModuleRegistryClient.groovy | 16 +- .../nextflow/cli/module/ModuleInfoTest.groovy | 7 +- .../nextflow/config/RegistryConfigTest.groovy | 356 ------------------ .../groovy/test/TestHelper.groovy | 12 + modules/nf-commons/build.gradle | 2 +- .../nextflow/config/RegistryConfig.groovy | 91 +++++ .../plugin/HttpPluginRepository.groovy | 2 +- .../nextflow/config/RegistryConfigTest.groovy | 107 ++++++ 12 files changed, 278 insertions(+), 539 deletions(-) delete mode 100644 modules/nextflow/src/main/groovy/nextflow/config/RegistryConfig.groovy delete mode 100644 modules/nextflow/src/test/groovy/nextflow/config/RegistryConfigTest.groovy create mode 100644 modules/nf-commons/src/main/nextflow/config/RegistryConfig.groovy create mode 100644 modules/nf-commons/src/test/nextflow/config/RegistryConfigTest.groovy diff --git a/modules/nextflow/build.gradle b/modules/nextflow/build.gradle index b0596807cc..859ae24cad 100644 --- a/modules/nextflow/build.gradle +++ b/modules/nextflow/build.gradle @@ -55,7 +55,7 @@ dependencies { api 'io.seqera:lib-trace:0.1.0' api 'com.fasterxml.woodstox:woodstox-core:7.1.1' api 'org.apache.commons:commons-compress:1.27.1' // For tar.gz extraction - api 'io.seqera:npr-api:0.20.1' + api 'io.seqera:npr-api:0.21.4-SNAPSHOT' testImplementation 'org.subethamail:subethasmtp:3.1.7' testImplementation (project(':nf-lineage')) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInfo.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInfo.groovy index ebf4a03897..a0a49827f1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInfo.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInfo.groovy @@ -141,7 +141,7 @@ class ModuleInfo extends CmdBase { println "" println "Usage Template:" println "-" * 80 - println generateUsageTemplate(reference, metadata) + println generateUsageTemplate(reference, metadata).join(" \\\n ") println "" } @@ -195,28 +195,59 @@ class ModuleInfo extends CmdBase { } } - private String generateUsageTemplate(ModuleReference reference, ModuleMetadata metadata) { - def template = new StringBuilder() - template.append("nextflow module run ${reference.nameWithoutPrefix}") + private List generateUsageTemplate(ModuleReference reference, ModuleMetadata metadata) { + def template = new ArrayList() + template.add("nextflow module run ${reference.nameWithoutPrefix}".toString()) if( version ) - template.append(" -version $version") + template.add(" -version $version".toString()) - // Use metadata from registry if available, otherwise use spec from meta.yml def inputs = metadata?.input ?: [] + inputs.each { input -> + input.items.each { ModuleChannelItem item -> + template.add(reference.scope == 'nf-core' + ? inferNfCoreParam(item.name, item.type) + : inferNormalParam(item.name, item.type)) - if( inputs ) { - inputs.each { input -> - input.items.each { ModuleChannelItem item -> - def placeholder = item.name.toUpperCase().replaceAll(/[^A-Z0-9_]/, '_') - if( item.type.equalsIgnoreCase("map") ) { - template.append(" --${item.name}. <${placeholder}_KEY>") - } else { - template.append(" --${item.name} <${placeholder}>") - } - } } } - return template.toString() + if( reference.scope == 'nf-core' ) { + template.add('--outdir ') + } + return template + } + + private static String inferNfCoreParam(String paramName, String type) { + if( type?.equalsIgnoreCase("map") && paramName.equalsIgnoreCase("meta") ) { + return "--${paramName}.id " + } + if( type?.equalsIgnoreCase("file") || type?.equalsIgnoreCase("path") ) { + return "--${paramName} ${inferBioFilePlaceholder(paramName)}" + } + return inferNormalParam(paramName, type) + } + + private static String inferBioFilePlaceholder(String paramName) { + final String lower = paramName.toLowerCase() + if( lower.contains("fasta") ) return "" + if( lower.contains("bam") ) return "" + if( lower.contains("fastq") || lower.equals("reads") ) return "" + if( lower.contains("vcf") ) return "" + if( lower.contains("ref") ) return "" + if( lower.contains("bed") ) return "" + if( lower.contains("gff") || lower.contains("gtf") ) return "" + + return "<${paramName.toUpperCase().replaceAll(/[^A-Z0-9]/, '_')}_PATH>" + } + + private static String inferNormalParam(String paramName, String type) { + final paramPlaceholder = paramName.toUpperCase().replaceAll(/[^A-Z0-9]/, '_') + if( type?.equalsIgnoreCase("map") ) { + return "--${paramName}. <${paramPlaceholder}_KEY_VALUE>" + } + if( type?.equalsIgnoreCase("file") || type?.equalsIgnoreCase("path") ) { + return "--${paramName} <${paramPlaceholder}_PATH>" + } + return "--${paramName} <${paramPlaceholder}>" } private void printJsonInfo(ModuleReference reference, ModuleRelease release) { @@ -276,7 +307,7 @@ class ModuleInfo extends CmdBase { } } - info.usageTemplate = generateUsageTemplate(reference, metadata) + info.usageTemplate = generateUsageTemplate(reference, metadata).join(" ") println JsonOutput.prettyPrint(JsonOutput.toJson(info)) } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy index 3a547295cb..8b59dbf876 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy @@ -19,20 +19,17 @@ package nextflow.cli.module import com.beust.jcommander.Parameter import com.beust.jcommander.Parameters import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j import nextflow.cli.CmdRun import nextflow.config.ConfigBuilder import nextflow.config.ModulesConfig import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException -import nextflow.file.FileHelper import nextflow.module.ModuleReference import nextflow.module.ModuleRegistryClient import nextflow.module.ModuleResolver import nextflow.pipeline.PipelineSpec import nextflow.util.TestOnly -import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths @@ -41,7 +38,6 @@ import java.nio.file.Paths * * @author Jorge Ejarque */ -@Slf4j @CompileStatic @Parameters(commandDescription = "Run a module directly from the registry") class ModuleRun extends CmdRun { @@ -90,20 +86,11 @@ class ModuleRun extends CmdRun { def modulesConfig = new ModulesConfig(specFile.getModules()) def resolver = new ModuleResolver(baseDir, client ?: new ModuleRegistryClient(registryConfig), modulesConfig) - try{ - Path moduleFile = resolver.installModule(reference, version) - if( moduleFile ) { - println "Executing module..." - args[0] = moduleFile.toAbsolutePath().toString() - super.run() - } - } - catch (AbortOperationException e) { - throw e - } - catch (Exception e) { - log.error("Failed to run module", e) - throw new AbortOperationException("Module run failed: ${e.message}", e) + Path moduleFile = resolver.installModule(reference, version) + if( moduleFile ) { + println "Executing module..." + args[0] = moduleFile.toAbsolutePath().toString() + super.run() } } } diff --git a/modules/nextflow/src/main/groovy/nextflow/config/RegistryConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/config/RegistryConfig.groovy deleted file mode 100644 index a7c582537e..0000000000 --- a/modules/nextflow/src/main/groovy/nextflow/config/RegistryConfig.groovy +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2013-2026, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package nextflow.config - -import groovy.transform.CompileStatic -import nextflow.config.spec.ConfigOption -import nextflow.config.spec.ConfigScope -import nextflow.config.spec.ScopeName -import nextflow.script.dsl.Description - -/** - * Configuration scope for module registry settings - * - * @author Jorge Ejarque - */ -@ScopeName("registry") -@Description(""" - The `registry` scope provides configuration for the Nextflow module registry. - This includes registry URL(s) and authentication settings. -""") -@CompileStatic -class RegistryConfig implements ConfigScope { - - final static public String DEFAULT_REGISTRY_URL = 'https://registry.nextflow.io/api' - - @ConfigOption - @Description("Primary registry URL") - final private String url - - @ConfigOption - @Description("List of registry URLs to try in order") - final private List urls - - @ConfigOption - @Description("Authentication configuration per registry (registry URL -> token)") - final private Map auth - - /* required by extension point -- do not remove */ - RegistryConfig() { - this.url = DEFAULT_REGISTRY_URL - this.urls = [] - this.auth = [:] - } - - RegistryConfig(Map opts) { - this.url = opts.url ? opts.url as String : DEFAULT_REGISTRY_URL - this.urls = opts.urls ? opts.urls as List : [] - this.auth = opts.auth ? opts.auth as Map : [:] - } - - /** - * Get the primary registry URL - * - * @return The registry URL - */ - String getUrl() { - return url - } - - /** - * Get all registry URLs (primary + fallbacks) - * - * @return List of registry URLs - */ - List getAllUrls() { - List result = [] - if (urls && !urls.isEmpty()) { - result.addAll(urls) - } else if (url) { - result.add(url) - } else { - result.add(DEFAULT_REGISTRY_URL) - } - return result - } - - /** - * Get authentication token for a registry - * - * @param registryUrl The registry URL - * @return The authentication token, or null if not configured - */ - String getAuthToken(String registryUrl) { - return auth?.get(registryUrl) - } - - /** - * Get authentication token from environment variable or config - * - * @param registryUrl The registry URL - * @return The authentication token, or null if not found - */ - String getAuthTokenResolved(String registryUrl) { - // First check config - def token = getAuthToken(registryUrl) - if (token) { - // Resolve environment variable references like ${NXF_REGISTRY_TOKEN} - if (token.startsWith('${') && token.endsWith('}')) { - def envVar = token.substring(2, token.length() - 1) - token = System.getenv(envVar) - } - return token - } - - // Fallback to NXF_REGISTRY_TOKEN environment variable - return System.getenv('NXF_REGISTRY_TOKEN') - } - - /** - * Check if authentication is configured for a registry - * - * @param registryUrl The registry URL - * @return true if authentication is available - */ - boolean hasAuth(String registryUrl) { - return getAuthTokenResolved(registryUrl) != null - } -} diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy index 7c54d8c73a..4d5e3cf6ed 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy @@ -101,7 +101,7 @@ class ModuleRegistryClient { // Add authentication if available log.debug "Getting auth from: ${registryUrl}" - def token = config.getAuthTokenResolved(registryUrl) + def token = config.getApiKey() if (token) { requestBuilder.header("Authorization", "Bearer ${token}") } @@ -177,7 +177,7 @@ class ModuleRegistryClient { .uri(uri) .GET() - def token = config.getAuthTokenResolved(registryUrl) + def token = config.getApiKey() if (token) { requestBuilder.header("Authorization", "Bearer ${token}") } @@ -252,7 +252,7 @@ class ModuleRegistryClient { .uri(uri) .GET() - def token = config.getAuthTokenResolved(registryUrl) + def token = config.getApiKey() if (token) { requestBuilder.header("Authorization", "Bearer ${token}") } @@ -368,7 +368,7 @@ class ModuleRegistryClient { .uri(uri) .GET() - def token = config.getAuthTokenResolved(registryUrl) + def token = config.getApiKey() if (token) { requestBuilder.header("Authorization", "Bearer ${token}") } @@ -410,16 +410,14 @@ class ModuleRegistryClient { */ PublishModuleResponse publishModule(String name, def request, String registry = null) { final registryUrl = registry ?: config.url - final authToken = config.getAuthTokenResolved(registryUrl) + final authToken = config.apiKey if (!authToken) { throw new AbortOperationException( "Authentication required to publish modules.\n" + - "Please set NXF_REGISTRY_TOKEN environment variable or configure registry.auth in nextflow.config:\n\n" + + "Please set 'NXF_REGISTRY_TOKEN' environment variable or configure 'registry.apiKey' in nextflow.config:\n\n" + " registry {\n" + - " auth {\n" + - " '${registryUrl}' = '\${NXF_REGISTRY_TOKEN}'\n" + - " }\n" + + " apiKey = '\${NXF_REGISTRY_TOKEN}'\n" + " }\n" ) } diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInfoTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInfoTest.groovy index f0d721e62b..1c409a6efc 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInfoTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInfoTest.groovy @@ -409,7 +409,7 @@ class ModuleInfoTest extends Specification { then: output.contains('Usage Template:') output.contains('nextflow module run nf-core/fastqc') - output.contains('--reads ') + output.contains('--reads ') output.contains('--sample-id ') } @@ -447,7 +447,8 @@ class ModuleInfoTest extends Specification { then: output.contains('Usage Template:') - output.contains('nextflow module run nf-core/fastqc -version 2.0.0') + output.contains('nextflow module run nf-core/fastqc') + output.contains('-version 2.0.0') } def 'should generate usage template with map inputs'() { @@ -498,7 +499,7 @@ class ModuleInfoTest extends Specification { then: output.contains('Usage Template:') - output.contains('--params. ') + output.contains('--params. ') } def 'should fail with no arguments'() { diff --git a/modules/nextflow/src/test/groovy/nextflow/config/RegistryConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/config/RegistryConfigTest.groovy deleted file mode 100644 index 02d18f6a6c..0000000000 --- a/modules/nextflow/src/test/groovy/nextflow/config/RegistryConfigTest.groovy +++ /dev/null @@ -1,356 +0,0 @@ -/* - * Copyright 2013-2026, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package nextflow.config - -import spock.lang.Specification - -/** - * Tests for RegistryConfig - * - * @author Jorge Ejarque - */ -class RegistryConfigTest extends Specification { - - def cleanup() { - // Clean up any environment variables set during tests - System.clearProperty('NXF_REGISTRY_TOKEN') - } - - def 'should create config with default values'() { - when: - def config = new RegistryConfig() - - then: - config.url == RegistryConfig.DEFAULT_REGISTRY_URL - config.allUrls == [RegistryConfig.DEFAULT_REGISTRY_URL] - config.getAuthToken(RegistryConfig.DEFAULT_REGISTRY_URL) == null - } - - def 'should initialize with custom URL'() { - given: - def opts = [url: 'https://custom.registry.com'] - - when: - def config = new RegistryConfig(opts) - - then: - config.url == 'https://custom.registry.com' - config.allUrls == ['https://custom.registry.com'] - } - - def 'should initialize with multiple URLs'() { - given: - def opts = [ - urls: [ - 'https://primary.registry.com', - 'https://fallback.registry.com' - ] - ] - - when: - def config = new RegistryConfig(opts) - - then: - config.allUrls == [ - 'https://primary.registry.com', - 'https://fallback.registry.com' - ] - } - - def 'should prefer urls list over single url'() { - given: - def opts = [ - url: 'https://single.registry.com', - urls: [ - 'https://primary.registry.com', - 'https://fallback.registry.com' - ] - ] - - when: - def config = new RegistryConfig(opts) - - then: - config.allUrls == [ - 'https://primary.registry.com', - 'https://fallback.registry.com' - ] - } - - def 'should fall back to default URL when no URL provided'() { - given: - def opts = [:] - - when: - def config = new RegistryConfig(opts) - - then: - config.url == RegistryConfig.DEFAULT_REGISTRY_URL - config.allUrls == [RegistryConfig.DEFAULT_REGISTRY_URL] - } - - def 'should initialize with authentication'() { - given: - def opts = [ - url: 'https://registry.com', - auth: [ - 'https://registry.com': 'token123' - ] - ] - - when: - def config = new RegistryConfig(opts) - - then: - config.getAuthToken('https://registry.com') == 'token123' - } - - def 'should return null for unconfigured auth'() { - given: - def config = new RegistryConfig([url: 'https://registry.com']) - - when: - def token = config.getAuthToken('https://registry.com') - - then: - token == null - } - - def 'should detect environment variable reference format'() { - given: - def opts = [ - url: 'https://registry.com', - auth: [ - 'https://registry.com': '${TEST_TOKEN_VAR}' - ] - ] - def config = new RegistryConfig(opts) - - when: - def token = config.getAuthToken('https://registry.com') - - then: - token == '${TEST_TOKEN_VAR}' - token.startsWith('${') && token.endsWith('}') - } - - def 'should return literal token when not environment variable reference'() { - given: - def opts = [ - url: 'https://registry.com', - auth: [ - 'https://registry.com': 'literal-token' - ] - ] - def config = new RegistryConfig(opts) - - when: - def token = config.getAuthTokenResolved('https://registry.com') - - then: - token == 'literal-token' - } - - def 'should prefer configured auth over environment variable'() { - given: - def opts = [ - url: 'https://registry.com', - auth: [ - 'https://registry.com': 'config-token' - ] - ] - def config = new RegistryConfig(opts) - - when: - def token = config.getAuthTokenResolved('https://registry.com') - - then: - // If NXF_REGISTRY_TOKEN env var exists, configured token takes precedence - token == 'config-token' - } - - def 'should check if auth is configured'() { - given: - def config = new RegistryConfig([ - url: 'https://registry.com', - auth: [ - 'https://registry.com': 'token123' - ] - ]) - - expect: - config.hasAuth('https://registry.com') - !config.hasAuth('https://other-registry.com') - } - - def 'should support multiple registry auths'() { - given: - def opts = [ - urls: [ - 'https://primary.registry.com', - 'https://fallback.registry.com' - ], - auth: [ - 'https://primary.registry.com': 'primary-token', - 'https://fallback.registry.com': 'fallback-token' - ] - ] - def config = new RegistryConfig(opts) - - expect: - config.getAuthToken('https://primary.registry.com') == 'primary-token' - config.getAuthToken('https://fallback.registry.com') == 'fallback-token' - config.hasAuth('https://primary.registry.com') - config.hasAuth('https://fallback.registry.com') - } - - def 'should handle null auth map'() { - given: - def config = new RegistryConfig([url: 'https://registry.com', auth: null]) - - expect: - config.getAuthToken('https://registry.com') == null - !config.hasAuth('https://registry.com') - } - - def 'should handle empty auth map'() { - given: - def config = new RegistryConfig([url: 'https://registry.com', auth: [:]]) - - expect: - config.getAuthToken('https://registry.com') == null - !config.hasAuth('https://registry.com') - } - - def 'should return null when environment variable is not set'() { - given: - def opts = [ - url: 'https://registry.com', - auth: [ - 'https://registry.com': '${NONEXISTENT_VAR}' - ] - ] - def config = new RegistryConfig(opts) - - when: - def token = config.getAuthTokenResolved('https://registry.com') - - then: - token == null - } - - def 'should handle complex URL patterns in auth keys'() { - given: - def opts = [ - auth: [ - 'https://registry.com': 'token1', - 'https://registry.com:8080': 'token2', - 'http://localhost:3000': 'token3' - ] - ] - def config = new RegistryConfig(opts) - - expect: - config.getAuthToken('https://registry.com') == 'token1' - config.getAuthToken('https://registry.com:8080') == 'token2' - config.getAuthToken('http://localhost:3000') == 'token3' - } - - def 'should handle empty urls list gracefully'() { - given: - def opts = [ - url: 'https://registry.com', - urls: [] - ] - - when: - def config = new RegistryConfig(opts) - - then: - config.allUrls == ['https://registry.com'] - } - - def 'should use default URL when both url and urls are empty'() { - given: - def opts = [ - url: null, - urls: [] - ] - - when: - def config = new RegistryConfig(opts) - - then: - config.allUrls == [RegistryConfig.DEFAULT_REGISTRY_URL] - } - - def 'should correctly identify auth when configured'() { - given: - def config = new RegistryConfig([ - url: 'https://registry.com', - auth: ['https://registry.com': 'token'] - ]) - - expect: - config.hasAuth('https://registry.com') - } - - def 'should handle registry URL without trailing slash'() { - given: - def opts = [ - url: 'https://registry.com', - auth: ['https://registry.com': 'token'] - ] - def config = new RegistryConfig(opts) - - expect: - config.getAuthToken('https://registry.com') == 'token' - } - - def 'should handle registry URL with trailing slash'() { - given: - def opts = [ - url: 'https://registry.com/', - auth: ['https://registry.com/': 'token'] - ] - def config = new RegistryConfig(opts) - - expect: - config.getAuthToken('https://registry.com/') == 'token' - } - - def 'should preserve order of URLs in list'() { - given: - def opts = [ - urls: [ - 'https://first.registry.com', - 'https://second.registry.com', - 'https://third.registry.com' - ] - ] - - when: - def config = new RegistryConfig(opts) - - then: - config.allUrls == [ - 'https://first.registry.com', - 'https://second.registry.com', - 'https://third.registry.com' - ] - } -} diff --git a/modules/nextflow/src/testFixtures/groovy/test/TestHelper.groovy b/modules/nextflow/src/testFixtures/groovy/test/TestHelper.groovy index 24433a8368..5c9d2fb1ec 100644 --- a/modules/nextflow/src/testFixtures/groovy/test/TestHelper.groovy +++ b/modules/nextflow/src/testFixtures/groovy/test/TestHelper.groovy @@ -15,6 +15,11 @@ */ package test + +import com.google.common.jimfs.JimfsPath +import nextflow.util.KryoHelper +import nextflow.util.PathSerializer + import java.nio.file.Files import java.nio.file.Path import java.util.zip.GZIPInputStream @@ -53,6 +58,13 @@ class TestHelper { static private fs = Jimfs.newFileSystem(Configuration.unix()); + static { + // Some tests failed after a Guava update (Guava 33.4+) when using Jimfs. + // Adding a default serializer for JimfsPaths to prevent Kryo's FieldSerializer from recursing into internal + // fields that may contain non-serializable objects such as lambdas + KryoHelper.kryo().addDefaultSerializer(JimfsPath.class, PathSerializer) + } + static Path createInMemTempDir() { Path tmp = fs.getPath("/tmp"); tmp.mkdir() diff --git a/modules/nf-commons/build.gradle b/modules/nf-commons/build.gradle index 308b2ac1ee..35650cea08 100644 --- a/modules/nf-commons/build.gradle +++ b/modules/nf-commons/build.gradle @@ -38,7 +38,7 @@ dependencies { api 'io.seqera:lib-retry:2.0.0' // patch gson dependency required by pf4j api 'com.google.code.gson:gson:2.13.1' - api 'io.seqera:npr-api:0.20.1' + api 'io.seqera:npr-api:0.21.4-SNAPSHOT' /* testImplementation inherited from top gradle build file */ testImplementation(testFixtures(project(":nextflow"))) diff --git a/modules/nf-commons/src/main/nextflow/config/RegistryConfig.groovy b/modules/nf-commons/src/main/nextflow/config/RegistryConfig.groovy new file mode 100644 index 0000000000..cb232af33b --- /dev/null +++ b/modules/nf-commons/src/main/nextflow/config/RegistryConfig.groovy @@ -0,0 +1,91 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.config + +import groovy.transform.CompileStatic +import nextflow.SysEnv +import nextflow.config.spec.ConfigOption +import nextflow.config.spec.ConfigScope +import nextflow.config.spec.ScopeName +import nextflow.script.dsl.Description + +/** + * Configuration scope for module registry settings + * + * @author Jorge Ejarque + */ +@ScopeName("registry") +@Description(""" + The `registry` scope provides configuration for the Nextflow module registry. + This includes the registry URL(s) and API key for authentication. +""") +@CompileStatic +class RegistryConfig implements ConfigScope { + + final static public String DEFAULT_REGISTRY_URL = 'https://registry.nextflow.io/api' + + @ConfigOption + @Description("Registry URL or list of registry URLs in priority order (primary URL first)") + final private Collection url + + @ConfigOption + @Description("API key for authenticating with the primary registry") + final private String apiKey + + /* required by extension point -- do not remove */ + RegistryConfig() { + this.url = [DEFAULT_REGISTRY_URL] + this.apiKey = null + } + + RegistryConfig(Map opts) { + final urlObject = opts.url ?: [DEFAULT_REGISTRY_URL] + if (urlObject instanceof Collection) + this.url = urlObject as Collection + else + this.url = [urlObject.toString()] + this.apiKey = opts.apiKey as String + } + + /** + * Get the primary (first) registry URL + * + * @return The primary registry URL + */ + String getUrl() { + return this.url ? url[0] as String : DEFAULT_REGISTRY_URL + } + + /** + * Get all registry URLs (primary first, fallbacks after) + * + * @return Collection of registry URLs + */ + Collection getAllUrls() { + return this.url ?: [DEFAULT_REGISTRY_URL] + } + + /** + * Get the API key for the primary registry. + * Authentication is only supported for the primary registry. + * + * @return The API key, or 'NXF_REGISTRY_TOKEN' env value if not configured + */ + String getApiKey() { + return apiKey ?: SysEnv.get('NXF_REGISTRY_TOKEN') + } +} diff --git a/modules/nf-commons/src/main/nextflow/plugin/HttpPluginRepository.groovy b/modules/nf-commons/src/main/nextflow/plugin/HttpPluginRepository.groovy index 48a81218ea..d6497fea4e 100644 --- a/modules/nf-commons/src/main/nextflow/plugin/HttpPluginRepository.groovy +++ b/modules/nf-commons/src/main/nextflow/plugin/HttpPluginRepository.groovy @@ -152,7 +152,7 @@ class HttpPluginRepository implements PrefetchUpdateRepository { } try { final ListDependenciesResponse decoded = encoder.decode(body) - if( decoded.plugins == null ) { + if( !decoded.plugins ) { throw new PluginRuntimeException("Failed to download plugin metadata: Failed to parse response body") } final result = new HashMap() diff --git a/modules/nf-commons/src/test/nextflow/config/RegistryConfigTest.groovy b/modules/nf-commons/src/test/nextflow/config/RegistryConfigTest.groovy new file mode 100644 index 0000000000..26c727d960 --- /dev/null +++ b/modules/nf-commons/src/test/nextflow/config/RegistryConfigTest.groovy @@ -0,0 +1,107 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.config + +import nextflow.SysEnv +import spock.lang.Specification + +/** + * Tests for RegistryConfig + * + * @author Jorge Ejarque + */ +class RegistryConfigTest extends Specification { + + def 'should create config with default values'() { + when: + def config = new RegistryConfig() + + then: + config.url == RegistryConfig.DEFAULT_REGISTRY_URL + config.allUrls == [RegistryConfig.DEFAULT_REGISTRY_URL] + config.apiKey == null + } + + def 'should initialize with custom URL as string'() { + when: + def config = new RegistryConfig([url: 'https://custom.registry.com']) + + then: + config.url == 'https://custom.registry.com' + config.allUrls == ['https://custom.registry.com'] + } + + def 'should initialize with URL as list'() { + when: + def config = new RegistryConfig([url: ['https://primary.registry.com', 'https://fallback.registry.com']]) + + then: + config.url == 'https://primary.registry.com' + config.allUrls == ['https://primary.registry.com', 'https://fallback.registry.com'] + } + + def 'should use default URL when none provided'() { + when: + def config = new RegistryConfig([:]) + + then: + config.url == RegistryConfig.DEFAULT_REGISTRY_URL + config.allUrls == [RegistryConfig.DEFAULT_REGISTRY_URL] + config.apiKey == null + } + + def 'should initialize with apiKey'() { + when: + def config = new RegistryConfig([url: 'https://registry.com', apiKey: 'token123']) + + then: + config.apiKey == 'token123' + } + + + + def 'should fall back to NXF_REGISTRY_TOKEN env var when apiKey not set'() { + given: + SysEnv.push([NXF_REGISTRY_TOKEN: 'env_var_token']) + def config = new RegistryConfig([url: 'https://registry.com']) + + expect: + // Without env var set, returns null + config.apiKey == 'env_var_token' + + cleanup: + SysEnv.pop() + } + + def 'should preserve order of URLs in list'() { + when: + def config = new RegistryConfig([url: ['https://first.com', 'https://second.com', 'https://third.com']]) + + then: + config.allUrls == ['https://first.com', 'https://second.com', 'https://third.com'] + config.url == 'https://first.com' + } + + def 'should handle empty list gracefully'() { + when: + def config = new RegistryConfig([url: []]) + + then: + config.allUrls == [RegistryConfig.DEFAULT_REGISTRY_URL] + config.url == RegistryConfig.DEFAULT_REGISTRY_URL + } +} \ No newline at end of file From 73e9707507a7e26dc727c529948cda9668c4d1bf Mon Sep 17 00:00:00 2001 From: jorgee Date: Thu, 26 Feb 2026 20:34:38 +0100 Subject: [PATCH 11/23] look for checksum in redirected headers Signed-off-by: jorgee --- .../module/ModuleRegistryClient.groovy | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy index 4d5e3cf6ed..b2cec401f5 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy @@ -296,13 +296,8 @@ class ModuleRegistryClient { } private void validateDownloadIntegrity(HttpResponse response, uri, Path targetPath, String name, String version) { - // Get checksum from headers (X-Checksum or Docker-Content-Digest) def checksumType = ModuleChecksum.CHECKSUM_ALGORITHM - def checksum = response.headers().firstValue("X-Checksum").orElse(null) - - if( !checksum ) { - checksum = response.headers().firstValue("Docker-Content-Digest").orElse(null) - } + def checksum = getChecksumFromHeaders(response) if( !checksum ) { log.warn "No X-Checksum or Docker-Content-Digest header found in response from ${uri}" @@ -331,6 +326,34 @@ class ModuleRegistryClient { log.debug "Checksum validated successfully: ${checksumType}:${checksum}" } + private String getChecksumFromHeaders(HttpResponse response) { + // Get X-Checksum from response headers + def checksum = getChecksumFromHeader(response) + if( checksum ) { + return checksum + } + // If not look if it is a previous redirected response header + Optional> prev = response.previousResponse() + while( prev.isPresent() ) { + HttpResponse r = prev.get(); + checksum = getChecksumFromHeader(r) + if( checksum ) { + return checksum + } + prev = r.previousResponse(); + } + return null + } + + private String getChecksumFromHeader(HttpResponse response) { + def checksum = response.headers().firstValue("X-Checksum").orElse(null) + if( !checksum ) { + checksum = response.headers().firstValue("Docker-Content-Digest").orElse(null) + } + return checksum + } + + /** * Search for modules in the registry * From c154d355f23bbccd94f87dd5d749f2fee9421498 Mon Sep 17 00:00:00 2001 From: jorgee Date: Thu, 26 Feb 2026 20:51:30 +0100 Subject: [PATCH 12/23] rename checksum header Signed-off-by: jorgee --- .../src/main/groovy/nextflow/module/ModuleRegistryClient.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy index b2cec401f5..40d91f9542 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy @@ -346,7 +346,7 @@ class ModuleRegistryClient { } private String getChecksumFromHeader(HttpResponse response) { - def checksum = response.headers().firstValue("X-Checksum").orElse(null) + def checksum = response.headers().firstValue("X-NF-Module-Checksum").orElse(null) if( !checksum ) { checksum = response.headers().firstValue("Docker-Content-Digest").orElse(null) } From e9245090ba0a994f4f95ecb6a5b6b406b07510b7 Mon Sep 17 00:00:00 2001 From: jorgee Date: Fri, 27 Feb 2026 12:46:29 +0100 Subject: [PATCH 13/23] update docs Signed-off-by: jorgee --- docs/cli.md | 2 +- docs/reference/cli.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 075ffa9a8c..4922b0e463 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -387,7 +387,7 @@ $ nextflow module publish myorg/my-module $ nextflow module publish myorg/my-module -dry-run ``` -Publishing requires authentication via the `NXF_REGISTRY_TOKEN` environment variable or `registry.auth` in the Nextflow configuration. The module must include `main.nf`, `meta.yaml`, and `README.md` files. +Publishing requires authentication via the `NXF_REGISTRY_TOKEN` environment variable or `registry.apiKey` in the Nextflow configuration. The module must include `main.nf`, `meta.yaml`, and `README.md` files. Use `-dry-run` to validate your module structure without uploading. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 1e43be63c8..0ff72efcf3 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1305,10 +1305,11 @@ The `module` command provides a comprehensive system for managing reusable, regi (cli-module-publish)= -`publish [options] [scope/name]` +`publish [options] [scope/name | path]` : Publish a module to the registry, making it available for others to install. -: Requires authentication via `NXF_REGISTRY_TOKEN` environment variable or `registry.auth` configuration. +: The argument can be either a `scope/name` reference (for an already-installed module) or a local directory path containing the module files. +: Requires authentication via `NXF_REGISTRY_TOKEN` environment variable or `registry.apiKey` configuration. : The module directory must contain `main.nf`, `meta.yaml`, and `README.md`. : The following options are available: From 412ba531d94292fb2bd2856912674d8b3dc95548 Mon Sep 17 00:00:00 2001 From: jorgee Date: Fri, 27 Feb 2026 12:59:22 +0100 Subject: [PATCH 14/23] fix NPE when no nextflow.config and change preference in registry url publish (-registry -> config.url -> default) Signed-off-by: jorgee --- .../src/main/groovy/nextflow/cli/module/ModuleInfo.groovy | 2 +- .../main/groovy/nextflow/cli/module/ModuleInstall.groovy | 2 +- .../main/groovy/nextflow/cli/module/ModulePublish.groovy | 8 ++++---- .../src/main/groovy/nextflow/cli/module/ModuleRun.groovy | 2 +- .../main/groovy/nextflow/cli/module/ModuleSearch.groovy | 2 +- .../groovy/nextflow/module/ModuleRegistryClient.groovy | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInfo.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInfo.groovy index a0a49827f1..1c9582e231 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInfo.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInfo.groovy @@ -85,7 +85,7 @@ class ModuleInfo extends CmdBase { .setOptions(launcher.options) .setBaseDir(baseDir) .build() - final registryConfig = config.navigate('registry') as RegistryConfig + final registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig() // Fetch full metadata from registry to get input/output parameters diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy index 915ab6afc2..2a37281297 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy @@ -80,7 +80,7 @@ class ModuleInstall extends CmdBase { .setOptions(launcher.options) .setBaseDir(baseDir) .build() - final registryConfig = config.navigate('registry') as RegistryConfig + final registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig() // Get modules versions from nextflow_spec.json. final specFile = new PipelineSpec(baseDir) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy index f17395c6ca..e3bc9cd77b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy @@ -48,8 +48,8 @@ class ModulePublish extends CmdBase { @Parameter(names = ["-dry-run"], description = "Validate without uploading", arity=0) boolean dryRun = false - @Parameter(names = ["-registry"], description = "Target registry URL") - String registryUrl = RegistryConfig.DEFAULT_REGISTRY_URL + @Parameter(names = ["-registry"], description = "Target registry URL.") + String registryUrl @Parameter(description = "Module directory path or scope/name") List args @@ -107,7 +107,7 @@ class ModulePublish extends CmdBase { .setBaseDir(moduleDir) .build() - def registryConfig = config.navigate('registry') as RegistryConfig + def registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig() publishModule(moduleDir, registryConfig, manifest) @@ -135,7 +135,7 @@ class ModulePublish extends CmdBase { ] // Publish to registry - log.info "Publishing module to registry: ${registryUrl}" + log.info "Publishing module to registry: ${registryUrl ?: registryConfig.url}" def registryClient = new ModuleRegistryClient(registryConfig) def response = registryClient.publishModule(manifest.name, request, registryUrl) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy index 8b59dbf876..76f179b2c2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy @@ -79,7 +79,7 @@ class ModuleRun extends CmdRun { .setBaseDir(baseDir) .build() - def registryConfig = config.navigate('registry') as RegistryConfig + def registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig() //Get module version from nextflow_spec.json. def specFile = new PipelineSpec(baseDir) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy index 3698067319..e424bd466f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy @@ -73,7 +73,7 @@ class ModuleSearch extends CmdBase { .setBaseDir(baseDir) .build() - final registryConfig = config.navigate('registry') as RegistryConfig + final registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig() // Create client to search final client = this.client ?: new ModuleRegistryClient(registryConfig) diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy index 40d91f9542..d7bf1c99e6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy @@ -440,7 +440,7 @@ class ModuleRegistryClient { "Authentication required to publish modules.\n" + "Please set 'NXF_REGISTRY_TOKEN' environment variable or configure 'registry.apiKey' in nextflow.config:\n\n" + " registry {\n" + - " apiKey = '\${NXF_REGISTRY_TOKEN}'\n" + + " apiKey = 'YOUR_REGISTRY_TOKEN'\n" + " }\n" ) } From 119c6329d7b99735a796b3d975c653a088b40015 Mon Sep 17 00:00:00 2001 From: jorgee Date: Fri, 27 Feb 2026 14:29:16 +0100 Subject: [PATCH 15/23] update speckit files [ci skip] Signed-off-by: jorgee --- specs/251117-module-system/data-model.md | 198 +++++++------------ specs/251117-module-system/plan.md | 59 +++--- specs/251117-module-system/quickstart.md | 117 ++++-------- specs/251117-module-system/research.md | 231 +++++++++++------------ specs/251117-module-system/spec.md | 15 +- 5 files changed, 256 insertions(+), 364 deletions(-) diff --git a/specs/251117-module-system/data-model.md b/specs/251117-module-system/data-model.md index d9345e33a4..448509e52b 100644 --- a/specs/251117-module-system/data-model.md +++ b/specs/251117-module-system/data-model.md @@ -2,6 +2,7 @@ **Date**: 2026-01-19 **Feature**: 251117-module-system +**Last Updated**: 2026-02-27 (reflects final implementation) ## Overview @@ -39,88 +40,39 @@ class ModuleReference { --- -### 2. ModuleManifest +### 2. ModuleSpec -Parsed representation of `meta.yaml` file. +Parsed representation of `meta.yaml` file. Class: `nextflow.module.ModuleSpec`. ```groovy @CompileStatic -class ModuleManifest { +class ModuleSpec { String name // e.g., "nf-core/fastqc" (without @) String version // e.g., "1.0.0" String description // Module description List keywords // Discovery keywords List authors // GitHub handles - List maintainers String license // SPDX identifier + Map requires // dependency -> version constraint - ModuleRequirements requires - List tools - List input - Map output -} - -@CompileStatic -class ModuleRequirements { - String nextflow // Version constraint, e.g., ">=24.04.0" - List plugins // e.g., ["nf-amazon@2.0.0"] - List modules // e.g., ["nf-core/samtools@>=1.0.0"] - List workflows // e.g., ["nf-core/fastq-align@1.0.0"] -} - -@CompileStatic -class ToolDefinition { - String name // Tool identifier - String description - String homepage - String documentation - String doi - List license - String identifier // bio.tools identifier - Map args -} - -@CompileStatic -class ArgDefinition { - String flag // CLI flag, e.g., "-K" - String type // boolean, integer, float, string, file, path - String description - Object defaultValue - List enumValues - boolean required = false + static ModuleSpec load(Path metaYamlPath) { ... } + List validate() { ... } // Returns list of validation errors + boolean isValid() { ... } } ``` **Validation Rules**: -- `version`: Must be valid SemVer (MAJOR.MINOR.PATCH) -- `type` in ArgDefinition: Must be one of: boolean, integer, float, string, file, path -- `enumValues`: If present, configured value must be in this list +- `name`: Must match `scope/name` or `scope/path/to/name` pattern +- `version`: Must be valid SemVer (`MAJOR.MINOR.PATCH[-prerelease]`) +- `description`, `license`: Required fields (validate() reports missing) ---- - -### 3. ModuleInfo - -Module metadata returned from registry API. - -```groovy -@CompileStatic -class ModuleInfo { - String name // e.g., "nf-core/fastqc" - String version // Specific version - String latestVersion // Latest available - String description - String checksum // SHA-256 of bundle - long downloadCount - Instant publishedAt - List versions // All available versions -} -``` +**Note**: Tool/argument definitions were removed from the ADR and are not part of `ModuleSpec`. --- -### 4. InstalledModule +### 3. InstalledModule -Represents a module in local `modules/` directory. +Represents a module in the local `modules/` directory. ```groovy @CompileStatic @@ -132,7 +84,6 @@ class InstalledModule { Path checksumFile // e.g., /project/modules/@nf-core/fastqc/.checksum String installedVersion String expectedChecksum - ModuleManifest manifest ModuleIntegrity getIntegrity() { // Compute and compare checksum @@ -158,84 +109,77 @@ enum ModuleIntegrity { --- -### 5. ModuleConfig +### 4. ModulesConfig and RegistryConfig -Module configuration from `nextflow.config`. +Modules configuration loaded from `nextflow_spec.json` ( or the `modules {}` block in `nextflow.config` as alternative). Registry settings from the `registry {}` block in `nextflow.config`. ```groovy +@ScopeName("modules") @CompileStatic -class ModuleConfig { - Map modules = [:] // name -> version - RegistryConfig registry +class ModulesConfig implements ConfigScope { + Map modules = [:] // module fullName -> version + + String getVersion(String moduleName) { ... } + boolean hasVersion(String moduleName) { ... } } +@ScopeName("registry") @CompileStatic -class RegistryConfig { - String url = 'https://registry.nextflow.io' - List urls = [] // Multiple registries - Map auth = [:] // registry -> token expression +class RegistryConfig implements ConfigScope { + static final String DEFAULT_REGISTRY_URL = 'https://registry.nextflow.io/api' + + Collection url // Registry URL(s) in priority order + String apiKey // API key (falls back to NXF_REGISTRY_TOKEN env var) + + String getUrl() // Returns primary (first) URL + Collection getAllUrls() + String getApiKey() // Returns apiKey or NXF_REGISTRY_TOKEN } ``` **Config Syntax**: -```groovy +```nextflow +// nextflow_spec.json (current approach) +{ + "modules": { + "@nf-core/fastqc": "1.0.0", + "@nf-core/bwa-align": "1.2.0" + } +} + +// nextflow.config (alternative not currently used) modules { '@nf-core/fastqc' = '1.0.0' '@nf-core/bwa-align' = '1.2.0' } registry { - url = 'https://registry.nextflow.io' - auth { - 'registry.nextflow.io' = '${NXF_REGISTRY_TOKEN}' - } + url = [ + 'https://private.registry.myorg.com', + 'https://registry.nextflow.io/api' + ] + apiKey = '${MYORG_TOKEN}' // Only applied to the primary registry } ``` --- -### 6. ToolArgsContext +### 5. PipelineSpec -Runtime context for tool arguments in process scripts. +Reads and writes `nextflow_spec.json` in the project root. Class: `nextflow.pipeline.PipelineSpec`. ```groovy -@CompileStatic -class ToolArgsContext { - private Map tools = [:] - - ToolArgs getAt(String toolName) { - return tools[toolName] - } -} - -@CompileStatic -class ToolArgs { - private Map schema - private Map values - - String getAt(String argName) { - def def = schema[argName] - def value = values[argName] - if (value == null) return '' - if (def.type == 'boolean') { - return value ? def.flag : '' - } - return "${def.flag} ${value}" - } - - String toString() { - schema.keySet() - .findAll { values.containsKey(it) && values[it] != null } - .collect { this[it] } - .findAll { it } - .join(' ') - } +class PipelineSpec { + PipelineSpec(Path baseDir) + Map getModules() + void addModuleEntry(String name, String version) + boolean removeModuleEntry(String name) } ``` --- -### 7. ModuleResolutionResult +### 6. ModuleResolutionResult Result of module resolution process. @@ -246,7 +190,6 @@ class ModuleResolutionResult { Path resolvedPath // Absolute path to main.nf ResolutionAction action String message // Warning/info message if any - ModuleManifest manifest } enum ResolutionAction { @@ -263,22 +206,14 @@ enum ResolutionAction { ## Relationships ``` -ModuleConfig (1) -----> (*) ModuleReference - | - v +PipelineSpec (1) -----> (*) ModuleReference (nextflow_spec.json) +ModulesConfig (1) -----> (*) ModuleReference (nextflow.config alternative) RegistryConfig (1) -----> (*) Registry URLs ModuleReference (1) -----> (0..1) InstalledModule | v (via registry) -ModuleInfo (1) -----> (1) ModuleManifest - -InstalledModule (1) -----> (1) ModuleManifest - -----> (*) ToolDefinition - -----> (*) ArgDefinition - -ToolArgsContext (1) -----> (*) ToolArgs - -----> (*) ArgDefinition (schema) +ModuleSpec (1) <----- InstalledModule (from meta.yaml) ``` --- @@ -287,15 +222,16 @@ ToolArgsContext (1) -----> (*) ToolArgs ``` project-root/ -├── nextflow.config # modules{}, registry{} blocks +├── nextflow.config # registry{} block; optional modules{} block +├── nextflow_spec.json # auto-managed module version pins ├── main.nf # include { X } from '@scope/name' └── modules/ └── @scope/ └── name/ - ├── .checksum # SHA-256 from registry + ├── .checksum # SHA-256 from registry (download integrity) ├── main.nf # Entry point (required) - ├── meta.yaml # Manifest (optional but recommended) - ├── README.md # Documentation + ├── meta.yaml # Manifest (required for publishing) + ├── README.md # Documentation (required for publishing) └── [other files] # Supporting files ``` @@ -306,9 +242,9 @@ project-root/ | Entity | Field | Validation | |--------|-------|------------| | ModuleReference | fullName | Pattern: `^@[a-z0-9][a-z0-9-]*/[a-z][a-z0-9_-]*$` | -| ModuleManifest | version | SemVer: `MAJOR.MINOR.PATCH` | -| ArgDefinition | type | Enum: boolean, integer, float, string, file, path | -| ArgDefinition | enumValues | If set, value must be member | +| ModuleSpec | name | Pattern: `scope/name` or `scope/path/to/name` | +| ModuleSpec | version | SemVer: `MAJOR.MINOR.PATCH[-prerelease]` | +| ModuleSpec | description, license | Required (non-empty) | | InstalledModule | directory | Must contain main.nf | -| ModuleConfig | modules | Keys must be valid module references | +| ModulesConfig | modules keys | Must be valid module fullName | | RegistryConfig | url | Valid HTTPS URL | \ No newline at end of file diff --git a/specs/251117-module-system/plan.md b/specs/251117-module-system/plan.md index 0d12cf3f96..03ecb5cfff 100644 --- a/specs/251117-module-system/plan.md +++ b/specs/251117-module-system/plan.md @@ -15,6 +15,7 @@ Implement client-side module system for Nextflow enabling pipeline developers to - Existing config parser (ConfigBuilder, ConfigParser) - Existing HTTP client (HxClient from io.seqera.http) - Existing plugin authentication infrastructure +- Existing npr-api (registry data models and schema validation) **Storage**: Local filesystem (`modules/@scope/name/` per-project, `.checksum` files) **Testing**: Spock Framework for unit tests, integration tests in `tests/` directory **Target Platform**: JVM 17+ (same as Nextflow core) @@ -49,6 +50,7 @@ Implement client-side module system for Nextflow enabling pipeline developers to ```text specs/251117-module-system/ ├── plan.md # This file +├── spec.md # Feature specification ├── research.md # Phase 0 output ├── data-model.md # Phase 1 output ├── quickstart.md # Phase 1 output @@ -61,37 +63,46 @@ specs/251117-module-system/ ```text modules/nextflow/src/main/groovy/nextflow/ ├── cli/ -│ └── CmdModule.groovy # NEW: Module CLI command +│ ├── CmdModule.groovy # Main module command (uses JCommander) +│ └── module/ +│ ├── ModuleInstall.groovy # Install subcommand (extends CmdBase) +│ ├── ModuleRun.groovy # Run subcommand (extends CmdRun) +│ ├── ModuleList.groovy # List subcommand (extends CmdBase) +│ ├── ModuleRemove.groovy # Remove subcommand (extends CmdBase) +│ ├── ModuleSearch.groovy # Search subcommand (extends CmdBase) +│ ├── ModuleInfo.groovy # Info subcommand (extends CmdBase) +│ └── ModulePublish.groovy # Publish subcommand (extends CmdBase) ├── config/ -│ ├── ConfigBuilder.groovy # MODIFY: Add modules/registry DSL -│ └── parser/v1/ -│ ├── ModulesDsl.groovy # NEW: modules {} block parser -│ └── RegistryDsl.groovy # NEW: registry {} block parser -└── module/ - ├── ModuleResolver.groovy # NEW: Core resolution logic - ├── ModuleStorage.groovy # NEW: Local storage management - ├── ModuleChecksum.groovy # NEW: Checksum verification - ├── ModuleManifest.groovy # NEW: meta.yaml parser - └── HttpModuleRepository.groovy # NEW: Registry HTTP client +│ ├── ModulesConfig.groovy # modules{} config scope +│ └── RegistryConfig.groovy # registry{} config scope (fields: url, apiKey) +├── module/ +│ ├── ModuleReference.groovy # @scope/name parser +│ ├── ModuleResolver.groovy # Core resolution logic +│ ├── ModuleStorage.groovy # Local filesystem operations +│ ├── ModuleRegistryClient.groovy # HTTP registry client +│ ├── ModuleChecksum.groovy # SHA-256 integrity verification +│ ├── ModuleSpec.groovy # Module manifest (meta.yaml) entity +│ └── InstalledModule.groovy # Installed module entity +└── pipeline/ + └── PipelineSpec.groovy # nextflow_spec.json read/write modules/nf-lang/src/main/java/nextflow/script/ └── ResolveIncludeVisitor.java # MODIFY: Add @scope/name detection modules/nextflow/src/test/groovy/nextflow/ -├── cli/ -│ └── CmdModuleTest.groovy # NEW: CLI unit tests -├── config/ -│ └── ModulesDslTest.groovy # NEW: Config parsing tests +├── cli/module/ +│ ├── ModuleInstallTest.groovy +│ ├── ModuleRunTest.groovy +│ └── [other subcommand tests] └── module/ - ├── ModuleResolverTest.groovy # NEW: Resolution logic tests - ├── ModuleStorageTest.groovy # NEW: Storage tests - └── ModuleChecksumTest.groovy # NEW: Checksum tests - -tests/ -└── modules/ # NEW: Integration tests - ├── install-module.nf # Test module install + include - ├── version-resolution.nf # Test version management - └── checksum-protection.nf # Test local modification protection + ├── ModuleResolverTest.groovy + ├── ModuleStorageTest.groovy + └── [other module tests] + +tests/modules/ +├── install-module.nf # Integration tests +├── run-module.nf +└── [other integration tests] ``` **Structure Decision**: Implementation extends existing Nextflow core modules following modular architecture. New code in `modules/nextflow` for CLI and core logic. DSL parser extension in `modules/nf-lang`. No new plugins required. diff --git a/specs/251117-module-system/quickstart.md b/specs/251117-module-system/quickstart.md index f9247dcb74..71e356e534 100644 --- a/specs/251117-module-system/quickstart.md +++ b/specs/251117-module-system/quickstart.md @@ -22,7 +22,7 @@ nextflow module install nf-core/fastqc nextflow module install nf-core/fastqc -version 1.0.0 ``` -This downloads the module to `modules/@nf-core/fastqc/` and updates `nextflow.config`. +This downloads the module to `modules/@nf-core/fastqc/` and updates `nextflow_spec.json` with the installed version. ### Use in your workflow @@ -52,11 +52,8 @@ Execute a module without writing a wrapper workflow: # Basic usage nextflow module run nf-core/fastqc --input 'data/*.fastq.gz' -# With tool arguments -nextflow module run nf-core/bwa-align \ - --reads 'samples/*_{1,2}.fastq.gz' \ - --reference genome.fa \ - --tools:bwa:K 100000000 +# Run specific version +nextflow module run nf-core/fastqc --input 'data/*.fastq.gz' -version 1.0.0 # With Nextflow options nextflow module run nf-core/salmon \ @@ -66,18 +63,43 @@ nextflow module run nf-core/salmon \ -resume ``` +## 3. View Module Information + +```bash +# Show module metadata and a generated usage template +nextflow module info nf-core/fastqc + +# Show a specific version +nextflow module info nf-core/fastqc -version 1.0.0 + +# JSON output for scripting +nextflow module info nf-core/fastqc -json +``` + --- -## 3. Manage Module Versions +## 4. Manage Module Versions -### Configure versions in nextflow.config +### Version tracking -```groovy -// nextflow.config +Module versions are automatically recorded in `nextflow_spec.json` by `nextflow module install`. You can also pin versions manually: + +```json +// nextflow_spec.json +{ + "modules": { + "@nf-core/fastqc": "1.0.0", + "@nf-core/bwa-align": "1.2.0" + } +} +``` + +Alternatively, declare versions in `nextflow.config` (not currently used): + +```nextflow modules { '@nf-core/fastqc' = '1.0.0' '@nf-core/bwa-align' = '1.2.0' - '@nf-core/samtools' = '2.1.0' } ``` @@ -96,17 +118,11 @@ nextflow module list ### Update a module -Change the version in `nextflow.config`, then run your workflow. Nextflow automatically downloads the new version. - -```groovy -modules { - '@nf-core/fastqc' = '1.2.0' // Changed from 1.0.0 -} -``` +Change the version in `nextflow_spec.json` (or `nextflow.config`), then run your workflow. Nextflow automatically downloads the new version. --- -## 4. Search for Modules +## 5. Search for Modules ```bash # Search by keyword @@ -121,69 +137,19 @@ nextflow module search bwa -json --- -## 5. Configure Tool Arguments - -### Define in meta.yaml (module author) - -```yaml -# modules/@nf-core/bwa-align/meta.yaml -tools: - - bwa: - description: BWA aligner - args: - K: - flag: "-K" - type: integer - description: "Process INT input bases in each batch" - Y: - flag: "-Y" - type: boolean - description: "Use soft clipping for supplementary alignments" -``` - -### Configure in nextflow.config (user) - -```groovy -// nextflow.config -process { - withName: 'BWA_ALIGN' { - tools.bwa.args.K = 100000000 - tools.bwa.args.Y = true - } -} -``` - -### Access in script (module author) - -```groovy -// main.nf -process BWA_ALIGN { - script: - """ - bwa mem ${tools.bwa.args} -t $task.cpus $index $reads - """ -} -``` - ---- - ## 6. Work with Private Registries ### Configure authentication -```groovy +```nextflow // nextflow.config registry { // Multiple registries (tried in order) url = [ 'https://private.registry.myorg.com', - 'https://registry.nextflow.io' + 'https://registry.nextflow.io/api' ] - - auth { - 'private.registry.myorg.com' = '${MYORG_TOKEN}' - 'registry.nextflow.io' = '${NXF_REGISTRY_TOKEN}' - } + apiKey = 'MYORG_TOKEN' // Applied to the primary (first) registry only } ``` @@ -255,13 +221,6 @@ nextflow module remove nf-core/fastqc -keep-files ## Common Patterns -### Install all configured modules - -```bash -# Installs all modules listed in nextflow.config -nextflow module install -``` - ### Offline operation Modules are cached locally in `modules/`. Once installed, workflows run without network access. diff --git a/specs/251117-module-system/research.md b/specs/251117-module-system/research.md index 270603da82..40cb783e2b 100644 --- a/specs/251117-module-system/research.md +++ b/specs/251117-module-system/research.md @@ -13,28 +13,44 @@ This document captures technical research and decisions for implementing the Nex **Research Question**: How should `nextflow module` CLI commands be implemented? -**Decision**: Follow CmdPlugin pattern with sub-command delegation +**Decision**: JCommander native subcommands — each subcommand extends `CmdBase` directly; no trait needed **Rationale**: -- CmdPlugin.groovy provides proven pattern for multi-action commands -- Uses JCommander `@Parameters` and `@Parameter` annotations -- Sub-commands (install, search, list, remove, publish, run) handled via positional args -- PluginExecAware interface allows plugin extensibility if needed later +- JCommander's subcommand support handles parameter parsing automatically per subcommand +- Each subcommand (install, run, list, remove, search, info, publish) is a separate class extending CmdBase +- `ModuleRun` extends `CmdRun` to reuse pipeline execution logic (PR #6381) +- No custom `ModuleSubCmd` trait needed; cleaner architecture +- `CmdModule` is registered in `Launcher` alongside all other top-level commands -**Reference Implementation**: -``` -Location: modules/nextflow/src/main/groovy/nextflow/cli/CmdPlugin.groovy -Pattern: - - Extends CmdBase - - @Parameters(commandNames = 'module', commandDescription = '...') - - @Parameter(names = ['-h', '--help']) - - args list for sub-command + module name - - run() method dispatches to install(), search(), etc. +**Implemented Pattern**: +```groovy +@Parameters(commandDescription = "Manage Nextflow modules") +class CmdModule extends CmdBase implements UsageAware { + static final List commands = [] + + static { + commands << new ModuleInstall() // extends CmdBase + commands << new ModuleRun() // extends CmdRun + commands << new ModuleList() // extends CmdBase + commands << new ModuleRemove() // extends CmdBase + commands << new ModuleSearch() // extends CmdBase + commands << new ModuleInfo() // extends CmdBase + commands << new ModulePublish() // extends CmdBase + } + + void run() { + final jc = commander() // JCommander with all subcommands registered + jc.parse(args as String[]) + final subcommand = jc.getCommands().get(jc.getParsedCommand()).getObjects()[0] + subcommand.run() + } +} ``` **Alternatives Considered**: -- Separate CmdModuleInstall, CmdModuleSearch classes: Rejected - too many entry points, doesn't match existing patterns -- Plugin-based CLI extension: Rejected - module system is core functionality, not optional +- CmdFs trait pattern: Considered initially; replaced by JCommander native subcommands — simpler and avoids custom parsing +- Separate top-level Cmd classes (CmdModuleInstall, etc.): Rejected — too many entry points +- Plugin-based CLI extension: Rejected — module system is core functionality, not optional --- @@ -76,46 +92,75 @@ Pattern: **Research Question**: How to add new config DSL blocks? -**Decision**: Create ModulesDsl and RegistryDsl classes following PluginsDsl pattern +**Decision**: Create ModulesConfig and RegistryConfig classes implementing ConfigScope interface **Rationale**: -- PluginsDsl.groovy provides exact template for DSL block handling -- ConfigBuilder already supports dynamic DSL registration -- Groovy's methodMissing enables clean config syntax +- ConfigScope is an ExtensionPoint (pf4j) that ConfigBuilder automatically discovers +- Classes implementing ConfigScope and annotated with @ScopeName are automatically parsed +- No need to modify ConfigBuilder or create custom DSL parsers +- Pattern used throughout Nextflow: FusionConfig, CondaConfig, DockerConfig, etc. +- Provides type safety via @CompileStatic and validation via @ConfigOption **Reference Implementation**: ``` -Location: modules/nextflow/src/main/groovy/nextflow/config/parser/v1/PluginsDsl.groovy +Location: modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy Pattern: + @ScopeName("modules") + @Description("Module version declarations") @CompileStatic - class ModulesDsl { - private Map modules = [:] + class ModulesConfig implements ConfigScope { + @ConfigOption + @Description("Module version mappings") + final Map modules = [:] - def methodMissing(String name, args) { - // modules { '@nf-core/fastqc' = '1.0.0' } - modules[name] = args[0].toString() - } + ModulesConfig() {} - Map getModules() { modules } + ModulesConfig(Map opts) { + // Parse from config map + } } ``` -**RegistryDsl Pattern**: +**ConfigScope Interface**: +``` +Location: modules/nf-lang/src/main/java/nextflow/config/spec/ConfigScope.java +public interface ConfigScope extends ExtensionPoint {} +``` + +**RegistryConfig Pattern**: ```groovy -class RegistryDsl { - String url = 'https://registry.nextflow.io' - List urls = [] // For multiple registries - Map auth = [:] - - void url(String value) { this.url = value } - void url(List values) { this.urls = values } - void auth(Closure config) { /* parse auth block */ } +@ScopeName("registry") +@Description("Module registry configuration") +@CompileStatic +class RegistryConfig implements ConfigScope { + static final String DEFAULT_REGISTRY_URL = 'https://registry.nextflow.io/api' + + @ConfigOption + final Collection url // One or more URLs in priority order + + @ConfigOption + final String apiKey // API key; falls back to NXF_REGISTRY_TOKEN env var + + RegistryConfig() { + url = [DEFAULT_REGISTRY_URL] + apiKey = null + } + + RegistryConfig(Map opts) { + url = opts.url ?: [DEFAULT_REGISTRY_URL] + apiKey = opts.apiKey as String + } + + String getUrl() { url ? url[0] : DEFAULT_REGISTRY_URL } + Collection getAllUrls() { url ?: [DEFAULT_REGISTRY_URL] } + String getApiKey() { apiKey ?: SysEnv.get('NXF_REGISTRY_TOKEN') } } ``` -**Integration Point**: ConfigBuilder.build() instantiates DSL objects +**Integration Point**: ConfigBuilder automatically discovers and parses ConfigScope implementations via ExtensionPoint mechanism **Alternatives Considered**: +- Custom DSL parsers (ModulesDsl/RegistryDsl): Rejected - unnecessary complexity, ConfigScope pattern handles this automatically - JSON/YAML config file: Rejected - inconsistent with Nextflow config style - Dedicated pipeline.yaml: Deferred per ADR Open Questions @@ -169,36 +214,33 @@ POST /api/modules/{name} # Publish (authenticated) **Research Question**: How to handle registry authentication? -**Decision**: Support NXF_REGISTRY_TOKEN env var + registry.auth config block +**Decision**: Support `NXF_REGISTRY_TOKEN` env var + `registry.apiKey` config field **Rationale**: - Environment variable provides CI/CD compatibility -- Config block allows per-registry tokens for private registries -- Follows existing plugin auth patterns +- `apiKey` config field allows explicit token configuration +- Authentication is only applied to the primary (first) registry URL - Bearer token in Authorization header (standard HTTP auth) -**Reference Implementation**: +**Implementation**: ``` -Location: modules/nextflow/src/main/groovy/nextflow/cli/CmdAuth.groovy -Pattern: - 1. Check NXF_REGISTRY_TOKEN environment variable - 2. Fall back to registry.auth.'registry.nextflow.io' in config - 3. Add header: Authorization: Bearer +RegistryConfig.getApiKey() returns: + 1. registry.apiKey config value if set + 2. NXF_REGISTRY_TOKEN environment variable as fallback + 3. null if neither is set (unauthenticated requests) ``` **Config Syntax**: -```groovy +```nextflow registry { - auth { - 'registry.nextflow.io' = '${NXF_REGISTRY_TOKEN}' - 'private.registry.com' = '${PRIVATE_TOKEN}' - } + apiKey = '${NXF_REGISTRY_TOKEN}' } ``` **Alternatives Considered**: +- Per-registry token map (`auth {}` block): Was in initial design; simplified to single `apiKey` since only the primary registry uses authentication - Secrets file (~/.nextflow/secrets.json): Possible future enhancement -- OAuth flow: Rejected for CLI - token-based simpler +- OAuth flow: Rejected for CLI — token-based simpler --- @@ -275,59 +317,7 @@ class ModuleChecksum { ## 8. Tool Arguments Implementation -**Research Question**: How to implement structured tool arguments (`tools..args`)? - -**Decision**: Implement as implicit variable in process scope, validated at parse time - -**Rationale**: -- `tools` variable accessible in script block like `task`, `params` -- Validation at parse time catches errors early (per clarification) -- Schema defined in meta.yaml, parsed by ModuleManifest -- Concatenation logic handles flag formatting - -**Implementation Pattern**: -```groovy -class ToolArgs { - private Map schema // From meta.yaml - private Map values // From config - - String getAt(String argName) { - def def = schema[argName] - def value = values[argName] - if (def.type == 'boolean' && value) { - return def.flag // e.g., "-Y" - } - return "${def.flag} ${value}" // e.g., "-K 100000" - } - - String toString() { - // Concatenate all configured args - values.collect { name, value -> - this[name] - }.join(' ') - } -} -``` - -**Config Access**: -```groovy -withName: 'BWA_MEM' { - tools.bwa.args.K = 100000 - tools.bwa.args.Y = true -} -``` - -**Script Access**: -```groovy -script: -""" -bwa mem ${tools.bwa.args} -t $task.cpus $index $reads -""" -``` - -**Alternatives Considered**: -- Runtime validation only: Rejected - late errors waste compute -- String-only values: Rejected - loses type safety benefits +> **⚠️ REMOVED FROM ADR** — The tool arguments feature (`tools..args` in meta.yaml and process config) was removed from the module system ADR. It is not implemented and not planned in the current scope. The `meta.yaml` format used in the actual implementation (`ModuleSpec`) does not include tool/argument definitions. --- @@ -335,26 +325,21 @@ bwa mem ${tools.bwa.args} -t $task.cpus $index $reads | Area | Decision | Key Reference | |------|----------|---------------| -| CLI | CmdModule extends CmdBase | CmdPlugin.groovy | -| DSL Parser | Extend ResolveIncludeVisitor | ResolveIncludeVisitor.java | -| Config | ModulesDsl + RegistryDsl | PluginsDsl.groovy | -| Registry HTTP | HttpModuleRepository | HttpPluginRepository.groovy | -| Authentication | NXF_REGISTRY_TOKEN + config | CmdAuth.groovy | -| Checksums | SHA-256, .checksum file | Standard Java security | +| CLI | JCommander subcommands; each extends CmdBase (ModuleRun extends CmdRun) | CmdModule.groovy | +| DSL Parser | Extend ResolveIncludeVisitor for `@scope/name` — pending | ResolveIncludeVisitor.java | +| Config | ModulesConfig + RegistryConfig (ConfigScope) | FusionConfig.groovy, ConfigScope.java | +| Registry HTTP | ModuleRegistryClient using HxClient + npr-api models | HttpPluginRepository.groovy | +| Authentication | `NXF_REGISTRY_TOKEN` env var or `registry.apiKey` config field (primary registry only) | RegistryConfig.groovy | +| Checksums | SHA-256/SHA-512, `.checksum` file, download integrity via X-Checksum header | ModuleChecksum.groovy | +| Version Storage | `nextflow_spec.json` (auto-managed); `modules {}` in nextflow.config (manual alternative) | PipelineSpec.groovy | | Version Syntax | Plugin-compatible constraints | VersionNumber class | -| Tool Args | Implicit variable, parse-time validation | New implementation | +| Tool Args | ~~Implicit variable, parse-time validation~~ — **Removed from ADR** | N/A | --- ## Open Items (Deferred) -These items are noted in the ADR as open questions and do not block implementation: - -1. **Local vs managed module distinction**: Whether local modules use `@` prefix or dot file marker -2. **Tool arguments CLI syntax**: Colon vs dot separator (`--tools:bwa:K` vs `--tools.bwa.K`) -3. **Module version location**: nextflow.config vs dedicated pipeline.yaml - -Current implementation uses: -- `@` prefix for registry modules only (local paths start with `.` or `/`) -- Colon-separated CLI syntax per ADR assumption -- Versions in nextflow.config per ADR decision \ No newline at end of file +1. **Local vs managed module distinction**: Resolved — `@` prefix for registry modules only; local paths start with `.` or `/` +2. **Tool arguments**: Removed from ADR — not in scope +3. **Module version location**: Resolved — `nextflow_spec.json` (auto-managed by `module install`); `modules {}` block in `nextflow.config` supported as alternative +4. **DSL parser `@scope/name` include**: Pending (T017) \ No newline at end of file diff --git a/specs/251117-module-system/spec.md b/specs/251117-module-system/spec.md index 652397bb80..fba59f7923 100644 --- a/specs/251117-module-system/spec.md +++ b/specs/251117-module-system/spec.md @@ -27,7 +27,7 @@ A pipeline developer wants to use a pre-built module from the Nextflow registry **Acceptance Scenarios**: -1. **Given** a new Nextflow project with no modules installed, **When** user runs `nextflow module install nf-core/fastqc`, **Then** the module is downloaded to `modules/@nf-core/fastqc/`, a `.checksum` file is created, and `nextflow.config` is updated with the version +1. **Given** a new Nextflow project with no modules installed, **When** user runs `nextflow module install nf-core/fastqc`, **Then** the module is downloaded to `modules/@nf-core/fastqc/`, a `.checksum` file is created, and `nextflow_spec.json` is updated with the version 2. **Given** a workflow file with `include { FASTQC } from '@nf-core/fastqc'`, **When** user runs `nextflow run main.nf`, **Then** Nextflow resolves the module from local storage and executes the process 3. **Given** a module version declared in `nextflow.config`, **When** user includes the module, **Then** the declared version is used (not latest) @@ -75,7 +75,7 @@ A pipeline developer wants to pin and manage module versions to ensure reproduci **Acceptance Scenarios**: -1. **Given** a module is installed at version 1.0.0, **When** user changes `nextflow.config` to specify version 1.1.0 and runs the workflow, **Then** version 1.1.0 is automatically downloaded and replaces the local copy +1. **Given** a module is installed at version 1.0.0, **When** user changes `nextflow_spec.json` to specify version 1.1.0 and runs the workflow, **Then** version 1.1.0 is automatically downloaded and replaces the local copy 2. **Given** modules installed locally, **When** user runs `nextflow module list`, **Then** configured version, installed version, latest available version, and status are displayed for each module --- @@ -106,7 +106,7 @@ A pipeline developer wants to remove a module they no longer need. **Acceptance Scenarios**: -1. **Given** a module is installed, **When** user runs `nextflow module remove nf-core/fastqc`, **Then** the module directory is deleted and the entry is removed from `nextflow.config` +1. **Given** a module is installed, **When** user runs `nextflow module remove nf-core/fastqc`, **Then** the module directory is deleted and the entry is removed from `nextflow_spec.json` 2. **Given** a module is referenced in workflow files, **When** user runs `nextflow module remove`, **Then** a warning is displayed about the reference but removal proceeds --- @@ -166,7 +166,7 @@ A module author wants to publish their module to the Nextflow registry for other - **FR-001**: System MUST recognize `@scope/name` syntax in `include` statements as registry module references - **FR-002**: System MUST distinguish between local file paths (starting with `.` or `/`) and registry modules (starting with `@`) -- **FR-003**: System MUST resolve module versions from `nextflow.config` `modules {}` block before downloading +- **FR-003**: System MUST resolve module versions from `nextflow_spec.json` before downloading - **FR-004**: System MUST parse and validate `meta.yaml` files for module metadata and dependencies #### Module Resolution @@ -192,12 +192,13 @@ A module author wants to publish their module to the Nextflow registry for other - **FR-017**: System MUST provide `nextflow module remove scope/name` command to delete modules - **FR-018**: System MUST provide `nextflow module publish scope/name` command to upload modules to registry - **FR-019**: System MUST provide `nextflow module run scope/name` command to execute modules directly +- **FR-019b**: System MUST provide `nextflow module info scope/name` command to display module metadata and a usage template #### Configuration -- **FR-020**: System MUST read module versions from `modules {}` block in `nextflow.config` -- **FR-021**: System MUST support `registry {}` block for configuring registry URL and authentication -- **FR-022**: System MUST support `NXF_REGISTRY_TOKEN` environment variable for authentication +- **FR-020**: System MUST persist module versions in `nextflow_spec.json`; MUST also read versions from `modules {}` block in `nextflow.config` as an alternative +- **FR-021**: System MUST support `registry {}` block with `url` and `apiKey` fields for configuring registry URL and authentication +- **FR-022**: System MUST support `NXF_REGISTRY_TOKEN` environment variable as fallback for `registry.apiKey` - **FR-023**: System MUST support multiple registry URLs with fallback ordering #### Module Parameters From c20a4085f14a2f674a7d14705e6592dfcdfba55c Mon Sep 17 00:00:00 2001 From: jorgee Date: Wed, 4 Mar 2026 15:53:52 +0100 Subject: [PATCH 16/23] rename module subcommands Signed-off-by: jorgee --- .../main/groovy/nextflow/cli/CmdModule.groovy | 28 +++++++------- ...ModuleInfo.groovy => CmdModuleInfo.groovy} | 7 +--- ...Install.groovy => CmdModuleInstall.groovy} | 2 +- ...ModuleList.groovy => CmdModuleList.groovy} | 2 +- ...Publish.groovy => CmdModulePublish.groovy} | 2 +- ...leRemove.groovy => CmdModuleRemove.groovy} | 2 +- .../{ModuleRun.groovy => CmdModuleRun.groovy} | 2 +- ...leSearch.groovy => CmdModuleSearch.groovy} | 2 +- ...foTest.groovy => CmdModuleInfoTest.groovy} | 38 +++++++++---------- ...est.groovy => CmdModuleInstallTest.groovy} | 24 ++++++------ ...stTest.groovy => CmdModuleListTest.groovy} | 14 +++---- ...est.groovy => CmdModulePublishTest.groovy} | 12 +++--- ...Test.groovy => CmdModuleRemoveTest.groovy} | 18 ++++----- ...RunTest.groovy => CmdModuleRunTest.groovy} | 12 +++--- ...Test.groovy => CmdModuleSearchTest.groovy} | 14 +++---- 15 files changed, 88 insertions(+), 91 deletions(-) rename modules/nextflow/src/main/groovy/nextflow/cli/module/{ModuleInfo.groovy => CmdModuleInfo.groovy} (98%) rename modules/nextflow/src/main/groovy/nextflow/cli/module/{ModuleInstall.groovy => CmdModuleInstall.groovy} (98%) rename modules/nextflow/src/main/groovy/nextflow/cli/module/{ModuleList.groovy => CmdModuleList.groovy} (98%) rename modules/nextflow/src/main/groovy/nextflow/cli/module/{ModulePublish.groovy => CmdModulePublish.groovy} (99%) rename modules/nextflow/src/main/groovy/nextflow/cli/module/{ModuleRemove.groovy => CmdModuleRemove.groovy} (99%) rename modules/nextflow/src/main/groovy/nextflow/cli/module/{ModuleRun.groovy => CmdModuleRun.groovy} (98%) rename modules/nextflow/src/main/groovy/nextflow/cli/module/{ModuleSearch.groovy => CmdModuleSearch.groovy} (99%) rename modules/nextflow/src/test/groovy/nextflow/cli/module/{ModuleInfoTest.groovy => CmdModuleInfoTest.groovy} (96%) rename modules/nextflow/src/test/groovy/nextflow/cli/module/{ModuleInstallTest.groovy => CmdModuleInstallTest.groovy} (96%) rename modules/nextflow/src/test/groovy/nextflow/cli/module/{ModuleListTest.groovy => CmdModuleListTest.groovy} (94%) rename modules/nextflow/src/test/groovy/nextflow/cli/module/{ModulePublishTest.groovy => CmdModulePublishTest.groovy} (93%) rename modules/nextflow/src/test/groovy/nextflow/cli/module/{ModuleRemoveTest.groovy => CmdModuleRemoveTest.groovy} (94%) rename modules/nextflow/src/test/groovy/nextflow/cli/module/{ModuleRunTest.groovy => CmdModuleRunTest.groovy} (97%) rename modules/nextflow/src/test/groovy/nextflow/cli/module/{ModuleSearchTest.groovy => CmdModuleSearchTest.groovy} (95%) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdModule.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdModule.groovy index e5312f7d56..241c8e631f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdModule.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdModule.groovy @@ -22,13 +22,13 @@ import com.beust.jcommander.ParameterException import com.beust.jcommander.Parameters import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import nextflow.cli.module.ModuleInfo -import nextflow.cli.module.ModuleInstall -import nextflow.cli.module.ModuleList -import nextflow.cli.module.ModulePublish -import nextflow.cli.module.ModuleRemove -import nextflow.cli.module.ModuleRun -import nextflow.cli.module.ModuleSearch +import nextflow.cli.module.CmdModuleInfo +import nextflow.cli.module.CmdModuleInstall +import nextflow.cli.module.CmdModuleList +import nextflow.cli.module.CmdModulePublish +import nextflow.cli.module.CmdModuleRemove +import nextflow.cli.module.CmdModuleRun +import nextflow.cli.module.CmdModuleSearch import nextflow.exception.AbortOperationException /** @@ -48,13 +48,13 @@ class CmdModule extends CmdBase implements UsageAware { static final List commands = new ArrayList<>() static { - commands << new ModuleInstall() - commands << new ModuleRun() - commands << new ModuleList() - commands << new ModuleRemove() - commands << new ModuleSearch() - commands << new ModuleInfo() - commands << new ModulePublish() + commands << new CmdModuleInstall() + commands << new CmdModuleRun() + commands << new CmdModuleList() + commands << new CmdModuleRemove() + commands << new CmdModuleSearch() + commands << new CmdModuleInfo() + commands << new CmdModulePublish() } protected JCommander commander() { diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInfo.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInfo.groovy similarity index 98% rename from modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInfo.groovy rename to modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInfo.groovy index 1c9582e231..2c6da59dba 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInfo.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInfo.groovy @@ -30,11 +30,8 @@ import nextflow.cli.CmdBase import nextflow.config.ConfigBuilder import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException -import nextflow.module.InstalledModule import nextflow.module.ModuleReference import nextflow.module.ModuleRegistryClient -import nextflow.module.ModuleSpec -import nextflow.module.ModuleStorage import nextflow.util.TestOnly import java.nio.file.Path @@ -48,7 +45,7 @@ import java.nio.file.Paths @Slf4j @CompileStatic @Parameters(commandDescription = "Show module information and usage template") -class ModuleInfo extends CmdBase { +class CmdModuleInfo extends CmdBase { @Parameter(names = ["-version"], description = "Module version") String version @@ -311,4 +308,4 @@ class ModuleInfo extends CmdBase { println JsonOutput.prettyPrint(JsonOutput.toJson(info)) } -} \ No newline at end of file +} diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy similarity index 98% rename from modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy rename to modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy index 2a37281297..4b7f69eeff 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleInstall.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy @@ -42,7 +42,7 @@ import java.nio.file.Paths @Slf4j @Parameters(commandDescription = "Install a module from the registry") @CompileStatic -class ModuleInstall extends CmdBase { +class CmdModuleInstall extends CmdBase { @Parameter(names = ["-version"], description = "Module version") String version diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleList.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy similarity index 98% rename from modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleList.groovy rename to modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy index 2f40d7a32e..6f08c7c8d9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleList.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy @@ -39,7 +39,7 @@ import java.nio.file.Paths @Slf4j @CompileStatic @Parameters(commandDescription = "List all installed modules") -class ModuleList extends CmdBase { +class CmdModuleList extends CmdBase { @Parameter(names = ["-json"], description = "Output in JSON format", arity=0) boolean jsonOutput = false diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy similarity index 99% rename from modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy rename to modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy index e3bc9cd77b..b385a11aba 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModulePublish.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy @@ -43,7 +43,7 @@ import java.nio.file.Paths @Slf4j @CompileStatic @Parameters(commandDescription = "Publish a module to the registry") -class ModulePublish extends CmdBase { +class CmdModulePublish extends CmdBase { @Parameter(names = ["-dry-run"], description = "Validate without uploading", arity=0) boolean dryRun = false diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRemove.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy similarity index 99% rename from modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRemove.groovy rename to modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy index ee207bb3ad..3c316cd281 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRemove.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy @@ -38,7 +38,7 @@ import java.nio.file.Paths @Slf4j @CompileStatic @Parameters(commandDescription = "Remove an installed module") -class ModuleRemove extends CmdBase { +class CmdModuleRemove extends CmdBase { @Parameter(description = "", required = true) List args diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy similarity index 98% rename from modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy rename to modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy index 76f179b2c2..e785c03bfa 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy @@ -40,7 +40,7 @@ import java.nio.file.Paths */ @CompileStatic @Parameters(commandDescription = "Run a module directly from the registry") -class ModuleRun extends CmdRun { +class CmdModuleRun extends CmdRun { @Parameter(names = ["-version"], description = "Module version") String version diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleSearch.groovy similarity index 99% rename from modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy rename to modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleSearch.groovy index e424bd466f..799989ee85 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/ModuleSearch.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleSearch.groovy @@ -40,7 +40,7 @@ import java.nio.file.Paths @Slf4j @CompileStatic @Parameters(commandDescription = "Search for modules in the registry") -class ModuleSearch extends CmdBase { +class CmdModuleSearch extends CmdBase { @Parameter(names = ["-limit"], description = "Maximum number of results") int limit = 20 diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInfoTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInfoTest.groovy similarity index 96% rename from modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInfoTest.groovy rename to modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInfoTest.groovy index 1c409a6efc..c2a6e9aae8 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInfoTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInfoTest.groovy @@ -34,11 +34,11 @@ import test.OutputCapture import java.nio.file.Path /** - * Tests for ModuleInfo command + * Tests for CmdModuleInfo command * * @author Jorge Ejarque */ -class ModuleInfoTest extends Specification { +class CmdModuleInfoTest extends Specification { @Rule OutputCapture capture = new OutputCapture() @@ -62,7 +62,7 @@ class ModuleInfoTest extends Specification { ) and: - def cmd = new ModuleInfo() + def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] cmd.launcher = Mock(Launcher) { getOptions() >> null @@ -110,7 +110,7 @@ class ModuleInfoTest extends Specification { ) and: - def cmd = new ModuleInfo() + def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] cmd.version = '0.9.0' cmd.launcher = Mock(Launcher) { @@ -149,7 +149,7 @@ class ModuleInfoTest extends Specification { ) and: - def cmd = new ModuleInfo() + def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] cmd.jsonOutput = true cmd.launcher = Mock(Launcher) { @@ -207,7 +207,7 @@ class ModuleInfoTest extends Specification { ) and: - def cmd = new ModuleInfo() + def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] cmd.launcher = Mock(Launcher) { getOptions() >> null @@ -266,7 +266,7 @@ class ModuleInfoTest extends Specification { ) and: - def cmd = new ModuleInfo() + def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] cmd.launcher = Mock(Launcher) { getOptions() >> null @@ -324,7 +324,7 @@ class ModuleInfoTest extends Specification { ) and: - def cmd = new ModuleInfo() + def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] cmd.launcher = Mock(Launcher) { getOptions() >> null @@ -386,7 +386,7 @@ class ModuleInfoTest extends Specification { ) and: - def cmd = new ModuleInfo() + def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] cmd.launcher = Mock(Launcher) { getOptions() >> null @@ -427,7 +427,7 @@ class ModuleInfoTest extends Specification { ) and: - def cmd = new ModuleInfo() + def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] cmd.version = '2.0.0' cmd.launcher = Mock(Launcher) { @@ -477,7 +477,7 @@ class ModuleInfoTest extends Specification { ) and: - def cmd = new ModuleInfo() + def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] cmd.launcher = Mock(Launcher) { getOptions() >> null @@ -504,7 +504,7 @@ class ModuleInfoTest extends Specification { def 'should fail with no arguments'() { given: - def cmd = new ModuleInfo() + def cmd = new CmdModuleInfo() cmd.launcher = Mock(Launcher) { getOptions() >> null } @@ -520,7 +520,7 @@ class ModuleInfoTest extends Specification { def 'should fail with multiple arguments'() { given: - def cmd = new ModuleInfo() + def cmd = new CmdModuleInfo() cmd.launcher = Mock(Launcher) { getOptions() >> null } @@ -550,7 +550,7 @@ class ModuleInfoTest extends Specification { ) and: - def cmd = new ModuleInfo() + def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] cmd.launcher = Mock(Launcher) { getOptions() >> null @@ -593,7 +593,7 @@ class ModuleInfoTest extends Specification { ) and: - def cmd = new ModuleInfo() + def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] cmd.launcher = Mock(Launcher) { getOptions() >> null @@ -649,7 +649,7 @@ class ModuleInfoTest extends Specification { ) and: - def cmd = new ModuleInfo() + def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] cmd.launcher = Mock(Launcher) { getOptions() >> null @@ -701,7 +701,7 @@ class ModuleInfoTest extends Specification { ) and: - def cmd = new ModuleInfo() + def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] cmd.jsonOutput = true cmd.launcher = Mock(Launcher) { @@ -776,7 +776,7 @@ class ModuleInfoTest extends Specification { ) and: - def cmd = new ModuleInfo() + def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] cmd.jsonOutput = true cmd.launcher = Mock(Launcher) { @@ -817,4 +817,4 @@ class ModuleInfoTest extends Specification { json.output.html.items[0].description == 'HTML report' json.output.html.items[0].pattern == '*.html' } -} \ No newline at end of file +} diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInstallTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInstallTest.groovy similarity index 96% rename from modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInstallTest.groovy rename to modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInstallTest.groovy index 11761d719a..c04dad81bb 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleInstallTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInstallTest.groovy @@ -34,11 +34,11 @@ import java.nio.file.Path import java.util.zip.GZIPOutputStream /** - * Tests for ModuleInstall command + * Tests for CmdModuleInstall command * * @author Jorge Ejarque */ -class ModuleInstallTest extends Specification { +class CmdModuleInstallTest extends Specification { @Rule OutputCapture capture = new OutputCapture() @@ -48,7 +48,7 @@ class ModuleInstallTest extends Specification { def 'should install module with latest version'() { given: - def cmd = new ModuleInstall() + def cmd = new CmdModuleInstall() cmd.launcher = Mock(Launcher) { getOptions() >> null } @@ -93,7 +93,7 @@ class ModuleInstallTest extends Specification { def 'should install module with specific version'() { given: - def cmd = new ModuleInstall() + def cmd = new CmdModuleInstall() cmd.launcher = Mock(Launcher) { getOptions() >> null } @@ -142,7 +142,7 @@ class ModuleInstallTest extends Specification { spec.addModuleEntry('@nf-core/fastqc', '1.0.0') and: - def cmd = new ModuleInstall() + def cmd = new CmdModuleInstall() cmd.launcher = Mock(Launcher) { getOptions() >> null } @@ -193,7 +193,7 @@ class ModuleInstallTest extends Specification { spec.addModuleEntry('@nf-core/fastqc', '1.0.0') and: - def cmd = new ModuleInstall() + def cmd = new CmdModuleInstall() cmd.launcher = Mock(Launcher) { getOptions() >> null } @@ -218,7 +218,7 @@ class ModuleInstallTest extends Specification { def 'should handle module with scope in name'() { given: - def cmd = new ModuleInstall() + def cmd = new CmdModuleInstall() cmd.launcher = Mock(Launcher) { getOptions() >> null } @@ -254,7 +254,7 @@ class ModuleInstallTest extends Specification { def 'should create modules directory if it does not exist'() { given: - def cmd = new ModuleInstall() + def cmd = new CmdModuleInstall() cmd.launcher = Mock(Launcher) { getOptions() >> null } @@ -286,7 +286,7 @@ class ModuleInstallTest extends Specification { def 'should create checksum file after installation'() { given: - def cmd = new ModuleInstall() + def cmd = new CmdModuleInstall() cmd.launcher = Mock(Launcher) { getOptions() >> null } @@ -322,7 +322,7 @@ class ModuleInstallTest extends Specification { def 'should fail with no arguments'() { given: - def cmd = new ModuleInstall() + def cmd = new CmdModuleInstall() cmd.launcher = Mock(Launcher) { getOptions() >> null } @@ -338,7 +338,7 @@ class ModuleInstallTest extends Specification { def 'should fail with too many arguments'() { given: - def cmd = new ModuleInstall() + def cmd = new CmdModuleInstall() cmd.launcher = Mock(Launcher) { getOptions() >> null } @@ -354,7 +354,7 @@ class ModuleInstallTest extends Specification { def 'should fail with invalid module reference'() { given: - def cmd = new ModuleInstall() + def cmd = new CmdModuleInstall() cmd.launcher = Mock(Launcher) { getOptions() >> null } diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleListTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy similarity index 94% rename from modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleListTest.groovy rename to modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy index 0ed761c20c..4f985ecf88 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleListTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy @@ -29,11 +29,11 @@ import java.nio.file.Files import java.nio.file.Path /** - * Tests for ModuleList command + * Tests for CmdModuleList command * * @author Jorge Ejarque */ -class ModuleListTest extends Specification { +class CmdModuleListTest extends Specification { @Rule OutputCapture capture = new OutputCapture() @@ -52,7 +52,7 @@ class ModuleListTest extends Specification { createTestModule(storage, 'nf-core', 'multiqc', '2.1.0') and: - def cmd = new ModuleList() + def cmd = new CmdModuleList() cmd.root = tempDir when: @@ -76,7 +76,7 @@ class ModuleListTest extends Specification { createTestModule(storage, 'nf-core', 'fastqc', '1.5.0') and: - def cmd = new ModuleList() + def cmd = new CmdModuleList() cmd.jsonOutput = true cmd.root = tempDir @@ -95,7 +95,7 @@ class ModuleListTest extends Specification { def 'should handle no installed modules'() { given: - def cmd = new ModuleList() + def cmd = new CmdModuleList() cmd.root = tempDir // Use test directory with no modules when: @@ -115,7 +115,7 @@ class ModuleListTest extends Specification { moduleDir.resolve('main.nf').text = 'process MODIFIED { }' and: - def cmd = new ModuleList() + def cmd = new CmdModuleList() cmd.root = tempDir when: @@ -137,7 +137,7 @@ class ModuleListTest extends Specification { createTestModule(storage, 'myorg', 'custom', '2.0.0') and: - def cmd = new ModuleList() + def cmd = new CmdModuleList() cmd.root = tempDir when: diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModulePublishTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModulePublishTest.groovy similarity index 93% rename from modules/nextflow/src/test/groovy/nextflow/cli/module/ModulePublishTest.groovy rename to modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModulePublishTest.groovy index 7cf6b41a7e..472f0593ef 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModulePublishTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModulePublishTest.groovy @@ -24,11 +24,11 @@ import java.nio.file.Files import java.nio.file.Path /** - * Tests for ModulePublish command + * Tests for CmdModulePublish command * * @author Jorge Ejarque */ -class ModulePublishTest extends Specification { +class CmdModulePublishTest extends Specification { @TempDir Path tempDir @@ -49,7 +49,7 @@ license: MIT ''' and: - def cmd = new ModulePublish() + def cmd = new CmdModulePublish() cmd.dryRun = true cmd.args = [moduleDir.toString()] @@ -69,7 +69,7 @@ license: MIT moduleDir.resolve('main.nf').text = 'process TEST { }' and: - def cmd = new ModulePublish() + def cmd = new CmdModulePublish() when: def errors = cmd.invokeMethod('validateModuleStructure', moduleDir) @@ -101,7 +101,7 @@ license: MIT Files.writeString(largeFile, content) and: - def cmd = new ModulePublish() + def cmd = new CmdModulePublish() when: def errors = cmd.invokeMethod('validateModuleStructure', moduleDir) @@ -127,7 +127,7 @@ license: MIT ''' and: - def cmd = new ModulePublish() + def cmd = new CmdModulePublish() def launcher = new Launcher() launcher.options = [:] cmd.launcher = launcher diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleRemoveTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRemoveTest.groovy similarity index 94% rename from modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleRemoveTest.groovy rename to modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRemoveTest.groovy index 3266b33106..0361b3a274 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleRemoveTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRemoveTest.groovy @@ -29,11 +29,11 @@ import java.nio.file.Files import java.nio.file.Path /** - * Tests for ModuleRemove command + * Tests for CmdModuleRemove command * * @author Jorge Ejarque */ -class ModuleRemoveTest extends Specification { +class CmdModuleRemoveTest extends Specification { @Rule OutputCapture capture = new OutputCapture() @@ -54,7 +54,7 @@ class ModuleRemoveTest extends Specification { specFile.addModuleEntry('@nf-core/fastqc', '1.0.0') and: - def cmd = new ModuleRemove() + def cmd = new CmdModuleRemove() cmd.args = ['nf-core/fastqc'] cmd.root = tempDir @@ -85,7 +85,7 @@ class ModuleRemoveTest extends Specification { specFile.addModuleEntry('@nf-core/fastqc', '1.0.0') and: - def cmd = new ModuleRemove() + def cmd = new CmdModuleRemove() cmd.args = ['nf-core/fastqc'] cmd.keepConfig = true cmd.root = tempDir @@ -115,7 +115,7 @@ class ModuleRemoveTest extends Specification { specFile.addModuleEntry('@nf-core/fastqc', '1.0.0') and: - def cmd = new ModuleRemove() + def cmd = new CmdModuleRemove() cmd.args = ['nf-core/fastqc'] cmd.keepFiles = true cmd.root = tempDir @@ -137,7 +137,7 @@ class ModuleRemoveTest extends Specification { def 'should fail when both keep flags are set'() { given: - def cmd = new ModuleRemove() + def cmd = new CmdModuleRemove() cmd.args = ['nf-core/fastqc'] cmd.keepConfig = true cmd.keepFiles = true @@ -153,7 +153,7 @@ class ModuleRemoveTest extends Specification { def 'should handle removing non-existent module'() { given: - def cmd = new ModuleRemove() + def cmd = new CmdModuleRemove() cmd.args = ['nf-core/nonexistent'] cmd.root = tempDir @@ -167,7 +167,7 @@ class ModuleRemoveTest extends Specification { def 'should fail with no arguments'() { given: - def cmd = new ModuleRemove() + def cmd = new CmdModuleRemove() cmd.args = [] cmd.root = tempDir @@ -180,7 +180,7 @@ class ModuleRemoveTest extends Specification { def 'should fail with too many arguments'() { given: - def cmd = new ModuleRemove() + def cmd = new CmdModuleRemove() cmd.args = ['nf-core/fastqc', 'extra-arg'] cmd.root = tempDir diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleRunTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRunTest.groovy similarity index 97% rename from modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleRunTest.groovy rename to modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRunTest.groovy index cebbc81518..c3741d0fe1 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleRunTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRunTest.groovy @@ -36,11 +36,11 @@ import java.nio.file.Path import java.util.zip.GZIPOutputStream /** - * Tests for ModuleRun command + * Tests for CmdModuleRun command * * @author Jorge Ejarque */ -class ModuleRunTest extends Specification { +class CmdModuleRunTest extends Specification { @Rule OutputCapture capture = new OutputCapture() @@ -98,7 +98,7 @@ class ModuleRunTest extends Specification { } and: - def cmd = new ModuleRun() + def cmd = new CmdModuleRun() cmd.launcher = Mock(Launcher) { getOptions() >> new CliOptions() getCliString() >> "nextflow module run nf-core/test-module" @@ -158,7 +158,7 @@ class ModuleRunTest extends Specification { } and: - def cmd = new ModuleRun() + def cmd = new CmdModuleRun() cmd.launcher = Mock(Launcher) { getOptions() >> new CliOptions() getCliString() >> "nextflow module run nf-core/test-module" @@ -186,7 +186,7 @@ class ModuleRunTest extends Specification { def 'should fail with no arguments'() { given: - def cmd = new ModuleRun() + def cmd = new CmdModuleRun() cmd.launcher = Mock(Launcher) { getOptions() >> null } @@ -202,7 +202,7 @@ class ModuleRunTest extends Specification { def 'should fail with invalid module reference'() { given: - def cmd = new ModuleRun() + def cmd = new CmdModuleRun() cmd.launcher = Mock(Launcher) { getOptions() >> null } diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleSearchTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleSearchTest.groovy similarity index 95% rename from modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleSearchTest.groovy rename to modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleSearchTest.groovy index e62be758c9..2790cb2b3f 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/ModuleSearchTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleSearchTest.groovy @@ -28,11 +28,11 @@ import test.OutputCapture /** - * Tests for ModuleSearch command + * Tests for CmdModuleSearch command * * @author Jorge Ejarque */ -class ModuleSearchTest extends Specification { +class CmdModuleSearchTest extends Specification { @Rule OutputCapture capture = new OutputCapture() @@ -61,7 +61,7 @@ class ModuleSearchTest extends Specification { ) and: - def cmd = new ModuleSearch() + def cmd = new CmdModuleSearch() cmd.args = ['quality'] cmd.launcher = Mock(Launcher){ getOptions() >> null @@ -107,7 +107,7 @@ class ModuleSearchTest extends Specification { ) and: - def cmd = new ModuleSearch() + def cmd = new CmdModuleSearch() cmd.launcher = Mock(Launcher){ getOptions() >> null } @@ -141,7 +141,7 @@ class ModuleSearchTest extends Specification { def 'should handle no search results'() { given: - def cmd = new ModuleSearch() + def cmd = new CmdModuleSearch() cmd.launcher = Mock(Launcher){ getOptions() >> null } @@ -169,7 +169,7 @@ class ModuleSearchTest extends Specification { def 'should fail with no arguments'() { given: - def cmd = new ModuleSearch() + def cmd = new CmdModuleSearch() cmd.launcher = Mock(Launcher){ getOptions() >> null } @@ -196,7 +196,7 @@ class ModuleSearchTest extends Specification { } and: - def cmd = new ModuleSearch() + def cmd = new CmdModuleSearch() cmd.launcher = Mock(Launcher){ getOptions() >> null } From 90dc444be6063d6f56f233dce22238aba9371daa Mon Sep 17 00:00:00 2001 From: jorgee Date: Thu, 5 Mar 2026 14:26:17 +0100 Subject: [PATCH 17/23] address review comments Signed-off-by: jorgee --- docs/cli.md | 12 +-- docs/reference/cli.md | 24 ++--- .../nextflow/cli/module/CmdModuleInfo.groovy | 54 ++++++++-- .../cli/module/CmdModuleInstall.groovy | 12 +-- .../nextflow/cli/module/CmdModuleList.groovy | 42 ++++++-- .../cli/module/CmdModulePublish.groovy | 4 +- .../cli/module/CmdModuleRemove.groovy | 18 ++-- .../nextflow/cli/module/CmdModuleRun.groovy | 10 +- .../cli/module/CmdModuleSearch.groovy | 57 ++++++---- .../nextflow/module/InstalledModule.groovy | 8 +- .../nextflow/module/ModuleChecksum.groovy | 49 +++++---- .../nextflow/module/ModuleReference.groovy | 8 +- .../module/ModuleRegistryClient.groovy | 101 +++++++++--------- .../nextflow/module/ModuleResolver.groovy | 38 +++---- .../groovy/nextflow/module/ModuleSpec.groovy | 16 +-- .../nextflow/module/ModuleStorage.groovy | 76 +++++++------ .../nextflow/pipeline/PipelineSpec.groovy | 3 +- .../cli/module/CmdModuleInfoTest.groovy | 6 +- .../cli/module/CmdModuleListTest.groovy | 2 +- .../cli/module/CmdModuleSearchTest.groovy | 2 +- .../module/ModuleRegistryClientTest.groovy | 4 +- 21 files changed, 317 insertions(+), 229 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 4922b0e463..0bc6655a9e 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -284,7 +284,7 @@ $ nextflow module install nf-core/fastqc $ nextflow module install nf-core/fastqc -version 1.0.0 ``` -After installation, module will be available in `modules/@nf-core/fastqc` and included in `nextflow_spec.json` +After installation, module will be available in `modules/@nf-core/fastqc` and included in `nextflow_spec.json` Use the `-force` flag to reinstall a module even if local modifications exist. @@ -321,7 +321,7 @@ Use this to review installed modules, check module versions, or detect local mod ```console $ nextflow module list -$ nextflow module list -json +$ nextflow module list -output json ``` The output shows each module's name, installed version, and whether it has been modified locally. Use `-json` for machine-readable output suitable for scripting. @@ -337,10 +337,10 @@ Use this to find modules for specific tasks, explore available tools, or discove ```console $ nextflow module search alignment $ nextflow module search "quality control" -limit 10 -$ nextflow module search bwa -json +$ nextflow module search bwa -output json ``` -Results include module names, versions, descriptions, and download statistics. Use `-limit` to control the number of results and `-json` for programmatic access. +Results include module names, versions, descriptions, and download statistics. Use `-limit` to control the number of results and `-output json` for programmatic access. See {ref}`cli-module-search` for more information. @@ -353,7 +353,7 @@ Use this to understand module requirements, view input/output specifications, se ```console $ nextflow module info nf-core/fastqc $ nextflow module info nf-core/fastqc -version 1.0.0 -$ nextflow module info nf-core/fastqc -json +$ nextflow module info nf-core/fastqc -output json ``` The output includes the module's version, description, authors, keywords, tools, input/output channels, and a generated usage template showing how to run the module. Use `-json` for machine-readable output suitable for programmatic access. @@ -513,7 +513,7 @@ Use this to understand input/output relationships between tasks, trace data flow $ nextflow lineage ``` -See {ref}`data-lineage-page` to get started and {ref}`cli-lineage` for more information. +See {ref}`data-lineage-page` to get started and {ref}`cli-lineage` for more information. ## Seqera Platform diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 0ff72efcf3..aad0ecceaf 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1209,8 +1209,8 @@ The `module` command provides a comprehensive system for managing reusable, regi : Shows module names, versions, and integrity status (whether they've been modified locally). : The following options are available: - `-json` - : Output results in JSON format for programmatic processing. + `-o, -output` (`table`) + : Output mode for list results. Options: `table` (default), `json`. : **Examples:** @@ -1219,7 +1219,7 @@ The `module` command provides a comprehensive system for managing reusable, regi $ nextflow module list # Output as JSON - $ nextflow module list -json + $ nextflow module list -output 'json' ``` (cli-module-search)= @@ -1233,8 +1233,8 @@ The `module` command provides a comprehensive system for managing reusable, regi `-limit` : Maximum number of results to return (default: varies by registry). - `-json` - : Output results in JSON format for programmatic processing. + `-o, -output` (`simple`) + : Output mode for search results. Options: `simple` (default), `json`. : **Examples:** @@ -1246,7 +1246,7 @@ The `module` command provides a comprehensive system for managing reusable, regi $ nextflow module search "quality control" -limit 10 # Get results as JSON - $ nextflow module search bwa -json + $ nextflow module search bwa -output json ``` (cli-module-info)= @@ -1260,8 +1260,8 @@ The `module` command provides a comprehensive system for managing reusable, regi `-version` : Specify the module version to query (e.g., `1.0.0`). If not specified, displays information for the latest version. - `-json` - : Output results in JSON format for programmatic processing. + `-o, -output` (`text`) + : Output mode for info results. Options: `text` (default), `json`. : **Examples:** @@ -1273,7 +1273,7 @@ The `module` command provides a comprehensive system for managing reusable, regi $ nextflow module info nf-core/fastqc -version 1.0.0 # Get results as JSON - $ nextflow module info nf-core/fastqc -json + $ nextflow module info nf-core/fastqc -output json ``` (cli-module-remove)= @@ -1315,7 +1315,7 @@ The `module` command provides a comprehensive system for managing reusable, regi `-dry-run` : Validate the module structure and metadata without uploading to the registry. Useful for testing before publishing. - + `-registry` : Specify the registry to publish the module (default: `https://registry.nextflow.io`) @@ -1328,7 +1328,7 @@ The `module` command provides a comprehensive system for managing reusable, regi # Publish to nextflow registry $ export NXF_REGISTRY_TOKEN=your-token $ nextflow module publish myorg/my-module - + # Publish to a custom registry $ export NXF_REGISTRY_TOKEN=your-token $ nextflow module publish myorg/my-module -registry 'https://custom.registry.com' @@ -1380,7 +1380,7 @@ The `pull` command downloads a pipeline from a Git-hosting platform into the glo : Update all downloaded projects. `-d, -deep` -: :::{deprecated} 25.12.0-edge. +: :::{deprecated} 25.12.0-edge. Ignored for new multi-revision asset management strategy. Still used in legacy assets. ::: : Create a shallow clone of the specified depth. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInfo.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInfo.groovy index 2c6da59dba..56bc69708e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInfo.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInfo.groovy @@ -16,7 +16,9 @@ package nextflow.cli.module +import com.beust.jcommander.IParameterValidator import com.beust.jcommander.Parameter +import com.beust.jcommander.ParameterException import com.beust.jcommander.Parameters import groovy.json.JsonOutput import groovy.transform.CompileStatic @@ -50,8 +52,23 @@ class CmdModuleInfo extends CmdBase { @Parameter(names = ["-version"], description = "Module version") String version - @Parameter(names = ["-json"], description = "Output in JSON format", arity = 0) - boolean jsonOutput = false + @Parameter( + names = ['-o', '-output'], + description = 'Output mode for reporting search results: text, json', + validateWith = OutputModeValidator + ) + String output = 'text' + + static class OutputModeValidator implements IParameterValidator { + + private static final List MODES = List.of('text', 'json') + + @Override + void validate(String name, String value) { + if( !MODES.contains(value) ) + throw new ParameterException("Output mode must be one of $MODES (found: $value)") + } + } @Parameter(description = "[scope/name]", required = true) List args @@ -99,21 +116,28 @@ class CmdModuleInfo extends CmdBase { } catch( Exception e ) { log.warn "Failed to fetch metadata from registry: ${e.message}" } - if( release?.metadata ) { - log.info("No metadata found for $reference.nameWithoutPrefix ${release?.version ? "($release.version)" : ''}") + if( !release ) { + throw new AbortOperationException("No release information available for ${reference.nameWithoutPrefix}") + } + if( !release.metadata ) { + log.info("No metadata found for $reference.nameWithoutPrefix ${release.version ? "($release.version)" : ''}") } - if( jsonOutput ) { - printJsonInfo(reference, release) + def moduleUrl = buildModuleUrl(registryConfig.url, reference, release.version) + if( !output || output == 'text' ) { + printFormattedInfo(reference, release, moduleUrl) + } else if( output == 'json' ) { + printJsonInfo(reference, release, moduleUrl) } else { - printFormattedInfo(reference, release) + throw new AbortOperationException("Not implemented output mode $output)") } } - private void printFormattedInfo(ModuleReference reference, ModuleRelease release) { + private void printFormattedInfo(ModuleReference reference, ModuleRelease release, String moduleUrl) { ModuleMetadata metadata = release.metadata println "" println "Module: ${reference.nameWithoutPrefix}" println "Version: ${release.version}" + println "URL: ${moduleUrl}" println "Description: ${metadata.description ?: release.description ?: 'N/A'}" if( metadata.authors ) { @@ -202,8 +226,8 @@ class CmdModuleInfo extends CmdBase { inputs.each { input -> input.items.each { ModuleChannelItem item -> template.add(reference.scope == 'nf-core' - ? inferNfCoreParam(item.name, item.type) - : inferNormalParam(item.name, item.type)) + ? inferNfCoreParam(item.name, item.type) + : inferNormalParam(item.name, item.type)) } } @@ -247,12 +271,20 @@ class CmdModuleInfo extends CmdBase { return "--${paramName} <${paramPlaceholder}>" } - private void printJsonInfo(ModuleReference reference, ModuleRelease release) { + private static String buildModuleUrl(String registryUrl, ModuleReference reference, String version) { + // Strip /api suffix to get the base UI URL + def baseUrl = registryUrl.endsWith('/api') ? registryUrl[0..-5] : registryUrl + def encodedName = URLEncoder.encode(reference.name, 'UTF-8') + return "${baseUrl}/admin/modules/${reference.scope}/${encodedName}@${version}" + } + + private void printJsonInfo(ModuleReference reference, ModuleRelease release, String moduleUrl) { def metadata = release?.metadata def info = [ name : reference.nameWithoutPrefix, fullName : reference.fullName, version : release.version, + url : moduleUrl, description: metadata.description ?: release.description, authors : metadata.authors, keywords : metadata.keywords, diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy index 4b7f69eeff..f509aef73e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy @@ -66,7 +66,7 @@ class CmdModuleInstall extends CmdBase { @Override void run() { - if (!args || args.size() != 1) { + if( !args || args.size() != 1 ) { throw new AbortOperationException("Incorrect number of arguments") } @@ -77,9 +77,9 @@ class CmdModuleInstall extends CmdBase { // Get config def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize() def config = new ConfigBuilder() - .setOptions(launcher.options) - .setBaseDir(baseDir) - .build() + .setOptions(launcher.options) + .setBaseDir(baseDir) + .build() final registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig() // Get modules versions from nextflow_spec.json. @@ -98,10 +98,10 @@ class CmdModuleInstall extends CmdBase { println "Module ${reference.nameWithoutPrefix}@${installedVersion} installed and configured successfully" } - catch (AbortOperationException e) { + catch( AbortOperationException e ) { throw e } - catch (Exception e) { + catch( Exception e ) { throw new AbortOperationException("Installation failed: ${e.message}", e) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy index 6f08c7c8d9..b4919b0c68 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy @@ -16,7 +16,9 @@ package nextflow.cli.module +import com.beust.jcommander.IParameterValidator import com.beust.jcommander.Parameter +import com.beust.jcommander.ParameterException import com.beust.jcommander.Parameters import groovy.json.JsonOutput import groovy.transform.CompileStatic @@ -41,8 +43,23 @@ import java.nio.file.Paths @Parameters(commandDescription = "List all installed modules") class CmdModuleList extends CmdBase { - @Parameter(names = ["-json"], description = "Output in JSON format", arity=0) - boolean jsonOutput = false + @Parameter( + names = ['-o', '-output'], + description = 'Output mode for reporting search results: table, json', + validateWith = OutputModeValidator + ) + String output = 'table' + + static class OutputModeValidator implements IParameterValidator { + + private static final List MODES = List.of('table', 'json') + + @Override + void validate(String name, String value) { + if( !MODES.contains(value) ) + throw new ParameterException("Output mode must be one of $MODES (found: $value)") + } + } @TestOnly protected Path root @@ -65,18 +82,21 @@ class CmdModuleList extends CmdBase { try { def installed = storage.listInstalled() - if (installed.isEmpty()) { + if( installed.isEmpty() ) { println "No modules installed" return } - if (jsonOutput) { + if( !output || output == 'table' ) { + printFormattedList(installed) + } else if( output == 'json' ) { printJsonList(installed) } else { - printFormattedList(installed) + throw new AbortOperationException("Not implemented output mode $output)") } + } - catch (Exception e) { + catch( Exception e ) { log.error("Failed to list modules", e) throw new AbortOperationException("List failed: ${e.message}", e) } @@ -87,7 +107,7 @@ class CmdModuleList extends CmdBase { println "Installed modules:" println "" println "Module".padRight(40) + "Version".padRight(15) + "Status" - println ("-" * 70) + println("-" * 70) installed.each { module -> def status = getStatusString(module.integrity) @@ -99,19 +119,19 @@ class CmdModuleList extends CmdBase { private void printJsonList(List installed) { def modules = installed.collect { module -> [ - name: module.reference.nameWithoutPrefix, - version: module.installedVersion ?: 'unknown', + name : module.reference.nameWithoutPrefix, + version : module.installedVersion ?: 'unknown', integrity: module.integrity.toString(), directory: module.directory.toString() ] } // Simple JSON output (could use groovy.json.JsonOutput for better formatting) - println JsonOutput.toJson(modules: modules) + println JsonOutput.prettyPrint(JsonOutput.toJson(modules: modules)) } private String getStatusString(ModuleIntegrity integrity) { - switch (integrity) { + switch( integrity ) { case ModuleIntegrity.VALID: return 'OK' case ModuleIntegrity.MODIFIED: diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy index b385a11aba..a3b46056fc 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy @@ -218,8 +218,8 @@ class CmdModulePublish extends CmdBase { } // Check bundle size (1MB uncompressed limit) - try { - long totalSize = Files.walk(moduleDir) + try (final sizeStream = Files.walk(moduleDir)){ + long totalSize = sizeStream .filter { Files.isRegularFile(it) } .mapToLong { Files.size(it) } .sum() diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy index 3c316cd281..b793974f32 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy @@ -59,12 +59,12 @@ class CmdModuleRemove extends CmdBase { @Override void run() { - if (!args || args.size() != 1) { + if( !args || args.size() != 1 ) { throw new AbortOperationException("Incorrect number of arguments") } // Validate flags - if (keepConfig && keepFiles) { + if( keepConfig && keepFiles ) { throw new AbortOperationException("Cannot use both -keep-config and -keep-files flags together") } @@ -86,10 +86,10 @@ class CmdModuleRemove extends CmdBase { def configRemoved = false // Remove local files unless -keep-files is set - if (!keepFiles) { + if( !keepFiles ) { println "Removing module files for ${reference.nameWithoutPrefix}..." filesRemoved = storage.removeModule(reference) - if (filesRemoved) { + if( filesRemoved ) { println "Module files removed successfully" } else { println "Module ${reference.nameWithoutPrefix} was not installed locally" @@ -99,10 +99,10 @@ class CmdModuleRemove extends CmdBase { } // Remove config entry unless -keep-config is set - if (!keepConfig) { + if( !keepConfig ) { println "Removing module entry from nextflow_spec.json..." configRemoved = specFile.removeModuleEntry(reference.fullName) - if (configRemoved) { + if( configRemoved ) { println "Module entry removed from configuration" } else { println "Module ${reference.nameWithoutPrefix} was not configured in nextflow_spec.json" @@ -112,16 +112,16 @@ class CmdModuleRemove extends CmdBase { } // Summary - if (filesRemoved || configRemoved) { + if( filesRemoved || configRemoved ) { println "\nModule ${reference.nameWithoutPrefix} removal completed" } else { println "\nModule ${reference.nameWithoutPrefix} was not found" } } - catch (AbortOperationException e) { + catch( AbortOperationException e ) { throw e } - catch (Exception e) { + catch( Exception e ) { log.error("Failed to remove module", e) throw new AbortOperationException("Removal failed: ${e.message}", e) } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy index e785c03bfa..4212ae0d2d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy @@ -57,7 +57,7 @@ class CmdModuleRun extends CmdRun { @Override void run() { - if (!args ) { + if( !args ) { throw new AbortOperationException("Arguments not provided") } @@ -68,16 +68,16 @@ class CmdModuleRun extends CmdRun { ModuleReference reference try { reference = ModuleReference.parse(moduleRef) - } catch (Exception e) { + } catch( Exception e ) { throw new AbortOperationException("Invalid module reference: ${moduleRef}", e) } // Get config def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize() def config = new ConfigBuilder() - .setOptions(launcher.options) - .setBaseDir(baseDir) - .build() + .setOptions(launcher.options) + .setBaseDir(baseDir) + .build() def registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig() diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleSearch.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleSearch.groovy index 799989ee85..2b55de24d8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleSearch.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleSearch.groovy @@ -16,7 +16,9 @@ package nextflow.cli.module +import com.beust.jcommander.IParameterValidator import com.beust.jcommander.Parameter +import com.beust.jcommander.ParameterException import com.beust.jcommander.Parameters import groovy.json.JsonOutput import groovy.transform.CompileStatic @@ -45,8 +47,23 @@ class CmdModuleSearch extends CmdBase { @Parameter(names = ["-limit"], description = "Maximum number of results") int limit = 20 - @Parameter(names = ["-json"], description = "Output in JSON format", arity=0) - boolean jsonOutput = false + @Parameter( + names = ['-o', '-output'], + description = 'Output mode for reporting search results: simple, json', + validateWith = OutputModeValidator + ) + String output = 'simple' + + static class OutputModeValidator implements IParameterValidator { + + private static final List MODES = List.of('simple', 'json') + + @Override + void validate(String name, String value) { + if( !MODES.contains(value) ) + throw new ParameterException("Output mode must be one of $MODES (found: $value)") + } + } @Parameter(description = "", required = true) List args @@ -61,7 +78,7 @@ class CmdModuleSearch extends CmdBase { @Override void run() { - if (!args && args.size() != 1 ) { + if( !args || args.size() != 1 ) { throw new AbortOperationException("Unexpected number of parameters") } String query = args[0] @@ -69,9 +86,9 @@ class CmdModuleSearch extends CmdBase { // Get config def baseDir = Paths.get('.').toAbsolutePath().normalize() def config = new ConfigBuilder() - .setOptions(launcher.options) - .setBaseDir(baseDir) - .build() + .setOptions(launcher.options) + .setBaseDir(baseDir) + .build() final registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig() @@ -82,21 +99,23 @@ class CmdModuleSearch extends CmdBase { println "Searching for '${query}'..." final results = client.search(query, limit) - if (!results || results.totalResults == 0 || !results.results || results.results.isEmpty()) { + if( !results || results.totalResults == 0 || !results.results || results.results.isEmpty() ) { println "No modules found" return } - if (jsonOutput) { + if( !output || output == 'simple' ) { + printFormattedResults(results) + } else if( output == 'json' ) { printJsonResults(results) } else { - printFormattedResults(results) + throw new AbortOperationException("Not implemented output mode $output)") } } - catch (AbortOperationException e) { + catch( AbortOperationException e ) { throw e } - catch (Exception e) { + catch( Exception e ) { log.error("Failed to search modules", e) throw new AbortOperationException("Search failed: ${e.message}", e) } @@ -109,7 +128,7 @@ class CmdModuleSearch extends CmdBase { response.results.each { ModuleSearchResult result -> println " ${result.name}" - if (result.description) { + if( result.description ) { println " Description: ${result.description}" } println "" @@ -119,20 +138,20 @@ class CmdModuleSearch extends CmdBase { private void printJsonResults(SearchModulesResponse response) { final modules = response.results.collect { ModuleSearchResult result -> [ - name: result.name, + name : result.name, repositoryPath: result.repositoryPath, - description: result.description, + description : result.description, relevanceScore: result.relevanceScore, - keywords: result.keywords, - tools: result.tools, - revoked: result.revoked + keywords : result.keywords, + tools : result.tools, + revoked : result.revoked ] } - println JsonOutput.toJson( + println JsonOutput.prettyPrint(JsonOutput.toJson( query: response.query, totalResults: response.totalResults, results: modules - ) + )) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy b/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy index 96e4b988a1..cb032d6c2a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy @@ -50,12 +50,12 @@ class InstalledModule { */ ModuleIntegrity getIntegrity() { // Check if main.nf exists - if (!Files.exists(mainFile) || !Files.exists(manifestFile)) { + if( !Files.exists(mainFile) || !Files.exists(manifestFile) ) { return ModuleIntegrity.CORRUPTED } // Check if checksum file exists - if (!Files.exists(checksumFile)) { + if( !Files.exists(checksumFile) ) { return ModuleIntegrity.MISSING_CHECKSUM } @@ -64,13 +64,13 @@ class InstalledModule { def actualChecksum = ModuleChecksum.compute(directory) // Compare with expected - if (actualChecksum == expectedChecksum) { + if( actualChecksum == expectedChecksum ) { return ModuleIntegrity.VALID } else { log.debug("Actual: $actualChecksum, expected: $expectedChecksum") return ModuleIntegrity.MODIFIED } - } catch (Exception e) { + } catch( Exception e ) { log.warn "Failed to compute checksum for module ${reference.nameWithoutPrefix}: ${e.message}" return ModuleIntegrity.CORRUPTED } diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy index 12822f54bf..fc795f1b60 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy @@ -42,7 +42,7 @@ class ModuleChecksum { * @return The hex-encoded SHA-256 checksum */ static String compute(Path moduleDir) { - if (!Files.exists(moduleDir) || !Files.isDirectory(moduleDir)) { + if( !Files.exists(moduleDir) || !Files.isDirectory(moduleDir) ) { throw new IllegalArgumentException("Module directory does not exist or is not a directory: ${moduleDir}") } @@ -51,27 +51,34 @@ class ModuleChecksum { // Collect all files in sorted order for consistent checksums List files = [] - Files.walk(moduleDir) - .filter { Path path -> Files.isRegularFile(path) } - .filter { Path path -> !path.fileName.toString().equals(CHECKSUM_FILE) } - .sorted() - .each { Path path -> files.add(path) } + try( final walkStream = Files.walk(moduleDir) ) { + walkStream + .filter { Path path -> Files.isRegularFile(path) } + .filter { Path path -> !path.fileName.toString().equals(CHECKSUM_FILE) } + .sorted() + .each { Path path -> files.add(path) } + } // Compute checksum over all file contents - for (Path file : files) { + byte[] buf = new byte[8192] + for( Path file : files ) { // Include relative path in checksum for directory structure integrity def relativePath = moduleDir.relativize(file).toString() digest.update(relativePath.bytes) - // Include file contents - def bytes = Files.readAllBytes(file) - digest.update(bytes) + // Include file contents via streaming to avoid loading large files into memory + Files.newInputStream(file).withCloseable { is -> + int n + while( (n = is.read(buf)) != -1 ) { + digest.update(buf, 0, n) + } + } } def hashBytes = digest.digest() return bytesToHex(hashBytes) } - catch (Exception e) { + catch( Exception e ) { log.error("Failed to compute checksum for module directory: ${moduleDir}", e) throw new RuntimeException("Failed to compute module checksum", e) } @@ -96,7 +103,7 @@ class ModuleChecksum { */ static String load(Path moduleDir) { def checksumFile = moduleDir.resolve(CHECKSUM_FILE) - if (!Files.exists(checksumFile)) { + if( !Files.exists(checksumFile) ) { return null } return checksumFile.text @@ -122,18 +129,22 @@ class ModuleChecksum { * @return The hex-encoded checksum */ static String computeFile(Path file, String type = CHECKSUM_ALGORITHM) { - if (!Files.exists(file) || !Files.isRegularFile(file)) { + if( !Files.exists(file) || !Files.isRegularFile(file) ) { throw new IllegalArgumentException("File does not exist or is not a regular file: ${file}") } try { final digest = MessageDigest.getInstance(type) - final bytes = Files.readAllBytes(file) - digest.update(bytes) - final hashBytes = digest.digest() - return bytesToHex(hashBytes) + final byte[] buf = new byte[8192] + Files.newInputStream(file).withCloseable { is -> + int n + while( (n = is.read(buf)) != -1 ) { + digest.update(buf, 0, n) + } + } + return bytesToHex(digest.digest()) } - catch (Exception e) { + catch( Exception e ) { log.error("Failed to compute checksum for file: ${file}", e) throw new RuntimeException("Failed to compute file checksum", e) } @@ -147,7 +158,7 @@ class ModuleChecksum { */ private static String bytesToHex(byte[] bytes) { def hexChars = new char[bytes.length * 2] - for (int i = 0; i < bytes.length; i++) { + for( int i = 0; i < bytes.length; i++ ) { int v = bytes[i] & 0xFF hexChars[i * 2] = Character.forDigit(v >>> 4, 16) hexChars[i * 2 + 1] = Character.forDigit(v & 0x0F, 16) diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleReference.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleReference.groovy index 76bbbcfc37..12bff1c6ac 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleReference.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleReference.groovy @@ -54,7 +54,7 @@ class ModuleReference { * @throws AbortOperationException if the format is invalid */ static ModuleReference parse(String source) { - if (!source) { + if( !source ) { throw new AbortOperationException("Module reference cannot be empty") } @@ -62,11 +62,11 @@ class ModuleReference { source = source.trim() def matcher = MODULE_NAME_PATTERN.matcher(source) - if (!matcher.matches()) { + if( !matcher.matches() ) { throw new AbortOperationException( "Invalid module reference: '${source}'. " + - "Expected format: [@]scope/name where scope is lowercase alphanumeric with dots/underscores/hyphens " + - "and name is lowercase alphanumeric with underscores/hyphens, optionally with slash-separated segments" + "Expected format: [@]scope/name where scope is lowercase alphanumeric with dots/underscores/hyphens " + + "and name is lowercase alphanumeric with underscores/hyphens, optionally with slash-separated segments" ) } diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy index d7bf1c99e6..fa1d827622 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy @@ -50,9 +50,9 @@ class ModuleRegistryClient { ModuleRegistryClient(RegistryConfig config) { this.config = config ?: new RegistryConfig() this.httpClient = HxClient.newBuilder() - .retryConfig(RetryConfig.config()) - .followRedirects(HttpClient.Redirect.NORMAL) - .build() + .retryConfig(RetryConfig.config()) + .followRedirects(HttpClient.Redirect.NORMAL) + .build() } private String encodeName(String name) { @@ -72,11 +72,11 @@ class ModuleRegistryClient { def registryUrls = config.allUrls Exception lastError = null - for (String registryUrl : registryUrls) { + for( String registryUrl : registryUrls ) { log.debug "Trying to fetch from $registryUrl" try { return fetchModuleFromRegistry(registryUrl, name) - } catch (Exception e) { + } catch( Exception e ) { log.debug "Failed to fetch module from ${registryUrl}: ${e.message}" lastError = e } @@ -102,7 +102,7 @@ class ModuleRegistryClient { // Add authentication if available log.debug "Getting auth from: ${registryUrl}" def token = config.getApiKey() - if (token) { + if( token ) { requestBuilder.header("Authorization", "Bearer ${token}") } log.debug "Building request: ${registryUrl}" @@ -115,15 +115,15 @@ class ModuleRegistryClient { log.debug "Registry request: ${response.uri()}\n- code: ${response.statusCode()}\n- body: ${body}" - if (response.statusCode() == 404) { + if( response.statusCode() == 404 ) { throw new AbortOperationException("Module not found: ${name}") } - if (response.statusCode() != 200) { + if( response.statusCode() != 200 ) { throw new AbortOperationException( "Invalid response from registry: ${uri}\n" + - "- http status: ${response.statusCode()}\n" + - "- response: ${body}" + "- http status: ${response.statusCode()}\n" + + "- response: ${body}" ) } @@ -131,11 +131,10 @@ class ModuleRegistryClient { def encoder = new GsonEncoder() {} return encoder.decode(body) } - catch (AbortOperationException e) { + catch( AbortOperationException e ) { throw e } - catch (Exception e) { - e.printStackTrace() + catch( Exception e ) { throw new AbortOperationException("Failed to fetch module from: ${uri}", e) } } @@ -151,10 +150,10 @@ class ModuleRegistryClient { def registryUrls = config.allUrls Exception lastError = null - for (String registryUrl : registryUrls) { + for( String registryUrl : registryUrls ) { try { return fetchReleaseFromRegistry(registryUrl, name, version) - } catch (Exception e) { + } catch( Exception e ) { log.debug "Failed to fetch release from ${registryUrl}: ${e.message}" lastError = e } @@ -178,7 +177,7 @@ class ModuleRegistryClient { .GET() def token = config.getApiKey() - if (token) { + if( token ) { requestBuilder.header("Authorization", "Bearer ${token}") } @@ -189,25 +188,25 @@ class ModuleRegistryClient { def response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()) def body = response.body() - if (response.statusCode() == 404) { + if( response.statusCode() == 404 ) { throw new AbortOperationException("Module version not found: ${name}@${version}") } - if (response.statusCode() != 200) { + if( response.statusCode() != 200 ) { throw new AbortOperationException( "Invalid response from registry: ${uri}\n" + - "- http status: ${response.statusCode()}\n" + - "- response: ${body}" + "- http status: ${response.statusCode()}\n" + + "- response: ${body}" ) } // Parse response using npr-api ModuleRelease model - return new GsonEncoder() {}.decode(body) + return new GsonEncoder() {}.decode(body) } - catch (AbortOperationException e) { + catch( AbortOperationException e ) { throw e } - catch (Exception e) { + catch( Exception e ) { throw new AbortOperationException("Failed to fetch module release from: ${uri}", e) } } @@ -222,14 +221,14 @@ class ModuleRegistryClient { */ Path downloadModule(String name, String version, Path targetPath) { def registryUrls = config.allUrls - if (targetPath.exists()){ + if( targetPath.exists() ) { targetPath.delete() } Exception lastError = null - for (String registryUrl : registryUrls) { + for( String registryUrl : registryUrls ) { try { return downloadModuleFromRegistry(registryUrl, name, version, targetPath) - } catch (Exception e) { + } catch( Exception e ) { log.debug "Failed to download from ${registryUrl}: ${e.message}" lastError = e } @@ -253,7 +252,7 @@ class ModuleRegistryClient { .GET() def token = config.getApiKey() - if (token) { + if( token ) { requestBuilder.header("Authorization", "Bearer ${token}") } @@ -263,19 +262,19 @@ class ModuleRegistryClient { log.debug "Downloading module from: ${uri}" def response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()) - if (response.statusCode() == 404) { + if( response.statusCode() == 404 ) { throw new AbortOperationException("Module bundle not found: ${name}@${version}") } - if (response.statusCode() != 200) { + if( response.statusCode() != 200 ) { throw new AbortOperationException( "Invalid response from registry: ${uri}\n" + - "- http status: ${response.statusCode()}" + "- http status: ${response.statusCode()}" ) } // Create parent directories if needed - if (targetPath.parent) { + if( targetPath.parent ) { Files.createDirectories(targetPath.parent) } @@ -287,10 +286,10 @@ class ModuleRegistryClient { return targetPath } - catch (AbortOperationException e) { + catch( AbortOperationException e ) { throw e } - catch (Exception e) { + catch( Exception e ) { throw new AbortOperationException("Failed to download module from: ${uri}", e) } } @@ -365,10 +364,10 @@ class ModuleRegistryClient { def registryUrls = config.allUrls Exception lastError = null - for (String registryUrl : registryUrls) { + for( String registryUrl : registryUrls ) { try { return searchInRegistry(registryUrl, query, limit) - } catch (Exception e) { + } catch( Exception e ) { log.debug "Failed to search in ${registryUrl}: ${e.message}" lastError = e } @@ -392,7 +391,7 @@ class ModuleRegistryClient { .GET() def token = config.getApiKey() - if (token) { + if( token ) { requestBuilder.header("Authorization", "Bearer ${token}") } @@ -403,11 +402,11 @@ class ModuleRegistryClient { def response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()) def body = response.body() - if (response.statusCode() != 200) { + if( response.statusCode() != 200 ) { throw new AbortOperationException( "Invalid response from registry: ${uri}\n" + - "- http status: ${response.statusCode()}\n" + - "- response: ${body}" + "- http status: ${response.statusCode()}\n" + + "- response: ${body}" ) } @@ -415,10 +414,10 @@ class ModuleRegistryClient { def encoder = new GsonEncoder() {} return encoder.decode(body) } - catch (AbortOperationException e) { + catch( AbortOperationException e ) { throw e } - catch (Exception e) { + catch( Exception e ) { throw new AbortOperationException("Failed to search modules in: ${uri}", e) } } @@ -435,7 +434,7 @@ class ModuleRegistryClient { final registryUrl = registry ?: config.url final authToken = config.apiKey - if (!authToken) { + if( !authToken ) { throw new AbortOperationException( "Authentication required to publish modules.\n" + "Please set 'NXF_REGISTRY_TOKEN' environment variable or configure 'registry.apiKey' in nextflow.config:\n\n" + @@ -455,10 +454,10 @@ class ModuleRegistryClient { * Publish module to a specific registry */ private PublishModuleResponse publishModuleToRegistry( - String registryUrl, - String name, - def request, - String authToken) { + String registryUrl, + String name, + def request, + String authToken) { String endpoint = "${registryUrl}/v1/modules/${encodeName(name)}".toString() URI uri = URI.create(endpoint) @@ -481,21 +480,21 @@ class ModuleRegistryClient { HttpResponse response = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()) String body = response.body() - if (response.statusCode() != 201) { + if( response.statusCode() != 201 ) { throw new AbortOperationException( "Failed to publish module: ${uri}\n" + - "- http status: ${response.statusCode()}\n" + - "- response: ${body}" + "- http status: ${response.statusCode()}\n" + + "- response: ${body}" ) } // Parse response using npr-api PublishModuleResponse model return new GsonEncoder() {}.decode(body) } - catch (AbortOperationException e) { + catch( AbortOperationException e ) { throw e } - catch (Exception e) { + catch( Exception e ) { throw new AbortOperationException("Failed to publish module to: ${uri}", e) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy index dc96c7a3b6..11ad0f2335 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy @@ -38,7 +38,7 @@ class ModuleResolver { private final ModuleStorage storage private final ModulesConfig modulesConfig - ModuleResolver (Path baseDir, ModuleRegistryClient registryClient, ModulesConfig modulesConfig = null) { + ModuleResolver(Path baseDir, ModuleRegistryClient registryClient, ModulesConfig modulesConfig = null) { this.registryClient = registryClient this.storage = new ModuleStorage(baseDir) this.modulesConfig = modulesConfig ?: new ModulesConfig() @@ -64,30 +64,30 @@ class ModuleResolver { // Check if module is already installed def installed = storage.getInstalledModule(reference) - if (installed) { + if( installed ) { // Check integrity def integrity = installed.integrity - if (integrity == ModuleIntegrity.CORRUPTED) { + if( integrity == ModuleIntegrity.CORRUPTED ) { throw new AbortOperationException( "Module ${reference.nameWithoutPrefix} is corrupted (missing required files). " + - "Please remove and reinstall." + "Please remove and reinstall." ) } - if (integrity == ModuleIntegrity.MODIFIED) { + if( integrity == ModuleIntegrity.MODIFIED ) { log.warn "Module ${reference.nameWithoutPrefix} has local modifications (checksum mismatch)" } // Check if version matches - if (targetVersion && installed.installedVersion != targetVersion) { - if (autoInstall) { + if( targetVersion && installed.installedVersion != targetVersion ) { + if( autoInstall ) { log.info "Upgrading module ${reference.nameWithoutPrefix} from ${installed.installedVersion} to ${targetVersion}" return installModule(reference, targetVersion) } else { throw new AbortOperationException( "Module ${reference.nameWithoutPrefix} version mismatch: " + - "installed=${installed.installedVersion}, required=${targetVersion}. " + - "Run 'nextflow module install ${reference.nameWithoutPrefix}@${targetVersion}' to update." + "installed=${installed.installedVersion}, required=${targetVersion}. " + + "Run 'nextflow module install ${reference.nameWithoutPrefix}@${targetVersion}' to update." ) } } @@ -97,20 +97,20 @@ class ModuleResolver { } // Module not installed - if (autoInstall) { + if( autoInstall ) { return installModule(reference, targetVersion) } else { throw new AbortOperationException( "Module ${reference.nameWithoutPrefix} is not installed. " + - "Run 'nextflow module install ${reference.nameWithoutPrefix}' to install." + "Run 'nextflow module install ${reference.nameWithoutPrefix}' to install." ) } } - String resolveVersion(ModuleReference reference){ + String resolveVersion(ModuleReference reference) { final version = modulesConfig.getVersion(reference.fullName) ?: registryClient.fetchModule(reference.fullName).latest?.version - if (!version) { + if( !version ) { throw new AbortOperationException("Module ${reference.nameWithoutPrefix} has no published versions") } return version @@ -125,22 +125,22 @@ class ModuleResolver { * @return Path to the installed module's main.nf file */ Path installModule(ModuleReference reference, String version = null, boolean force = false) { - if (!version) + if( !version ) version = resolveVersion(reference) // Check if already installed - if (storage.isInstalled(reference)) { + if( storage.isInstalled(reference) ) { def installed = storage.getInstalledModule(reference) - if (installed.installedVersion == version) { + if( installed.installedVersion == version ) { log.info "Module ${reference.nameWithoutPrefix}@${installed.installedVersion} is already installed (version $version)" return installed.mainFile } // No desired version, check for local modifications def integrity = installed.integrity - if (integrity == ModuleIntegrity.MODIFIED && !force) { + if( integrity == ModuleIntegrity.MODIFIED && !force ) { throw new AbortOperationException( "Module ${reference.nameWithoutPrefix} has local modifications. " + - "Use --force to override, or save your changes first." + "Use --force to override, or save your changes first." ) } } @@ -162,7 +162,7 @@ class ModuleResolver { } finally { // Clean up temporary file - if (Files.exists(tempFile)) { + if( Files.exists(tempFile) ) { Files.delete(tempFile) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpec.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpec.groovy index 613a9fe959..7b27b44abf 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpec.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleSpec.groovy @@ -48,7 +48,7 @@ class ModuleSpec { * @return ModuleSpec instance */ static ModuleSpec load(Path metaYamlPath) { - if (!Files.exists(metaYamlPath)) { + if( !Files.exists(metaYamlPath) ) { throw new AbortOperationException("Module manifest not found: ${metaYamlPath}") } @@ -67,7 +67,7 @@ class ModuleSpec { return manifest } - catch (Exception e) { + catch( Exception e ) { throw new AbortOperationException("Failed to parse module manifest: ${metaYamlPath}", e) } } @@ -80,26 +80,26 @@ class ModuleSpec { List validate() { List errors = [] - if (!name) { + if( !name ) { errors << "Missing required field: name" } - if (!version) { + if( !version ) { errors << "Missing required field: version" } - if (!description) { + if( !description ) { errors << "Missing required field: description" } - if (!license) { + if( !license ) { errors << "Missing required field: license" } // Validate version format (semantic versioning) - if (version && !version.matches(/^\d+\.\d+\.\d+(-[\w.-]+)?$/)) { + if( version && !version.matches(/^\d+\.\d+\.\d+(-[\w.-]+)?$/) ) { errors << "Invalid version format: ${version} (expected semantic versioning, e.g., 1.0.0)".toString() } // Validate name format (scope/name or scope/path/to/name for nested modules) - if (name && !name.matches(/^[a-zA-Z0-9._-]+\/[a-zA-Z0-9_-]+(?:\/[a-zA-Z0-9_-]+)*$/)) { + if( name && !name.matches(/^[a-zA-Z0-9._-]+\/[a-zA-Z0-9_-]+(?:\/[a-zA-Z0-9_-]+)*$/) ) { errors << "Invalid module name format: ${name} (expected scope/name or scope/path/to/name, e.g., nf-core/fastqc or nf-core/gfatools/gfa2fa)".toString() } diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy index e169df789f..97802c711d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy @@ -123,15 +123,17 @@ class ModuleStorage { List modules = [] // Iterate over scope directories - Files.list(modulesDir).each { Path scopeDir -> - if (!Files.isDirectory(scopeDir)) return + try( final scopeStream = Files.list(modulesDir) ){ + scopeStream.each { Path scopeDir -> + if (!Files.isDirectory(scopeDir)) return - def scopeDirName = scopeDir.fileName.toString() - // Remove @ prefix from directory name to get scope - def scope = scopeDirName.startsWith('@') ? scopeDirName.substring(1) : scopeDirName + def scopeDirName = scopeDir.fileName.toString() + // Remove @ prefix from directory name to get scope + def scope = scopeDirName.startsWith('@') ? scopeDirName.substring(1) : scopeDirName - // Recursively find all directories containing meta.yml under this scope - findModulesRecursive(scopeDir, scope, modules) + // Recursively find all directories containing meta.yml under this scope + findModulesRecursive(scopeDir, scope, modules) + } } return modules @@ -167,12 +169,13 @@ class ModuleStorage { } // Recursively search subdirectories - try { - Files.list(dir).each { Path subDir -> - if (Files.isDirectory(subDir)) { - findModulesRecursive(subDir, scope, modules) + try (final subStream = Files.list(dir) ) { + subStream.each { Path subDir -> + if (Files.isDirectory(subDir)) { + findModulesRecursive(subDir, scope, modules) + } } - } + } catch (IOException e) { log.warn "Failed to list directory ${dir}: ${e.message}" } @@ -406,31 +409,34 @@ class ModuleStorage { * @param currentPath The current path being added */ private void addToTarArchive(TarArchiveOutputStream tos, Path sourceDir, Path currentPath) { - Files.list(currentPath).each { Path path -> - // Skip .checksum file when creating bundle - if (path.fileName.toString() == ModuleChecksum.CHECKSUM_FILE) { - return - } - - def relativePath = sourceDir.relativize(path).toString() - if (Files.isDirectory(path)) { - // Add directory entry - def entry = new TarArchiveEntry(path.toFile(), "${relativePath}/") - tos.putArchiveEntry(entry) - tos.closeArchiveEntry() + try ( def tarStream = Files.list(currentPath)) { + tarStream.each { Path path -> + // Skip .checksum file when creating bundle + if (path.fileName.toString() == ModuleChecksum.CHECKSUM_FILE) { + return + } - // Recursively add directory contents - addToTarArchive(tos, sourceDir, path) - } else { - // Add file entry - def entry = new TarArchiveEntry(path.toFile(), relativePath) - entry.setSize(Files.size(path)) - tos.putArchiveEntry(entry) - - // Copy file content - Files.copy(path, tos) - tos.closeArchiveEntry() + def relativePath = sourceDir.relativize(path).toString() + + if (Files.isDirectory(path)) { + // Add directory entry + def entry = new TarArchiveEntry(path.toFile(), "${relativePath}/") + tos.putArchiveEntry(entry) + tos.closeArchiveEntry() + + // Recursively add directory contents + addToTarArchive(tos, sourceDir, path) + } else { + // Add file entry + def entry = new TarArchiveEntry(path.toFile(), relativePath) + entry.setSize(Files.size(path)) + tos.putArchiveEntry(entry) + + // Copy file content + Files.copy(path, tos) + tos.closeArchiveEntry() + } } } } diff --git a/modules/nextflow/src/main/groovy/nextflow/pipeline/PipelineSpec.groovy b/modules/nextflow/src/main/groovy/nextflow/pipeline/PipelineSpec.groovy index a333b720de..766b71f62e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/pipeline/PipelineSpec.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/pipeline/PipelineSpec.groovy @@ -89,7 +89,8 @@ class PipelineSpec { return false } final modules = spec.modules as Map - modules.remove(normalizedName) + if( modules.remove(normalizedName) == null ) + return false writeSpecFile(spec) log.info "Removed ${normalizedName} from ${SPEC_FILE_NAME}" return true diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInfoTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInfoTest.groovy index c2a6e9aae8..7668fa7cf7 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInfoTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInfoTest.groovy @@ -151,7 +151,7 @@ class CmdModuleInfoTest extends Specification { and: def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] - cmd.jsonOutput = true + cmd.output = 'json' cmd.launcher = Mock(Launcher) { getOptions() >> null } @@ -703,7 +703,7 @@ class CmdModuleInfoTest extends Specification { and: def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] - cmd.jsonOutput = true + cmd.output = 'json' cmd.launcher = Mock(Launcher) { getOptions() >> null } @@ -778,7 +778,7 @@ class CmdModuleInfoTest extends Specification { and: def cmd = new CmdModuleInfo() cmd.args = ['nf-core/fastqc'] - cmd.jsonOutput = true + cmd.output = 'json' cmd.launcher = Mock(Launcher) { getOptions() >> null } diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy index 4f985ecf88..8d745562a3 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy @@ -77,7 +77,7 @@ class CmdModuleListTest extends Specification { and: def cmd = new CmdModuleList() - cmd.jsonOutput = true + cmd.output = 'json' cmd.root = tempDir when: diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleSearchTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleSearchTest.groovy index 2790cb2b3f..9118dc8966 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleSearchTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleSearchTest.groovy @@ -113,7 +113,7 @@ class CmdModuleSearchTest extends Specification { } cmd.args = ['fastqc'] cmd.limit = 10 - cmd.jsonOutput = true + cmd.output = 'json' and: // Mock the registry client diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleRegistryClientTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleRegistryClientTest.groovy index 57311f809f..16e44ec9b1 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleRegistryClientTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleRegistryClientTest.groovy @@ -157,7 +157,7 @@ class ModuleRegistryClientTest extends Specification { .willReturn(aResponse() .withStatus(200) .withHeader('Content-Type', 'application/gzip') - .withHeader('X-Checksum', "sha256:${expectedChecksum}") + .withHeader('X-NF-Module-Checksum', "sha256:${expectedChecksum}") .withBody(modulePackage))) and: @@ -338,7 +338,7 @@ class ModuleRegistryClientTest extends Specification { .willReturn(aResponse() .withStatus(200) .withHeader('Content-Type', 'application/gzip') - .withHeader('X-Checksum', checksum) + .withHeader('X-NF-Module-Checksum', checksum) .withHeader('Docker-Content-Digest', checksum) .withBody(modulePackage))) From afddf21e5e51518f1f91d1e426a5b33f9499c96f Mon Sep 17 00:00:00 2001 From: jorgee Date: Thu, 5 Mar 2026 14:50:44 +0100 Subject: [PATCH 18/23] fix test Signed-off-by: jorgee --- .../groovy/nextflow/cli/module/CmdModuleSearchTest.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleSearchTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleSearchTest.groovy index 9118dc8966..155a3ae994 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleSearchTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleSearchTest.groovy @@ -127,7 +127,10 @@ class CmdModuleSearchTest extends Specification { when: cmd.run() - def output = capture.toString().readLines().last + def output = capture.toString().readLines() + .findResults { line -> !line.contains('DEBUG') ? line : null } + .findResults { line -> !line.contains('INFO') ? line : null } + .findResults { line -> !line.contains('Searching for') ? line : null }.join("\n") def json = new JsonSlurper().parseText(output) then: From 18a9e709c3b772946f50c06d7d2e903277fa082a Mon Sep 17 00:00:00 2001 From: Jorge Ejarque Date: Fri, 6 Mar 2026 16:08:32 +0100 Subject: [PATCH 19/23] Remote module inclusion (#6815) --- docs/module.md | 173 +++++++++++++++++- .../nextflow/cli/module/CmdModuleInfo.groovy | 13 +- .../cli/module/CmdModuleInstall.groovy | 6 +- .../nextflow/cli/module/CmdModuleList.groovy | 8 +- .../cli/module/CmdModulePublish.groovy | 22 ++- .../cli/module/CmdModuleRemove.groovy | 16 +- .../nextflow/cli/module/CmdModuleRun.groovy | 7 +- .../module/DefaultRemoteModuleResolver.groovy | 100 ++++++++++ .../nextflow/module/InstalledModule.groovy | 12 +- .../nextflow/module/ModuleChecksum.groovy | 25 ++- .../nextflow/module/ModuleReference.groovy | 15 +- .../module/ModuleRegistryClient.groovy | 11 +- .../nextflow/module/ModuleResolver.groovy | 34 ++-- .../nextflow/module/ModuleStorage.groovy | 102 ++++------- .../nextflow/pipeline/PipelineSpec.groovy | 12 +- .../groovy/nextflow/script/IncludeDef.groovy | 26 ++- .../nextflow.module.spi.RemoteModuleResolver | 1 + .../cli/module/CmdModuleInfoTest.groovy | 2 +- .../cli/module/CmdModuleInstallTest.groovy | 59 +++--- .../cli/module/CmdModuleListTest.groovy | 7 +- .../cli/module/CmdModuleRemoveTest.groovy | 12 +- .../cli/module/CmdModuleRunTest.groovy | 4 +- .../nextflow/config/ModulesConfigTest.groovy | 98 +++++----- .../DefaultRemoteModuleResolverTest.groovy | 46 +++++ .../module/InstalledModuleTest.groovy | 28 +-- .../nextflow/module/ModuleChecksumTest.groovy | 21 +-- .../module/ModuleReferenceTest.groovy | 66 ++++--- .../nextflow/module/ModuleResolverTest.groovy | 10 +- .../nextflow/module/ModuleStorageTest.groovy | 36 ++-- .../nextflow/script/IncludeDefTest.groovy | 28 ++- .../spi/FallbackRemoteModuleResolver.java | 45 +++++ .../module/spi/RemoteModuleResolver.java | 68 +++++++ .../spi/RemoteModuleResolverProvider.java | 92 ++++++++++ .../script/control/ModuleResolver.java | 26 ++- .../script/control/ResolveIncludeVisitor.java | 9 +- settings.gradle | 1 + specs/251117-module-system/data-model.md | 24 ++- specs/251117-module-system/plan.md | 33 +++- specs/251117-module-system/research.md | 47 +++-- 39 files changed, 965 insertions(+), 380 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/module/DefaultRemoteModuleResolver.groovy create mode 100644 modules/nextflow/src/main/resources/META-INF/services/nextflow.module.spi.RemoteModuleResolver create mode 100644 modules/nextflow/src/test/groovy/nextflow/module/DefaultRemoteModuleResolverTest.groovy create mode 100644 modules/nf-lang/src/main/java/nextflow/module/spi/FallbackRemoteModuleResolver.java create mode 100644 modules/nf-lang/src/main/java/nextflow/module/spi/RemoteModuleResolver.java create mode 100644 modules/nf-lang/src/main/java/nextflow/module/spi/RemoteModuleResolverProvider.java diff --git a/docs/module.md b/docs/module.md index 73d92fe4e2..9f8afdd7b0 100644 --- a/docs/module.md +++ b/docs/module.md @@ -279,8 +279,179 @@ This feature requires the use of a local or shared file system for the pipeline ## Sharing modules -Modules are designed to be easy to share and re-use across different pipelines, which helps eliminate duplicate work and spread improvements throughout the community. While Nextflow does not provide an explicit mechanism for sharing modules, there are several ways to do it: +Modules are designed to be easy to share and re-use across different pipelines, which helps eliminate duplicate work and spread improvements throughout the community. There are several ways to share modules: +- Use the Nextflow module registry (recommended, see below) - Simply copy the module files into your pipeline repository - Use [Git submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) to fetch modules from other Git repositories without maintaining a separate copy - Use the [nf-core](https://nf-co.re/tools#modules) CLI to install and update modules with a standard approach used by the nf-core community + +(module-registry)= + +## Registry-based modules + +:::{versionadded} 26.04.0 +::: + +Nextflow provides a module registry that enables you to install, share, and manage modules from centralized registries. This system provides version management, integrity checking, and seamless integration with the Nextflow DSL. + +### Installing modules from a registry + +Use the `module install` command to download modules from a registry: + +```console +$ nextflow module install nf-core/fastqc +$ nextflow module install nf-core/fastqc -version 1.0.0 +``` + +Installed modules are stored in the `modules/` directory and can be included using the registry syntax with the `@` prefix: + +```nextflow +include { FASTQC } from '@nf-core/fastqc' + +workflow { + reads = Channel.fromFilePairs('data/*_{1,2}.fastq.gz') + FASTQC(reads) +} +``` + +### Running modules directly + +For ad-hoc tasks or testing, you can run a module directly without creating a workflow: + +```console +$ nextflow module run nf-core/fastqc --input 'data/*.fastq.gz' +``` + +This command accepts all standard Nextflow options (`-profile`, `-resume`, etc.) and automatically downloads the module if not already installed. + +### Managing module versions + +Module versions are tracked in `nextflow_spec.json` in your project directory: + +```json +{ + "modules": { + "@nf-core/fastqc": "1.0.0", + "@nf-core/bwa-align": "1.2.0" + } +} +``` + +When you run your workflow, Nextflow automatically installs or updates modules to match the specified versions. + +### Discovering modules + +Search for available modules using the `module search` command: + +```console +$ nextflow module search alignment +$ nextflow module search "quality control" -limit 10 +``` + +List installed modules in your project: + +```console +$ nextflow module list +``` + +### Module checksum verification + +Nextflow automatically verifies module integrity using checksums. If you modify a module locally, Nextflow will detect the change and prevent accidental overwrites: + +```console +$ nextflow module install nf-core/fastqc -version 1.1.0 +Warning: Module @nf-core/fastqc has local modifications. Use -force to override. +``` + +Use the `-force` flag to override local modifications when needed. + +### Removing modules + +Use the `module remove` command to uninstall a module: + +```console +$ nextflow module remove nf-core/fastqc +``` + +By default, both the local module files and the entry in `nextflow_spec.json` are removed. Use the flags below to control this behaviour: + +- `-keep-files` — Remove the entry from `nextflow_spec.json` but keep the local module files +- `-keep-config` — Remove the local module files but keep the entry in `nextflow_spec.json` + +### Viewing module information + +Use the `module info` command to display metadata and a usage template for a module: + +```console +$ nextflow module info nf-core/fastqc +$ nextflow module info nf-core/fastqc -version 1.0.0 +``` + +The output includes the module description, authors, keywords, tools, inputs, outputs, and a ready-to-use command-line template. Use `-json` to get machine-readable output. + +### Publishing modules + +To share your own modules, use the `module publish` command: + +```console +$ nextflow module publish myorg/my-module +``` + +The argument can be either a `scope/name` reference (for an already-installed module) or a local directory path containing the module files. + +Your module directory must include: + +- `main.nf` - The module entry point +- `meta.yaml` - Module metadata (name, description, version, etc.) +- `README.md` - Module documentation + +Authentication is required for publishing and can be provided via the `NXF_REGISTRY_TOKEN` environment variable or in your configuration: + +```groovy +registry { + apiKey = 'YOUR_REGISTRY_TOKEN' +} +``` + +Use `-dry-run` to validate your module structure without uploading: + +```console +$ nextflow module publish myorg/my-module -dry-run +``` + +### Registry configuration + +By default, Nextflow uses the public registry at `https://registry.nextflow.io`. You can configure alternative or additional registries: + +```groovy +registry { + url = [ + 'https://private.registry.myorg.com', + 'https://registry.nextflow.io' + ] + apiKey = '${MYORG_TOKEN}' +} +``` + +Registries are queried in the order specified until a module is found. The `apiKey` is used only for the primary (first) registry. + +### Module directory structure + +Registry modules follow a standard directory structure: + +``` +modules/ +└── @scope/ + └── module-name/ + ├── .checksum # Integrity checksum (generated automatically) + ├── README.md # Documentation (required for publishing) + ├── main.nf # Module entry point (required) + ├── meta.yaml # Module metadata (required for publishing) + ├── resources/ # Optional: module binaries and resources + └── templates/ # Optional: process templates +``` + +The `modules/` directory should be committed to your Git repository to ensure reproducibility. + +See the {ref}`cli-page` documentation for complete details on all module commands. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInfo.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInfo.groovy index 56bc69708e..26a438a682 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInfo.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInfo.groovy @@ -90,8 +90,7 @@ class CmdModuleInfo extends CmdBase { throw new AbortOperationException("Incorrect number of arguments") } - def moduleRef = '@' + args[0] - def reference = ModuleReference.parse(moduleRef) + def reference = ModuleReference.parse(args[0]) // Get config def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize() @@ -117,10 +116,10 @@ class CmdModuleInfo extends CmdBase { log.warn "Failed to fetch metadata from registry: ${e.message}" } if( !release ) { - throw new AbortOperationException("No release information available for ${reference.nameWithoutPrefix}") + throw new AbortOperationException("No release information available for ${reference}") } if( !release.metadata ) { - log.info("No metadata found for $reference.nameWithoutPrefix ${release.version ? "($release.version)" : ''}") + log.info("No metadata found for $reference ${release.version ? "($release.version)" : ''}") } def moduleUrl = buildModuleUrl(registryConfig.url, reference, release.version) if( !output || output == 'text' ) { @@ -135,7 +134,7 @@ class CmdModuleInfo extends CmdBase { private void printFormattedInfo(ModuleReference reference, ModuleRelease release, String moduleUrl) { ModuleMetadata metadata = release.metadata println "" - println "Module: ${reference.nameWithoutPrefix}" + println "Module: ${reference}" println "Version: ${release.version}" println "URL: ${moduleUrl}" println "Description: ${metadata.description ?: release.description ?: 'N/A'}" @@ -218,7 +217,7 @@ class CmdModuleInfo extends CmdBase { private List generateUsageTemplate(ModuleReference reference, ModuleMetadata metadata) { def template = new ArrayList() - template.add("nextflow module run ${reference.nameWithoutPrefix}".toString()) + template.add("nextflow module run ${reference}".toString()) if( version ) template.add(" -version $version".toString()) @@ -281,7 +280,7 @@ class CmdModuleInfo extends CmdBase { private void printJsonInfo(ModuleReference reference, ModuleRelease release, String moduleUrl) { def metadata = release?.metadata def info = [ - name : reference.nameWithoutPrefix, + name : reference.toString(), fullName : reference.fullName, version : release.version, url : moduleUrl, diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy index f509aef73e..bcc52b6ed7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy @@ -70,9 +70,7 @@ class CmdModuleInstall extends CmdBase { throw new AbortOperationException("Incorrect number of arguments") } - def moduleRef = '@' + args[0] - - def reference = ModuleReference.parse(moduleRef) + def reference = ModuleReference.parse(args[0]) // Get config def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize() @@ -96,7 +94,7 @@ class CmdModuleInstall extends CmdBase { def installedVersion = version ?: resolver.resolveVersion(reference) specFile.addModuleEntry(reference.fullName, installedVersion) - println "Module ${reference.nameWithoutPrefix}@${installedVersion} installed and configured successfully" + println "Module ${reference}@${installedVersion} installed and configured successfully" } catch( AbortOperationException e ) { throw e diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy index b4919b0c68..5992196c2b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy @@ -111,7 +111,7 @@ class CmdModuleList extends CmdBase { installed.each { module -> def status = getStatusString(module.integrity) - println "${module.reference.nameWithoutPrefix.padRight(40)}${(module.installedVersion ?: 'unknown').padRight(15)}${status}" + println "${module.reference.toString().padRight(40)}${(module.installedVersion ?: 'unknown').padRight(15)}${status}" } println "" } @@ -119,7 +119,7 @@ class CmdModuleList extends CmdBase { private void printJsonList(List installed) { def modules = installed.collect { module -> [ - name : module.reference.nameWithoutPrefix, + name : module.reference.toString(), version : module.installedVersion ?: 'unknown', integrity: module.integrity.toString(), directory: module.directory.toString() @@ -136,8 +136,8 @@ class CmdModuleList extends CmdBase { return 'OK' case ModuleIntegrity.MODIFIED: return 'MODIFIED' - case ModuleIntegrity.MISSING_CHECKSUM: - return 'NO CHECKSUM' + case ModuleIntegrity.NO_REMOTE_MODULE: + return 'LOCAL' case ModuleIntegrity.CORRUPTED: return 'CORRUPTED' default: diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy index a3b46056fc..23cc9f74cc 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy @@ -25,6 +25,7 @@ import nextflow.cli.CmdBase import nextflow.config.ConfigBuilder import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException +import nextflow.module.ModuleChecksum import nextflow.module.ModuleSpec import nextflow.module.ModuleReference import nextflow.module.ModuleRegistryClient @@ -60,6 +61,9 @@ class CmdModulePublish extends CmdBase { @TestOnly protected ModuleRegistryClient client + //Flag if publish is invoked from a scope/name. In this case we should create/update the .module-info with the correct checksum + private boolean useModuleReference = false + @Override String getName() { return 'publish' @@ -115,14 +119,10 @@ class CmdModulePublish extends CmdBase { private void publishModule(Path moduleDir, RegistryConfig registryConfig, ModuleSpec manifest){ log.info "Creating module bundle..." - def storage = new ModuleStorage(moduleDir.parent) def tempBundleFile = Files.createTempFile("nf-module-publish-", ".tar.gz") try { - storage.createBundle(moduleDir, tempBundleFile) - - // Compute bundle checksum - def checksum = storage.computeBundleChecksum(tempBundleFile) + def checksum = ModuleStorage.createBundle(moduleDir, tempBundleFile) log.info "Bundle checksum: ${checksum}" // Read bundle content as bytes @@ -139,6 +139,14 @@ class CmdModulePublish extends CmdBase { def registryClient = new ModuleRegistryClient(registryConfig) def response = registryClient.publishModule(manifest.name, request, registryUrl) + if (useModuleReference) { + // If publish is performed using the module reference we should create/update the .module-info with the correct checksum + try { + ModuleChecksum.save(moduleDir, ModuleChecksum.compute(moduleDir)) + }catch (Exception e){ + log.warn("Unable to save the checksum - ${e.message}") + } + } println "✓ Module published successfully!" println "" println "Module details:" @@ -246,13 +254,13 @@ class CmdModulePublish extends CmdBase { return Paths.get(module).toAbsolutePath().normalize() } - final ref = ModuleReference.parse('@' + module) + final ref = ModuleReference.parse(module) final localStorage = new ModuleStorage(root ?: Paths.get('.').toAbsolutePath().normalize()) if (!localStorage.isInstalled(ref)){ throw new AbortOperationException("No module diretory found for $module") } - + useModuleReference = true return localStorage.getModuleDir(ref) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy index b793974f32..3c3ab9d026 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy @@ -68,9 +68,7 @@ class CmdModuleRemove extends CmdBase { throw new AbortOperationException("Cannot use both -keep-config and -keep-files flags together") } - def moduleRef = '@' + args[0] - - def reference = ModuleReference.parse(moduleRef) + def reference = ModuleReference.parse(args[0]) // Get config def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize() @@ -87,15 +85,15 @@ class CmdModuleRemove extends CmdBase { // Remove local files unless -keep-files is set if( !keepFiles ) { - println "Removing module files for ${reference.nameWithoutPrefix}..." + println "Removing module files for ${reference}..." filesRemoved = storage.removeModule(reference) if( filesRemoved ) { println "Module files removed successfully" } else { - println "Module ${reference.nameWithoutPrefix} was not installed locally" + println "Module ${reference} was not installed locally" } } else { - println "Keeping module files for ${reference.nameWithoutPrefix} (due to -keep-files flag)" + println "Keeping module files for ${reference} (due to -keep-files flag)" } // Remove config entry unless -keep-config is set @@ -105,7 +103,7 @@ class CmdModuleRemove extends CmdBase { if( configRemoved ) { println "Module entry removed from configuration" } else { - println "Module ${reference.nameWithoutPrefix} was not configured in nextflow_spec.json" + println "Module ${reference} was not configured in nextflow_spec.json" } } else { println "Keeping module entry in nextflow_spec.json (due to -keep-config flag)" @@ -113,9 +111,9 @@ class CmdModuleRemove extends CmdBase { // Summary if( filesRemoved || configRemoved ) { - println "\nModule ${reference.nameWithoutPrefix} removal completed" + println "\nModule ${reference} removal completed" } else { - println "\nModule ${reference.nameWithoutPrefix} was not found" + println "\nModule ${reference} was not found" } } catch( AbortOperationException e ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy index 4212ae0d2d..56f3cf18bf 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy @@ -61,15 +61,12 @@ class CmdModuleRun extends CmdRun { throw new AbortOperationException("Arguments not provided") } - // Parse module reference (first argument starting with @) - String moduleRef = '@' + args[0] - // Parse and validate module reference ModuleReference reference try { - reference = ModuleReference.parse(moduleRef) + reference = ModuleReference.parse(args[0]) } catch( Exception e ) { - throw new AbortOperationException("Invalid module reference: ${moduleRef}", e) + throw new AbortOperationException("Invalid module reference: ${args[0]}", e) } // Get config diff --git a/modules/nextflow/src/main/groovy/nextflow/module/DefaultRemoteModuleResolver.groovy b/modules/nextflow/src/main/groovy/nextflow/module/DefaultRemoteModuleResolver.groovy new file mode 100644 index 0000000000..e1db3366f0 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/module/DefaultRemoteModuleResolver.groovy @@ -0,0 +1,100 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.Global +import nextflow.NF +import nextflow.Session +import nextflow.config.ConfigBuilder +import nextflow.config.ModulesConfig +import nextflow.config.RegistryConfig +import nextflow.exception.IllegalModulePath +import nextflow.module.spi.RemoteModuleResolver +import nextflow.pipeline.PipelineSpec + +import java.nio.file.Path + +/** + * Default implementation of RemoteModuleResolver using the Nextflow module registry. + * + *

This implementation: + *

    + *
  • Checks for locally installed modules in the project's modules directory
    • + *
  • Downloads modules from the configured registry if not present
    • + *
  • Reads version constraints from nextflow_spec.json
    • + *
  • Uses the Session's registry configuration
    • + *
+ * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +class DefaultRemoteModuleResolver implements RemoteModuleResolver { + + @Override + Path resolve(String moduleName, Path baseDir) { + + final modulesConfig = getModuleConfig(baseDir) + + final config = Global.config ?: new ConfigBuilder().setBaseDir(baseDir).build() + final registryConfig = config.navigate('registry') as RegistryConfig + + // Create module resolver + def resolver = new ModuleResolver(baseDir, modulesConfig, registryConfig) + + try { + log.debug "Resolving remote module: ${moduleName}" + + // Parse module reference + def reference = ModuleReference.parse(moduleName) + + // Resolve module (will auto-install if missing or version mismatch) + def mainFile = resolver.resolve(reference, null, true) + + log.debug "Module ${reference} resolved to ${mainFile}" + return mainFile + } catch (Exception e) { + throw new IllegalModulePath( + "Failed to resolve remote module ${moduleName}: ${e.message}", + e + ) + } + } + + @Override + int getPriority() { + return 0 // Default implementation has lowest priority + } + + private ModulesConfig getModuleConfig(Path baseDir) { + def specFile = new PipelineSpec(baseDir) + + if (!specFile.exists()) { + log.warn1("Remote module specified and 'nextflow_spec.json' not found") + return new ModulesConfig() + } + + def modules = specFile.getModules() + if (!modules || modules.isEmpty()) { + log.warn1("Remote module specified and no modules configured in 'nextflow_spec.json'") + return new ModulesConfig() + } + return new ModulesConfig(modules) + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy b/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy index cb032d6c2a..88fddb02a6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy @@ -39,7 +39,7 @@ class InstalledModule { Path directory Path mainFile Path manifestFile - Path checksumFile + Path moduleInfoFile String installedVersion String expectedChecksum @@ -54,9 +54,9 @@ class InstalledModule { return ModuleIntegrity.CORRUPTED } - // Check if checksum file exists - if( !Files.exists(checksumFile) ) { - return ModuleIntegrity.MISSING_CHECKSUM + // Check if .module-info file exists + if( !Files.exists(moduleInfoFile) ) { + return ModuleIntegrity.NO_REMOTE_MODULE } try { @@ -71,7 +71,7 @@ class InstalledModule { return ModuleIntegrity.MODIFIED } } catch( Exception e ) { - log.warn "Failed to compute checksum for module ${reference.nameWithoutPrefix}: ${e.message}" + log.warn "Failed to compute checksum for module ${reference}: ${e.message}" return ModuleIntegrity.CORRUPTED } } @@ -84,6 +84,6 @@ class InstalledModule { enum ModuleIntegrity { VALID, // Checksum matches MODIFIED, // Checksum mismatch (local changes) - MISSING_CHECKSUM, // No .checksum file + NO_REMOTE_MODULE, // No .module-info file (local-only module, no registry origin) CORRUPTED // Missing required files } diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy index fc795f1b60..94235bc696 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy @@ -33,7 +33,7 @@ import java.security.MessageDigest class ModuleChecksum { public static final String CHECKSUM_ALGORITHM = "SHA-256" - public static final String CHECKSUM_FILE = ".checksum" + public static final String MODULE_INFO_FILE = ".module-info" /** * Compute the SHA-256 checksum of a module directory @@ -54,7 +54,7 @@ class ModuleChecksum { try( final walkStream = Files.walk(moduleDir) ) { walkStream .filter { Path path -> Files.isRegularFile(path) } - .filter { Path path -> !path.fileName.toString().equals(CHECKSUM_FILE) } + .filter { Path path -> !path.fileName.toString().equals(MODULE_INFO_FILE) } .sorted() .each { Path path -> files.add(path) } } @@ -85,28 +85,35 @@ class ModuleChecksum { } /** - * Save a checksum to the .checksum file in the module directory + * Save a checksum to the .module-info file in the module directory * * @param moduleDir The module directory path * @param checksum The checksum to save */ static void save(Path moduleDir, String checksum) { - def checksumFile = moduleDir.resolve(CHECKSUM_FILE) - Files.writeString(checksumFile, checksum) + def moduleInfoFile = moduleDir.resolve(MODULE_INFO_FILE) + def props = new Properties() + // If file exists loads to update current just checksum property + if( Files.exists( moduleInfoFile)) + moduleInfoFile.withInputStream { is -> props.load(is) } + props.setProperty('checksum', checksum) + moduleInfoFile.withOutputStream { os -> props.store(os, null) } } /** - * Load a checksum from the .checksum file in the module directory + * Load a checksum from the .module-info file in the module directory * * @param moduleDir The module directory path * @return The checksum, or null if file doesn't exist */ static String load(Path moduleDir) { - def checksumFile = moduleDir.resolve(CHECKSUM_FILE) - if( !Files.exists(checksumFile) ) { + def moduleInfoFile = moduleDir.resolve(MODULE_INFO_FILE) + if( !Files.exists(moduleInfoFile) ) { return null } - return checksumFile.text + def props = new Properties() + moduleInfoFile.withInputStream { is -> props.load(is) } + return props.getProperty('checksum') } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleReference.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleReference.groovy index 12bff1c6ac..23cbf7ba3d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleReference.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleReference.groovy @@ -34,7 +34,7 @@ class ModuleReference { // Pattern allows: optional @, scope with letters/digits/hyphens/dots/underscores, name segments separated by slashes (no trailing slash) // Scope: starts with letter/digit, followed by letters/digits/dots/underscores/hyphens // Name: one or more segments (each starting with letter, followed by letters/digits/underscores/hyphens), separated by slashes - private static final Pattern MODULE_NAME_PATTERN = ~/^@?([a-z0-9][a-z0-9._\-]*)\/([a-z][a-z0-9_\-]*(?:\/[a-z][a-z0-9_\-]*)*)$/ + private static final Pattern MODULE_NAME_PATTERN = ~/^([a-z0-9][a-z0-9._\-]*)\/([a-z][a-z0-9._\-]*(?:\/[a-z][a-z0-9._\-]*)*)$/ final String scope final String name @@ -43,7 +43,7 @@ class ModuleReference { ModuleReference(String scope, String name) { this.scope = scope this.name = name - this.fullName = "@${scope}/${name}" + this.fullName = "${scope}/${name}" } /** @@ -65,7 +65,7 @@ class ModuleReference { if( !matcher.matches() ) { throw new AbortOperationException( "Invalid module reference: '${source}'. " + - "Expected format: [@]scope/name where scope is lowercase alphanumeric with dots/underscores/hyphens " + + "Expected format: scope/name where scope is lowercase alphanumeric with dots/underscores/hyphens " + "and name is lowercase alphanumeric with underscores/hyphens, optionally with slash-separated segments" ) } @@ -73,15 +73,6 @@ class ModuleReference { return new ModuleReference(matcher.group(1), matcher.group(2)) } - /** - * Get the module name without the @ prefix - * - * @return Module name in format "scope/name" - */ - String getNameWithoutPrefix() { - return "${scope}/${name}" - } - @Override String toString() { return fullName diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy index fa1d827622..4b2fc225ec 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy @@ -56,10 +56,7 @@ class ModuleRegistryClient { } private String encodeName(String name) { - return URLEncoder.encode( - name.startsWith('@') ? name.substring(1) : name, - 'UTF-8' - ) + return URLEncoder.encode(name, 'UTF-8') } /** @@ -443,11 +440,7 @@ class ModuleRegistryClient { " }\n" ) } - try { - return publishModuleToRegistry(registryUrl, name, request, authToken) - } catch( Exception e ) { - throw new AbortOperationException("Failed to publish to ${registryUrl}", e) - } + return publishModuleToRegistry(registryUrl, name, request, authToken) } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy index 11ad0f2335..a5a55c2fe8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy @@ -69,25 +69,27 @@ class ModuleResolver { def integrity = installed.integrity if( integrity == ModuleIntegrity.CORRUPTED ) { throw new AbortOperationException( - "Module ${reference.nameWithoutPrefix} is corrupted (missing required files). " + + "Module ${reference} is corrupted (missing required files). " + "Please remove and reinstall." ) } if( integrity == ModuleIntegrity.MODIFIED ) { - log.warn "Module ${reference.nameWithoutPrefix} has local modifications (checksum mismatch)" + log.warn1 "Module ${reference} has local modifications (checksum mismatch)" + } else if( integrity == ModuleIntegrity.NO_REMOTE_MODULE ) { + log.warn1 "Module ${reference} has no registry origin (.module-info missing)" } // Check if version matches if( targetVersion && installed.installedVersion != targetVersion ) { if( autoInstall ) { - log.info "Upgrading module ${reference.nameWithoutPrefix} from ${installed.installedVersion} to ${targetVersion}" + log.info "Upgrading module ${reference} from ${installed.installedVersion} to ${targetVersion}" return installModule(reference, targetVersion) } else { throw new AbortOperationException( - "Module ${reference.nameWithoutPrefix} version mismatch: " + + "Module ${reference} version mismatch: " + "installed=${installed.installedVersion}, required=${targetVersion}. " + - "Run 'nextflow module install ${reference.nameWithoutPrefix}@${targetVersion}' to update." + "Run 'nextflow module install ${reference}@${targetVersion}' to update." ) } } @@ -101,17 +103,17 @@ class ModuleResolver { return installModule(reference, targetVersion) } else { throw new AbortOperationException( - "Module ${reference.nameWithoutPrefix} is not installed. " + - "Run 'nextflow module install ${reference.nameWithoutPrefix}' to install." + "Module ${reference} is not installed. " + + "Run 'nextflow module install ${reference}' to install." ) } } String resolveVersion(ModuleReference reference) { final version = modulesConfig.getVersion(reference.fullName) - ?: registryClient.fetchModule(reference.fullName).latest?.version + ?: registryClient.fetchModule(reference.fullName)?.latest?.version if( !version ) { - throw new AbortOperationException("Module ${reference.nameWithoutPrefix} has no published versions") + throw new AbortOperationException("Module ${reference} has no published versions") } return version } @@ -131,7 +133,7 @@ class ModuleResolver { if( storage.isInstalled(reference) ) { def installed = storage.getInstalledModule(reference) if( installed.installedVersion == version ) { - log.info "Module ${reference.nameWithoutPrefix}@${installed.installedVersion} is already installed (version $version)" + log.info "Module ${reference}@${installed.installedVersion} is already installed (version $version)" return installed.mainFile } @@ -139,14 +141,20 @@ class ModuleResolver { def integrity = installed.integrity if( integrity == ModuleIntegrity.MODIFIED && !force ) { throw new AbortOperationException( - "Module ${reference.nameWithoutPrefix} has local modifications. " + + "Module ${reference} has local modifications. " + + "Use --force to override, or save your changes first." + ) + } + if( integrity == ModuleIntegrity.NO_REMOTE_MODULE && !force ) { + throw new AbortOperationException( + " Folder 'modules/${reference}' already exists and is not a valid remote module. " + "Use --force to override, or save your changes first." ) } } - log.info "Installing module ${reference.nameWithoutPrefix}@${version}..." + log.info "Installing module ${reference}@${version}..." // Download module package to temporary location Path tempFile = Files.createTempFile("nf-module-", ".tgz") @@ -157,7 +165,7 @@ class ModuleResolver { // Install to modules directory (will compute directory checksum for future integrity checks) InstalledModule installed = storage.installModule(reference, version, tempFile) - log.info "Module ${reference.nameWithoutPrefix}@${version} installed successfully at ${installed.mainFile.parent}" + log.info "Module ${reference}@${version} installed successfully at ${installed.mainFile.parent}" return installed.mainFile } finally { diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy index 97802c711d..9cffaa0279 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy @@ -70,7 +70,7 @@ class ModuleStorage { * @return The module directory path */ Path getModuleDir(ModuleReference reference) { - return modulesDir.resolve("@${reference.scope}").resolve(reference.name) + return modulesDir.resolve(reference.scope).resolve(reference.name) } /** @@ -101,7 +101,7 @@ class ModuleStorage { directory: moduleDir, mainFile: moduleDir.resolve(Const.DEFAULT_MAIN_FILE_NAME), manifestFile: moduleDir.resolve(MODULE_MANIFEST_FILE), - checksumFile: moduleDir.resolve(ModuleChecksum.CHECKSUM_FILE), + moduleInfoFile: moduleDir.resolve(ModuleChecksum.MODULE_INFO_FILE), ) // Load checksum if available @@ -111,7 +111,7 @@ class ModuleStorage { } /** - * List all installed modules + * List all installed modules by scanning for directories containing a .module-info marker file. * * @return List of InstalledModule objects */ @@ -122,63 +122,27 @@ class ModuleStorage { List modules = [] - // Iterate over scope directories - try( final scopeStream = Files.list(modulesDir) ){ - scopeStream.each { Path scopeDir -> - if (!Files.isDirectory(scopeDir)) return - - def scopeDirName = scopeDir.fileName.toString() - // Remove @ prefix from directory name to get scope - def scope = scopeDirName.startsWith('@') ? scopeDirName.substring(1) : scopeDirName - - // Recursively find all directories containing meta.yml under this scope - findModulesRecursive(scopeDir, scope, modules) - } - } - - return modules - } - - /** - * Recursively find modules in subdirectories - * @param dir Current directory to search - * @param scope Module scope - * @param modules List to accumulate found modules - */ - private void findModulesRecursive(Path dir, String scope, List modules) { - if (!Files.isDirectory(dir)) return - - // Check if current directory contains meta.yml (is a module) - if (Files.exists(dir.resolve(MODULE_MANIFEST_FILE))) { - // Calculate the module name from the path relative to scope directory - def scopeDir = dir.getParent() - while (scopeDir != null && !scopeDir.fileName.toString().equals('@' + scope)) { - scopeDir = scopeDir.getParent() - } - - if (scopeDir != null) { - def relativePath = scopeDir.relativize(dir).toString() - def name = relativePath.replace('\\', '/') // Normalize path separators - def reference = new ModuleReference(scope, name) - - def installed = getInstalledModule(reference) - if (installed) { - modules.add(installed) - } - } - } - - // Recursively search subdirectories - try (final subStream = Files.list(dir) ) { - subStream.each { Path subDir -> - if (Files.isDirectory(subDir)) { - findModulesRecursive(subDir, scope, modules) + try( final walkStream = Files.walk(modulesDir) ) { + walkStream + .filter { Path path -> Files.isDirectory(path) } + .filter { Path path -> Files.exists(path.resolve(ModuleChecksum.MODULE_INFO_FILE)) } + .each { Path moduleDir -> + try { + def rel = modulesDir.relativize(moduleDir) + if( rel.nameCount < 2 ) return // Need at least scope/name + def reference = ModuleReference.parse(rel.toString()) + def installed = getInstalledModule(reference) + if( installed ) modules.add(installed) + } catch(Exception e){ + // Catching exception to go on inspecting other valid folders + log.debug("Not a valid module reference - $e.message") } } - } catch (IOException e) { - log.warn "Failed to list directory ${dir}: ${e.message}" + log.warn "Failed to scan modules directory ${modulesDir}: ${e.message}" } + + return modules } /** @@ -214,7 +178,7 @@ class ModuleStorage { def checksum = ModuleChecksum.compute(moduleDir) ModuleChecksum.save(moduleDir, checksum) - log.debug "Installed module ${reference.nameWithoutPrefix}@${version} to ${moduleDir}" + log.debug "Installed module ${reference}@${version} to ${moduleDir}" return getInstalledModule(reference) } @@ -227,7 +191,7 @@ class ModuleStorage { log.warn "Failed to clean up after installation failure: ${cleanupError.message}" } } - throw new AbortOperationException("Failed to install module ${reference.nameWithoutPrefix}@${version}", e) + throw new AbortOperationException("Failed to install module ${reference}@${version}", e) } } @@ -246,7 +210,7 @@ class ModuleStorage { try { FileHelper.deletePath(moduleDir) - log.debug "Removed module: ${reference.nameWithoutPrefix}" + log.debug "Removed module: ${reference}" // Clean up empty scope directory def scopeDir = moduleDir.parent @@ -257,7 +221,7 @@ class ModuleStorage { return true } catch (Exception e) { - throw new AbortOperationException("Failed to remove module ${reference.nameWithoutPrefix}", e) + throw new AbortOperationException("Failed to remove module ${reference}", e) } } @@ -362,9 +326,9 @@ class ModuleStorage { * * @param moduleDir The module directory to bundle * @param targetFile The target bundle file path - * @return The created bundle file with its checksum + * @return The created bundle file checksum */ - Path createBundle(Path moduleDir, Path targetFile) { + static String createBundle(Path moduleDir, Path targetFile) { if (!Files.exists(moduleDir) || !Files.isDirectory(moduleDir)) { throw new AbortOperationException("Module directory not found: ${moduleDir}") } @@ -384,9 +348,9 @@ class ModuleStorage { } } } - - log.debug "Created module bundle: ${targetFile} (size: ${Files.size(targetFile)} bytes)" - return targetFile + final checksum = computeBundleChecksum(targetFile) + log.debug "Created module bundle: ${targetFile} (size: ${Files.size(targetFile)} bytes, checksum: $checksum)" + return checksum } catch (Exception e) { // Clean up partial file on failure @@ -408,12 +372,12 @@ class ModuleStorage { * @param sourceDir The source directory being archived * @param currentPath The current path being added */ - private void addToTarArchive(TarArchiveOutputStream tos, Path sourceDir, Path currentPath) { + private static void addToTarArchive(TarArchiveOutputStream tos, Path sourceDir, Path currentPath) { try ( def tarStream = Files.list(currentPath)) { tarStream.each { Path path -> - // Skip .checksum file when creating bundle - if (path.fileName.toString() == ModuleChecksum.CHECKSUM_FILE) { + // Skip .module-info file when creating bundle + if (path.fileName.toString() == ModuleChecksum.MODULE_INFO_FILE) { return } @@ -447,7 +411,7 @@ class ModuleStorage { * @param bundleFile The bundle file * @return The SHA-256 checksum as hex string */ - String computeBundleChecksum(Path bundleFile) { + static String computeBundleChecksum(Path bundleFile) { if (!Files.exists(bundleFile)) { throw new AbortOperationException("Bundle file not found: ${bundleFile}") } diff --git a/modules/nextflow/src/main/groovy/nextflow/pipeline/PipelineSpec.groovy b/modules/nextflow/src/main/groovy/nextflow/pipeline/PipelineSpec.groovy index 766b71f62e..2e109cdc4f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/pipeline/PipelineSpec.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/pipeline/PipelineSpec.groovy @@ -51,8 +51,8 @@ class PipelineSpec { * @param version The module version */ void addModuleEntry(String moduleName, String version) { - // Normalize module name (ensure it starts with @) - def normalizedName = moduleName.startsWith('@') ? moduleName : '@' + moduleName + // Normalize module name (strip leading @ if present) + def normalizedName = moduleName.startsWith('@') ? moduleName.substring(1) : moduleName def spec = readSpecFile() @@ -80,8 +80,6 @@ class PipelineSpec { * @return true if entry was removed, false if it didn't exist */ boolean removeModuleEntry(String moduleName) { - // Normalize module name (ensure it starts with @) - def normalizedName = moduleName.startsWith('@') ? moduleName : '@' + moduleName def spec = readSpecFile() @@ -89,10 +87,10 @@ class PipelineSpec { return false } final modules = spec.modules as Map - if( modules.remove(normalizedName) == null ) + if( modules.remove(moduleName) == null ) return false writeSpecFile(spec) - log.info "Removed ${normalizedName} from ${SPEC_FILE_NAME}" + log.info "Removed ${moduleName} from ${SPEC_FILE_NAME}" return true } /** @@ -145,4 +143,4 @@ class PipelineSpec { throw new RuntimeException("Failed to write spec file ${specFile}: ${e.message}", e) } } -} \ No newline at end of file +} diff --git a/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy index b19f814745..531e6c4cc4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy @@ -29,6 +29,8 @@ import nextflow.NF import nextflow.Session import nextflow.exception.IllegalModulePath import nextflow.exception.ScriptCompilationException +import nextflow.module.ModuleReference +import nextflow.module.spi.RemoteModuleResolverProvider import nextflow.plugin.Plugins import nextflow.plugin.extension.PluginExtensionProvider import nextflow.script.parser.v1.ScriptLoaderV1 @@ -162,14 +164,24 @@ class IncludeDef { @PackageScope Path resolveModulePath(include) { assert include - final result = include as Path if( result.isAbsolute() ) { if( result.scheme == 'file' ) return result throw new IllegalModulePath("Cannot resolve module path: ${result.toUriString()}") } + final str = include.toString() + if( str.startsWith('./') || str.startsWith('../') ) { + return getOwnerPath().resolveSibling(str).normalize() + } + // Not a local path — treat as remote module reference (scope/name) + return resolveRemoteModulePath(str) + } - return getOwnerPath().resolveSibling(include.toString()) + @PackageScope + Path resolveRemoteModulePath(String moduleName) { + // Use SPI to get the remote module resolver implementation + def resolver = RemoteModuleResolverProvider.getInstance() + return resolver.resolve(moduleName, session.baseDir) } @PackageScope @@ -206,9 +218,15 @@ class IncludeDef { throw new IllegalModulePath("Remote modules are not allowed -- Offending module: ${path.toUriString()}") final str = path.toString() - if( !str.startsWith('/') && !str.startsWith('./') && !str.startsWith('../') && !str.startsWith('plugin/') ) - throw new IllegalModulePath("Module path must start with / or ./ prefix -- Offending module: $str") + if( str.startsWith('/') || str.startsWith('./') || str.startsWith('../') || str.startsWith('plugin/') ) + return + // Otherwise must be a valid remote module reference in scope/name format + try { + ModuleReference.parse(str) + } catch( Exception e ) { + throw new IllegalModulePath("Module path must start with '/', './', '../' or 'plugin/' prefix, or be a valid remote module reference (scope/name) -- Offending module: $str") + } } @PackageScope diff --git a/modules/nextflow/src/main/resources/META-INF/services/nextflow.module.spi.RemoteModuleResolver b/modules/nextflow/src/main/resources/META-INF/services/nextflow.module.spi.RemoteModuleResolver new file mode 100644 index 0000000000..d12a870946 --- /dev/null +++ b/modules/nextflow/src/main/resources/META-INF/services/nextflow.module.spi.RemoteModuleResolver @@ -0,0 +1 @@ +nextflow.module.DefaultRemoteModuleResolver diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInfoTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInfoTest.groovy index 7668fa7cf7..ba90e5d005 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInfoTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInfoTest.groovy @@ -177,7 +177,7 @@ class CmdModuleInfoTest extends Specification { then: json.name == 'nf-core/fastqc' - json.fullName == '@nf-core/fastqc' + json.fullName == 'nf-core/fastqc' json.version == '1.0.0' json.description == 'FastQC quality control' json.authors == ['nf-core'] diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInstallTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInstallTest.groovy index c04dad81bb..a2f088b544 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInstallTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInstallTest.groovy @@ -20,6 +20,7 @@ import io.seqera.npr.api.schema.v1.Module import io.seqera.npr.api.schema.v1.ModuleRelease import nextflow.cli.Launcher import nextflow.exception.AbortOperationException +import nextflow.module.ModuleChecksum import nextflow.module.ModuleRegistryClient import nextflow.pipeline.PipelineSpec import org.apache.commons.compress.archivers.tar.TarArchiveEntry @@ -61,11 +62,11 @@ class CmdModuleInstallTest extends Specification { // Mock registry client def mockClient = Mock(ModuleRegistryClient) - mockClient.fetchModule('@nf-core/fastqc') >> new Module( - name: '@nf-core/fastqc', + mockClient.fetchModule('nf-core/fastqc') >> new Module( + name: 'nf-core/fastqc', latest: new ModuleRelease(version: '1.0.0') ) - mockClient.downloadModule('@nf-core/fastqc', '1.0.0', _) >> { String name, String version, Path dest -> + mockClient.downloadModule('nf-core/fastqc', '1.0.0', _) >> { String name, String version, Path dest -> Files.write(dest, modulePackage) return dest } @@ -81,14 +82,14 @@ class CmdModuleInstallTest extends Specification { output.contains('1.0.0') and: - def moduleDir = tempDir.resolve('modules/@nf-core/fastqc') + def moduleDir = tempDir.resolve('modules/nf-core/fastqc') Files.exists(moduleDir) Files.exists(moduleDir.resolve('main.nf')) Files.exists(moduleDir.resolve('meta.yml')) and: def spec = new PipelineSpec(tempDir) - spec.getModules().get('@nf-core/fastqc') == '1.0.0' + spec.getModules().get('nf-core/fastqc') == '1.0.0' } def 'should install module with specific version'() { @@ -122,14 +123,14 @@ class CmdModuleInstallTest extends Specification { and: def spec = new PipelineSpec(tempDir) - spec.getModules().get('@nf-core/fastqc') == '2.0.0' + spec.getModules().get('nf-core/fastqc') == '2.0.0' } def 'should update existing module with force flag'() { given: // Pre-install version 1.0.0 - def moduleDir = tempDir.resolve('modules/@nf-core/fastqc') + def moduleDir = tempDir.resolve('modules/nf-core/fastqc') Files.createDirectories(moduleDir) moduleDir.resolve('main.nf').text = 'process OLD { }' moduleDir.resolve('meta.yml').text = """ @@ -139,7 +140,7 @@ class CmdModuleInstallTest extends Specification { """.stripIndent() def spec = new PipelineSpec(tempDir) - spec.addModuleEntry('@nf-core/fastqc', '1.0.0') + spec.addModuleEntry('nf-core/fastqc', '1.0.0') and: def cmd = new CmdModuleInstall() @@ -155,7 +156,7 @@ class CmdModuleInstallTest extends Specification { def modulePackage = createModulePackage('nf-core', 'fastqc', '2.0.0') def mockClient = Mock(ModuleRegistryClient) - mockClient.downloadModule('@nf-core/fastqc', '2.0.0', _) >> { String name, String version, Path dest -> + mockClient.downloadModule('nf-core/fastqc', '2.0.0', _) >> { String name, String version, Path dest -> Files.write(dest, modulePackage) return dest } @@ -171,7 +172,7 @@ class CmdModuleInstallTest extends Specification { and: def updatedSpec = new PipelineSpec(tempDir) - updatedSpec.getModules().get('@nf-core/fastqc') == '2.0.0' + updatedSpec.getModules().get('nf-core/fastqc') == '2.0.0' and: moduleDir.resolve('main.nf').text.contains('FASTQC') // New content @@ -180,7 +181,7 @@ class CmdModuleInstallTest extends Specification { def 'should fail when module already installed without force'() { given: // Pre-install the module - def moduleDir = tempDir.resolve('modules/@nf-core/fastqc') + def moduleDir = tempDir.resolve('modules/nf-core/fastqc') Files.createDirectories(moduleDir) moduleDir.resolve('main.nf').text = 'process FASTQC { }' moduleDir.resolve('meta.yml').text = """ @@ -188,9 +189,9 @@ class CmdModuleInstallTest extends Specification { version: '1.0.0' description: Test module """.stripIndent() - moduleDir.resolve('.checksum').text = 'wrong-checksum' + ModuleChecksum.save(moduleDir, 'wrong-checksum') def spec = new PipelineSpec(tempDir) - spec.addModuleEntry('@nf-core/fastqc', '1.0.0') + spec.addModuleEntry('nf-core/fastqc', '1.0.0') and: def cmd = new CmdModuleInstall() @@ -202,8 +203,8 @@ class CmdModuleInstallTest extends Specification { cmd.root = tempDir def mockClient = Mock(ModuleRegistryClient) - mockClient.fetchModule('@nf-core/fastqc') >> new Module( - name: '@nf-core/fastqc', + mockClient.fetchModule('nf-core/fastqc') >> new Module( + name: 'nf-core/fastqc', latest: new ModuleRelease(version: '2.0.0') ) cmd.client = mockClient @@ -229,11 +230,11 @@ class CmdModuleInstallTest extends Specification { def modulePackage = createModulePackage('myorg', 'custom-module', '1.0.0') def mockClient = Mock(ModuleRegistryClient) - mockClient.fetchModule('@myorg/custom-module') >> new Module( - name: '@myorg/custom-module', + mockClient.fetchModule('myorg/custom-module') >> new Module( + name: 'myorg/custom-module', latest: new ModuleRelease(version: '1.0.0') ) - mockClient.downloadModule('@myorg/custom-module', '1.0.0', _) >> { String name, String version, Path dest -> + mockClient.downloadModule('myorg/custom-module', '1.0.0', _) >> { String name, String version, Path dest -> Files.write(dest, modulePackage) return dest } @@ -243,13 +244,13 @@ class CmdModuleInstallTest extends Specification { cmd.run() then: - def moduleDir = tempDir.resolve('modules/@myorg/custom-module') + def moduleDir = tempDir.resolve('modules/myorg/custom-module') Files.exists(moduleDir) Files.exists(moduleDir.resolve('main.nf')) and: def spec = new PipelineSpec(tempDir) - spec.getModules().get('@myorg/custom-module') == '1.0.0' + spec.getModules().get('myorg/custom-module') == '1.0.0' } def 'should create modules directory if it does not exist'() { @@ -265,11 +266,11 @@ class CmdModuleInstallTest extends Specification { def modulePackage = createModulePackage('nf-core', 'fastqc', '1.0.0') def mockClient = Mock(ModuleRegistryClient) - mockClient.fetchModule('@nf-core/fastqc') >> new Module( + mockClient.fetchModule('nf-core/fastqc') >> new Module( name: 'nf-core/fastqc', latest: new ModuleRelease(version: '1.0.0') ) - mockClient.downloadModule('@nf-core/fastqc', '1.0.0', _) >> { String name, String version, Path dest -> + mockClient.downloadModule('nf-core/fastqc', '1.0.0', _) >> { String name, String version, Path dest -> Files.write(dest, modulePackage) return dest } @@ -280,8 +281,8 @@ class CmdModuleInstallTest extends Specification { then: Files.exists(tempDir.resolve('modules')) - Files.exists(tempDir.resolve('modules/@nf-core')) - Files.exists(tempDir.resolve('modules/@nf-core/fastqc')) + Files.exists(tempDir.resolve('modules/nf-core')) + Files.exists(tempDir.resolve('modules/nf-core/fastqc')) } def 'should create checksum file after installation'() { @@ -297,11 +298,11 @@ class CmdModuleInstallTest extends Specification { def modulePackage = createModulePackage('nf-core', 'fastqc', '1.0.0') def mockClient = Mock(ModuleRegistryClient) - mockClient.fetchModule('@nf-core/fastqc') >> new Module( + mockClient.fetchModule('nf-core/fastqc') >> new Module( name: 'nf-core/fastqc', latest: new ModuleRelease(version: '1.0.0') ) - mockClient.downloadModule('@nf-core/fastqc', '1.0.0', _) >> { String name, String version, Path dest -> + mockClient.downloadModule('nf-core/fastqc', '1.0.0', _) >> { String name, String version, Path dest -> Files.write(dest, modulePackage) return dest } @@ -311,11 +312,11 @@ class CmdModuleInstallTest extends Specification { cmd.run() then: - def moduleDir = tempDir.resolve('modules/@nf-core/fastqc') - Files.exists(moduleDir.resolve('.checksum')) + def moduleDir = tempDir.resolve('modules/nf-core/fastqc') + Files.exists(moduleDir.resolve('.module-info')) and: - def checksum = moduleDir.resolve('.checksum').text + def checksum = ModuleChecksum.load(moduleDir) checksum != null !checksum.isEmpty() } diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy index 8d745562a3..00bafc75b8 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy @@ -65,7 +65,7 @@ class CmdModuleListTest extends Specification { output.contains('1.0.0') output.contains('nf-core/multiqc') output.contains('2.1.0') - output.contains('OK') || output.contains('NO CHECKSUM') + output.contains('OK') } def 'should list installed modules with JSON output'() { @@ -177,9 +177,8 @@ class CmdModuleListTest extends Specification { description: Test module """.stripIndent() - // Create checksum - def checksum = ModuleChecksum.compute(moduleDir) - moduleDir.resolve('.checksum').text = checksum + // Create .module-info + ModuleChecksum.save(moduleDir, ModuleChecksum.compute(moduleDir)) return moduleDir } diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRemoveTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRemoveTest.groovy index 0361b3a274..0b4972c3ef 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRemoveTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRemoveTest.groovy @@ -51,7 +51,7 @@ class CmdModuleRemoveTest extends Specification { // Create spec file with module entry def specFile = new PipelineSpec(tempDir) - specFile.addModuleEntry('@nf-core/fastqc', '1.0.0') + specFile.addModuleEntry('nf-core/fastqc', '1.0.0') and: def cmd = new CmdModuleRemove() @@ -71,7 +71,7 @@ class CmdModuleRemoveTest extends Specification { and: def spec = new PipelineSpec(tempDir) - spec.getModules().get('@nf-core/fastqc') == null + spec.getModules().get('nf-core/fastqc') == null } def 'should keep config with -keep-config flag'() { @@ -82,7 +82,7 @@ class CmdModuleRemoveTest extends Specification { // Create spec file def specFile = new PipelineSpec(tempDir) - specFile.addModuleEntry('@nf-core/fastqc', '1.0.0') + specFile.addModuleEntry('nf-core/fastqc', '1.0.0') and: def cmd = new CmdModuleRemove() @@ -101,7 +101,7 @@ class CmdModuleRemoveTest extends Specification { and: def spec = new PipelineSpec(tempDir) - spec.getModules().get('@nf-core/fastqc') == '1.0.0' + spec.getModules().get('nf-core/fastqc') == '1.0.0' } def 'should keep files with -keep-files flag'() { @@ -112,7 +112,7 @@ class CmdModuleRemoveTest extends Specification { // Create spec file def specFile = new PipelineSpec(tempDir) - specFile.addModuleEntry('@nf-core/fastqc', '1.0.0') + specFile.addModuleEntry('nf-core/fastqc', '1.0.0') and: def cmd = new CmdModuleRemove() @@ -132,7 +132,7 @@ class CmdModuleRemoveTest extends Specification { and: def spec = new PipelineSpec(tempDir) - spec.getModules().get('@nf-core/fastqc') == null + spec.getModules().get('nf-core/fastqc') == null } def 'should fail when both keep flags are set'() { diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRunTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRunTest.groovy index c3741d0fe1..d93f58be53 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRunTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRunTest.groovy @@ -89,7 +89,7 @@ class CmdModuleRunTest extends Specification { def moduleRelease = new ModuleRelease() moduleRelease.version = '1.0.0' def module = new Module() - module.name = '@nf-core/test-module' + module.name = 'nf-core/test-module' module.latest = moduleRelease mockClient.fetchModule(_) >> module // Use wildcard to match any argument mockClient.downloadModule(_, _, _) >> { String name, String version, Path dest -> @@ -152,7 +152,7 @@ class CmdModuleRunTest extends Specification { def modulePackage = createModulePackage(moduleScript) def mockClient = Mock(ModuleRegistryClient) - mockClient.downloadModule('@nf-core/test-module', '2.0.0', _) >> { String name, String version, Path dest -> + mockClient.downloadModule('nf-core/test-module', '2.0.0', _) >> { String name, String version, Path dest -> Files.write(dest, modulePackage) return dest } diff --git a/modules/nextflow/src/test/groovy/nextflow/config/ModulesConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/config/ModulesConfigTest.groovy index d4c28cbf6f..f320aa472d 100644 --- a/modules/nextflow/src/test/groovy/nextflow/config/ModulesConfigTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/config/ModulesConfigTest.groovy @@ -38,11 +38,11 @@ class ModulesConfigTest extends Specification { def config = new ModulesConfig() when: - config.setVersion('@nf-core/fastqc', '1.0.0') + config.setVersion('nf-core/fastqc', '1.0.0') then: - config.getVersion('@nf-core/fastqc') == '1.0.0' - config.hasVersion('@nf-core/fastqc') + config.getVersion('nf-core/fastqc') == '1.0.0' + config.hasVersion('nf-core/fastqc') } def 'should return null for unconfigured module'() { @@ -50,33 +50,33 @@ class ModulesConfigTest extends Specification { def config = new ModulesConfig() when: - def version = config.getVersion('@nf-core/bwa') + def version = config.getVersion('nf-core/bwa') then: version == null - !config.hasVersion('@nf-core/bwa') + !config.hasVersion('nf-core/bwa') } def 'should override existing version'() { given: def config = new ModulesConfig() - config.setVersion('@nf-core/fastqc', '1.0.0') + config.setVersion('nf-core/fastqc', '1.0.0') when: - config.setVersion('@nf-core/fastqc', '2.0.0') + config.setVersion('nf-core/fastqc', '2.0.0') then: - config.getVersion('@nf-core/fastqc') == '2.0.0' + config.getVersion('nf-core/fastqc') == '2.0.0' } def 'should return unmodifiable map from getModules'() { given: def config = new ModulesConfig() - config.setVersion('@nf-core/fastqc', '1.0.0') + config.setVersion('nf-core/fastqc', '1.0.0') when: def modules = config.getAllModules() - modules.put('@nf-core/bwa', '2.0.0') + modules.put('nf-core/bwa', '2.0.0') then: thrown(UnsupportedOperationException) @@ -85,18 +85,18 @@ class ModulesConfigTest extends Specification { def 'should return all configured modules'() { given: def config = new ModulesConfig() - config.setVersion('@nf-core/fastqc', '1.0.0') - config.setVersion('@nf-core/bwa', '2.0.0') - config.setVersion('@myorg/custom', '0.5.0') + config.setVersion('nf-core/fastqc', '1.0.0') + config.setVersion('nf-core/bwa', '2.0.0') + config.setVersion('myorg/custom', '0.5.0') when: def modules = config.getAllModules() then: modules.size() == 3 - modules['@nf-core/fastqc'] == '1.0.0' - modules['@nf-core/bwa'] == '2.0.0' - modules['@myorg/custom'] == '0.5.0' + modules['nf-core/fastqc'] == '1.0.0' + modules['nf-core/bwa'] == '2.0.0' + modules['myorg/custom'] == '0.5.0' } def 'should handle empty initialization'() { @@ -105,7 +105,7 @@ class ModulesConfigTest extends Specification { then: config.getAllModules().isEmpty() - !config.hasVersion('@nf-core/fastqc') + !config.hasVersion('nf-core/fastqc') } def 'should store multiple versions independently'() { @@ -113,14 +113,14 @@ class ModulesConfigTest extends Specification { def config = new ModulesConfig() when: - config.setVersion('@nf-core/fastqc', '1.0.0') - config.setVersion('@nf-core/bwa', '2.0.0') - config.setVersion('@myorg/custom', '0.5.0') + config.setVersion('nf-core/fastqc', '1.0.0') + config.setVersion('nf-core/bwa', '2.0.0') + config.setVersion('myorg/custom', '0.5.0') then: - config.getVersion('@nf-core/fastqc') == '1.0.0' - config.getVersion('@nf-core/bwa') == '2.0.0' - config.getVersion('@myorg/custom') == '0.5.0' + config.getVersion('nf-core/fastqc') == '1.0.0' + config.getVersion('nf-core/bwa') == '2.0.0' + config.getVersion('myorg/custom') == '0.5.0' config.allModules.size() == 3 } @@ -129,13 +129,13 @@ class ModulesConfigTest extends Specification { def config = new ModulesConfig() when: - config.setVersion('@org-name/module-name', '1.0.0') - config.setVersion('@org_name/module_name', '2.0.0') + config.setVersion('org-name/module-name', '1.0.0') + config.setVersion('org_name/module_name', '2.0.0') config.setVersion('simple-module', '3.0.0') then: - config.getVersion('@org-name/module-name') == '1.0.0' - config.getVersion('@org_name/module_name') == '2.0.0' + config.getVersion('org-name/module-name') == '1.0.0' + config.getVersion('org_name/module_name') == '2.0.0' config.getVersion('simple-module') == '3.0.0' } @@ -144,41 +144,41 @@ class ModulesConfigTest extends Specification { def config = new ModulesConfig() when: - config.setVersion('@nf-core/fastqc', '1.0.0') - config.setVersion('@nf-core/bwa', 'v2.0.0') - config.setVersion('@nf-core/samtools', '1.0.0-beta') - config.setVersion('@nf-core/bowtie', '1.0.0-rc.1') + config.setVersion('nf-core/fastqc', '1.0.0') + config.setVersion('nf-core/bwa', 'v2.0.0') + config.setVersion('nf-core/samtools', '1.0.0-beta') + config.setVersion('nf-core/bowtie', '1.0.0-rc.1') then: - config.getVersion('@nf-core/fastqc') == '1.0.0' - config.getVersion('@nf-core/bwa') == 'v2.0.0' - config.getVersion('@nf-core/samtools') == '1.0.0-beta' - config.getVersion('@nf-core/bowtie') == '1.0.0-rc.1' + config.getVersion('nf-core/fastqc') == '1.0.0' + config.getVersion('nf-core/bwa') == 'v2.0.0' + config.getVersion('nf-core/samtools') == '1.0.0-beta' + config.getVersion('nf-core/bowtie') == '1.0.0-rc.1' } def 'should check if multiple modules have versions'() { given: def config = new ModulesConfig() - config.setVersion('@nf-core/fastqc', '1.0.0') - config.setVersion('@nf-core/bwa', '2.0.0') + config.setVersion('nf-core/fastqc', '1.0.0') + config.setVersion('nf-core/bwa', '2.0.0') expect: - config.hasVersion('@nf-core/fastqc') - config.hasVersion('@nf-core/bwa') - !config.hasVersion('@nf-core/samtools') + config.hasVersion('nf-core/fastqc') + config.hasVersion('nf-core/bwa') + !config.hasVersion('nf-core/samtools') } def 'should handle version updates'() { given: def config = new ModulesConfig() - config.setVersion('@nf-core/fastqc', '1.0.0') + config.setVersion('nf-core/fastqc', '1.0.0') when: - config.setVersion('@nf-core/fastqc', '1.1.0') - config.setVersion('@nf-core/fastqc', '2.0.0') + config.setVersion('nf-core/fastqc', '1.1.0') + config.setVersion('nf-core/fastqc', '2.0.0') then: - config.getVersion('@nf-core/fastqc') == '2.0.0' + config.getVersion('nf-core/fastqc') == '2.0.0' } def 'should maintain separate versions for different modules'() { @@ -186,12 +186,12 @@ class ModulesConfigTest extends Specification { def config = new ModulesConfig() when: - config.setVersion('@nf-core/fastqc', '1.0.0') - config.setVersion('@nf-core/bwa', '2.0.0') + config.setVersion('nf-core/fastqc', '1.0.0') + config.setVersion('nf-core/bwa', '2.0.0') then: - config.getVersion('@nf-core/fastqc') == '1.0.0' - config.getVersion('@nf-core/bwa') == '2.0.0' - config.getVersion('@nf-core/fastqc') != config.getVersion('@nf-core/bwa') + config.getVersion('nf-core/fastqc') == '1.0.0' + config.getVersion('nf-core/bwa') == '2.0.0' + config.getVersion('nf-core/fastqc') != config.getVersion('nf-core/bwa') } } diff --git a/modules/nextflow/src/test/groovy/nextflow/module/DefaultRemoteModuleResolverTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/DefaultRemoteModuleResolverTest.groovy new file mode 100644 index 0000000000..bda7b7f5ee --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/module/DefaultRemoteModuleResolverTest.groovy @@ -0,0 +1,46 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import nextflow.module.spi.RemoteModuleResolverProvider +import spock.lang.Specification + +/** + * Test for DefaultRemoteModuleResolver SPI implementation + * + * @author Jorge Ejarque + */ +class DefaultRemoteModuleResolverTest extends Specification { + + def 'should load resolver via SPI'() { + when: + def resolver = RemoteModuleResolverProvider.getInstance() + + then: + resolver != null + resolver.class.name == 'nextflow.module.DefaultRemoteModuleResolver' + resolver.priority == 0 + } + + def 'should return default priority'() { + given: + def resolver = new DefaultRemoteModuleResolver() + + expect: + resolver.getPriority() == 0 + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/module/InstalledModuleTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/InstalledModuleTest.groovy index 850466ab48..e526dde81f 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/InstalledModuleTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/InstalledModuleTest.groovy @@ -60,7 +60,7 @@ class InstalledModuleTest extends Specification { directory: moduleDir, mainFile: mainFile, manifestFile: metaFile, - checksumFile: moduleDir.resolve('.checksum'), + moduleInfoFile: moduleDir.resolve('.module-info'), expectedChecksum: actualChecksum, installedVersion: "0.0.1" ) @@ -95,7 +95,7 @@ class InstalledModuleTest extends Specification { directory: moduleDir, mainFile: mainFile, manifestFile: metaFile, - checksumFile: moduleDir.resolve('.checksum'), + moduleInfoFile: moduleDir.resolve('.module-info'), expectedChecksum: originalChecksum, installedVersion: "0.0.1" ) @@ -118,15 +118,15 @@ class InstalledModuleTest extends Specification { def metaFile = moduleDir.resolve('meta.yml') metaFile.text = 'name: test/module\nversion: 0.0.1' - def checksumFile = moduleDir.resolve('.checksum') - checksumFile.text = 'some-checksum' + def moduleInfoFile = moduleDir.resolve('.module-info') + // CORRUPTED check happens before .module-info check, so content doesn't matter here def installed = new InstalledModule( reference: new ModuleReference('test', 'module'), directory: moduleDir, mainFile: mainFile, manifestFile: metaFile, - checksumFile: checksumFile, + moduleInfoFile: moduleInfoFile, expectedChecksum: 'some-checksum' ) @@ -137,7 +137,7 @@ class InstalledModuleTest extends Specification { integrity == ModuleIntegrity.CORRUPTED } - def 'should report MISSING_CHECKSUM when checksum file absent'() { + def 'should report NO_REMOTE_MODULE when .module-info file absent'() { given: def moduleDir = tempDir.resolve('module') Files.createDirectories(moduleDir) @@ -148,15 +148,15 @@ class InstalledModuleTest extends Specification { def metaFile = moduleDir.resolve('meta.yml') metaFile.text = 'name: test/module\nversion: 0.0.1' - def checksumFile = moduleDir.resolve('.checksum') - // Don't create checksum file + def moduleInfoFile = moduleDir.resolve('.module-info') + // Don't create .module-info file def installed = new InstalledModule( reference: new ModuleReference('test', 'module'), directory: moduleDir, mainFile: mainFile, manifestFile: metaFile, - checksumFile: checksumFile, + moduleInfoFile: moduleInfoFile, expectedChecksum: null ) @@ -164,7 +164,7 @@ class InstalledModuleTest extends Specification { def integrity = installed.getIntegrity() then: - integrity == ModuleIntegrity.MISSING_CHECKSUM + integrity == ModuleIntegrity.NO_REMOTE_MODULE } def 'should handle checksum computation failure gracefully'() { @@ -178,16 +178,16 @@ class InstalledModuleTest extends Specification { def metaFile = moduleDir.resolve('meta.yml') metaFile.text = 'name: test/module\nversion: 0.0.1' - // Create checksum file with a value that won't match the computed checksum - def checksumFile = moduleDir.resolve('.checksum') - checksumFile.text = 'expected-checksum-that-will-not-match' + // Create .module-info with a checksum that won't match the computed checksum + ModuleChecksum.save(moduleDir, 'expected-checksum-that-will-not-match') + def moduleInfoFile = moduleDir.resolve('.module-info') def installed = new InstalledModule( reference: new ModuleReference('test', 'module'), directory: moduleDir, mainFile: mainFile, manifestFile: metaFile, - checksumFile: checksumFile, + moduleInfoFile: moduleInfoFile, expectedChecksum: 'expected-checksum-that-will-not-match' ) diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleChecksumTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleChecksumTest.groovy index 31c6600373..4bb9185681 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleChecksumTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleChecksumTest.groovy @@ -97,7 +97,7 @@ class ModuleChecksumTest extends Specification { checksum1 != checksum2 } - def 'should exclude .checksum file from computation'() { + def 'should exclude .module-info file from computation'() { given: def moduleDir = tempDir.resolve('module') Files.createDirectories(moduleDir) @@ -107,14 +107,14 @@ class ModuleChecksumTest extends Specification { // Compute initial checksum def checksum1 = ModuleChecksum.compute(moduleDir) - // Add .checksum file - moduleDir.resolve('.checksum').text = 'some-checksum-value' + // Add .module-info file + ModuleChecksum.save(moduleDir, 'some-checksum-value') // Compute checksum again def checksum2 = ModuleChecksum.compute(moduleDir) expect: - checksum1 == checksum2 // Should be the same, .checksum is ignored + checksum1 == checksum2 // Should be the same, .module-info is ignored } def 'should include subdirectories in checksum'() { @@ -139,7 +139,7 @@ class ModuleChecksumTest extends Specification { checksum1 != checksum2 // Checksums should differ } - def 'should save checksum to .checksum file'() { + def 'should save checksum to .module-info file'() { given: def moduleDir = tempDir.resolve('module') Files.createDirectories(moduleDir) @@ -149,17 +149,16 @@ class ModuleChecksumTest extends Specification { ModuleChecksum.save(moduleDir, checksumValue) then: - def checksumFile = moduleDir.resolve('.checksum') - Files.exists(checksumFile) - checksumFile.text.trim() == checksumValue + def moduleInfoFile = moduleDir.resolve('.module-info') + Files.exists(moduleInfoFile) + ModuleChecksum.load(moduleDir) == checksumValue } - def 'should load checksum from .checksum file'() { + def 'should load checksum from .module-info file'() { given: def moduleDir = tempDir.resolve('module') Files.createDirectories(moduleDir) - def checksumFile = moduleDir.resolve('.checksum') - checksumFile.text = 'abc123def456' + ModuleChecksum.save(moduleDir, 'abc123def456') when: def checksum = ModuleChecksum.load(moduleDir) diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleReferenceTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleReferenceTest.groovy index 64b0613be8..b906b83c9f 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleReferenceTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleReferenceTest.groovy @@ -26,34 +26,32 @@ import spock.lang.Specification */ class ModuleReferenceTest extends Specification { - def 'should parse valid module reference with @'() { + def 'should parse valid module reference without @'() { when: - def ref = ModuleReference.parse('@nf-core/fastqc') + def ref = ModuleReference.parse('nf-core/fastqc') then: ref.scope == 'nf-core' ref.name == 'fastqc' - ref.fullName == '@nf-core/fastqc' + ref.fullName == 'nf-core/fastqc' } - def 'should parse valid module reference without @'() { + def 'should reject module reference with @ prefix'() { when: - def ref = ModuleReference.parse('nf-core/fastqc') + ModuleReference.parse('@nf-core/fastqc') then: - ref.scope == 'nf-core' - ref.name == 'fastqc' - ref.fullName == '@nf-core/fastqc' + thrown(AbortOperationException) } def 'should parse module reference with multiple slashes'() { when: - def ref = ModuleReference.parse('@myorg/samtools/view') + def ref = ModuleReference.parse('myorg/samtools/view') then: ref.scope == 'myorg' ref.name == 'samtools/view' - ref.fullName == '@myorg/samtools/view' + ref.fullName == 'myorg/samtools/view' } def 'should reject invalid module reference without scope'() { @@ -80,7 +78,7 @@ class ModuleReferenceTest extends Specification { thrown(AbortOperationException) } - def 'should reject module reference with only @'() { + def 'should reject bare @ character'() { when: ModuleReference.parse('@') @@ -90,15 +88,15 @@ class ModuleReferenceTest extends Specification { def 'should reject module reference with only scope'() { when: - ModuleReference.parse('@nf-core/') + ModuleReference.parse('nf-core/') then: thrown(AbortOperationException) } - def 'should handle module reference with trailing slash'() { + def 'should reject module reference with trailing slash'() { when: - ModuleReference.parse('@nf-core/fastqc/') + ModuleReference.parse('nf-core/fastqc/') then: thrown(AbortOperationException) @@ -111,12 +109,12 @@ class ModuleReferenceTest extends Specification { then: ref.scope == 'nf-core' ref.name == 'fastqc' - ref.fullName == '@nf-core/fastqc' + ref.fullName == 'nf-core/fastqc' } def 'should handle scope names with hyphens'() { when: - def ref = ModuleReference.parse('@my-org/my-module') + def ref = ModuleReference.parse('my-org/my-module') then: ref.scope == 'my-org' @@ -125,7 +123,7 @@ class ModuleReferenceTest extends Specification { def 'should handle scope names with underscores'() { when: - def ref = ModuleReference.parse('@my_org/my_module') + def ref = ModuleReference.parse('my_org/my_module') then: ref.scope == 'my_org' @@ -134,7 +132,7 @@ class ModuleReferenceTest extends Specification { def 'should handle module names with numbers'() { when: - def ref = ModuleReference.parse('@nf-core/bwa-mem2') + def ref = ModuleReference.parse('nf-core/bwa-mem2') then: ref.scope == 'nf-core' @@ -143,9 +141,9 @@ class ModuleReferenceTest extends Specification { def 'should implement equals correctly'() { given: - def ref1 = ModuleReference.parse('@nf-core/fastqc') - def ref2 = ModuleReference.parse('@nf-core/fastqc') - def ref3 = ModuleReference.parse('@nf-core/multiqc') + def ref1 = ModuleReference.parse('nf-core/fastqc') + def ref2 = ModuleReference.parse('nf-core/fastqc') + def ref3 = ModuleReference.parse('nf-core/multiqc') expect: ref1 == ref2 @@ -154,8 +152,8 @@ class ModuleReferenceTest extends Specification { def 'should implement hashCode correctly'() { given: - def ref1 = ModuleReference.parse('@nf-core/fastqc') - def ref2 = ModuleReference.parse('@nf-core/fastqc') + def ref1 = ModuleReference.parse('nf-core/fastqc') + def ref2 = ModuleReference.parse('nf-core/fastqc') expect: ref1.hashCode() == ref2.hashCode() @@ -163,17 +161,17 @@ class ModuleReferenceTest extends Specification { def 'should implement toString correctly'() { given: - def ref = ModuleReference.parse('@nf-core/fastqc') + def ref = ModuleReference.parse('nf-core/fastqc') expect: - ref.toString() == '@nf-core/fastqc' + ref.toString() == 'nf-core/fastqc' } def 'should be usable as map key'() { given: - def ref1 = ModuleReference.parse('@nf-core/fastqc') - def ref2 = ModuleReference.parse('@nf-core/fastqc') - def ref3 = ModuleReference.parse('@nf-core/multiqc') + def ref1 = ModuleReference.parse('nf-core/fastqc') + def ref2 = ModuleReference.parse('nf-core/fastqc') + def ref3 = ModuleReference.parse('nf-core/multiqc') def map = [:] map[ref1] = 'value1' @@ -187,7 +185,7 @@ class ModuleReferenceTest extends Specification { def 'should handle org-style scopes'() { when: - def ref = ModuleReference.parse('@mycompany.io/custom-module') + def ref = ModuleReference.parse('mycompany.io/custom-module') then: ref.scope == 'mycompany.io' @@ -196,7 +194,7 @@ class ModuleReferenceTest extends Specification { def 'should reject module reference with spaces'() { when: - ModuleReference.parse('@nf-core/fast qc') + ModuleReference.parse('nf-core/fast qc') then: thrown(AbortOperationException) @@ -204,7 +202,7 @@ class ModuleReferenceTest extends Specification { def 'should reject module reference with special characters'() { when: - ModuleReference.parse('@nf-core/fastqc!') + ModuleReference.parse('nf-core/fastqc!') then: thrown(AbortOperationException) @@ -212,17 +210,17 @@ class ModuleReferenceTest extends Specification { def 'should handle deeply nested module names'() { when: - def ref = ModuleReference.parse('@nf-core/samtools/sort/parallel') + def ref = ModuleReference.parse('nf-core/samtools/sort/parallel') then: ref.scope == 'nf-core' ref.name == 'samtools/sort/parallel' - ref.fullName == '@nf-core/samtools/sort/parallel' + ref.fullName == 'nf-core/samtools/sort/parallel' } def 'should parse from string with leading/trailing whitespace'() { when: - def ref = ModuleReference.parse(' @nf-core/fastqc ') + def ref = ModuleReference.parse(' nf-core/fastqc ') then: ref.scope == 'nf-core' diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverTest.groovy index 8c145bb89c..ab959dfb57 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverTest.groovy @@ -88,7 +88,7 @@ class ModuleResolverTest extends Specification { name: nf-core/fastqc version: 1.0.0 ''' - moduleDir.resolve('.checksum').text = 'wrong-checksum' + ModuleChecksum.save(moduleDir, 'wrong-checksum') when: def result = resolver.resolve(reference, null, false) @@ -103,7 +103,7 @@ class ModuleResolverTest extends Specification { def 'should throw exception when version mismatch without auto-install'() { given: - def modulesConfig = new ModulesConfig(['@nf-core/fastqc': '2.0.0']) + def modulesConfig = new ModulesConfig(['nf-core/fastqc': '2.0.0']) def resolver = new ModuleResolver(tempDir, modulesConfig, null) def reference = new ModuleReference('nf-core', 'fastqc') def storage = new ModuleStorage(tempDir) @@ -119,7 +119,7 @@ class ModuleResolverTest extends Specification { // Compute and save correct checksum def checksum = ModuleChecksum.compute(moduleDir) - moduleDir.resolve('.checksum').text = checksum + ModuleChecksum.save(moduleDir, checksum) when: resolver.resolve(reference, null, false) @@ -152,7 +152,7 @@ class ModuleResolverTest extends Specification { // Compute and save correct checksum def checksum = ModuleChecksum.compute(moduleDir) - moduleDir.resolve('.checksum').text = checksum + ModuleChecksum.save(moduleDir, checksum) when: def result = resolver.resolve(reference, '1.0.0', false) @@ -178,7 +178,7 @@ class ModuleResolverTest extends Specification { name: nf-core/fastqc version: 1.0.0 ''' - moduleDir.resolve('.checksum').text = 'wrong-checksum' + ModuleChecksum.save(moduleDir, 'wrong-checksum') when: resolver.installModule(reference, '2.0.0', false) diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy index dab9517ddc..258c4c354b 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy @@ -50,7 +50,7 @@ class ModuleStorageTest extends Specification { def moduleDir = storage.getModuleDir(reference) then: - moduleDir == tempDir.resolve('modules/@nf-core/fastqc') + moduleDir == tempDir.resolve('modules/nf-core/fastqc') } def 'should check if module is installed'() { @@ -107,9 +107,9 @@ class ModuleStorageTest extends Specification { - fastqc '''.stripIndent() - // Create .checksum file - def checksumFile = moduleDir.resolve('.checksum') - checksumFile.text = 'abc123def456' + // Create .module-info file + ModuleChecksum.save(moduleDir, 'abc123def456') + def moduleInfoFile = moduleDir.resolve('.module-info') when: def installed = storage.getInstalledModule(reference) @@ -120,7 +120,7 @@ class ModuleStorageTest extends Specification { installed.directory == moduleDir installed.mainFile == mainFile installed.manifestFile == moduleDir.resolve('meta.yml') - installed.checksumFile == checksumFile + installed.moduleInfoFile == moduleInfoFile installed.expectedChecksum == 'abc123def456' installed.installedVersion == '1.0.0' } @@ -145,12 +145,12 @@ class ModuleStorageTest extends Specification { // Create meta.yml with version moduleDir.resolve('meta.yml').text = """ - name: ${ref.nameWithoutPrefix} + name: ${ref} version: 1.0.0 """.stripIndent() - // Create .checksum - moduleDir.resolve('.checksum').text = 'checksum' + // Create .module-info + ModuleChecksum.save(moduleDir, 'checksum') } when: @@ -158,7 +158,7 @@ class ModuleStorageTest extends Specification { then: installed.size() == 3 - installed*.reference.fullName.sort() == ['@myorg/custom', '@nf-core/fastqc', '@nf-core/multiqc'] + installed*.reference.fullName.sort() == ['myorg/custom', 'nf-core/fastqc', 'nf-core/multiqc'] } def 'should list nested modules recursively'() { @@ -182,14 +182,14 @@ class ModuleStorageTest extends Specification { // Create meta.yml with version moduleDir.resolve('meta.yml').text = """ - name: ${ref.nameWithoutPrefix} + name: ${ref} version: 1.0.0 description: Test module license: MIT """.stripIndent() - // Create .checksum - moduleDir.resolve('.checksum').text = 'checksum' + // Create .module-info + ModuleChecksum.save(moduleDir, 'checksum') } when: @@ -198,10 +198,10 @@ class ModuleStorageTest extends Specification { then: installed.size() == 4 installed*.reference.fullName.sort() == [ - '@myorg/tools/subtools/module', - '@nf-core/fastqc', - '@nf-core/gfatools/gfa2fa', - '@nf-core/gfatools/gfa2gfa' + 'myorg/tools/subtools/module', + 'nf-core/fastqc', + 'nf-core/gfatools/gfa2fa', + 'nf-core/gfatools/gfa2gfa' ] } @@ -234,7 +234,7 @@ class ModuleStorageTest extends Specification { installed.reference == reference installed.installedVersion == '1.0.0' Files.exists(installed.mainFile) - Files.exists(installed.checksumFile) + Files.exists(installed.moduleInfoFile) cleanup: packageFile?.delete() @@ -313,7 +313,7 @@ class ModuleStorageTest extends Specification { then: installed.expectedChecksum != null installed.expectedChecksum.length() > 0 - Files.exists(installed.checksumFile) + Files.exists(installed.moduleInfoFile) cleanup: packageFile?.delete() diff --git a/modules/nextflow/src/test/groovy/nextflow/script/IncludeDefTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/IncludeDefTest.groovy index b5e1b7425e..527cbbfbbd 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/IncludeDefTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/IncludeDefTest.groovy @@ -45,8 +45,8 @@ class IncludeDefTest extends Specification { expect: include.resolveModulePath('/abs/foo.nf') == '/abs/foo.nf' as Path - include.resolveModulePath('module.nf') == '/some/path/module.nf' as Path - include.resolveModulePath('foo/bar.nf') == '/some/path/foo/bar.nf' as Path + include.resolveModulePath('./module.nf') == '/some/path/module.nf' as Path + include.resolveModulePath('./foo/bar.nf') == '/some/path/foo/bar.nf' as Path when: include.resolveModulePath('http://foo.com/bar') @@ -66,17 +66,17 @@ class IncludeDefTest extends Specification { include.getOwnerPath() >> script when: - def result = include.realModulePath( 'mod-x.nf') + def result = include.realModulePath( './mod-x.nf') then: result == module when: - result = include.realModulePath('mod-x') + result = include.realModulePath('./mod-x') then: result == module when: - include.realModulePath('xyz') + include.realModulePath('./xyz') then: thrown(NoSuchFileException) @@ -98,21 +98,21 @@ class IncludeDefTest extends Specification { // when the module name reference a directory that contains // a file named 'main.nf', it's considered a module 'bundle' when: - def result = include.realModulePath('foo') + def result = include.realModulePath('./foo') then: result == module when: - include.realModulePath('bar') + include.realModulePath('./bar') then: thrown(NoSuchFileException) when: folder.resolve('bar').mkdir() - include.realModulePath('bar') + include.realModulePath('./bar') then: def e = thrown(ScriptCompilationException) - e.message == "Include 'bar' does not provide any module script -- the following path should contain a 'main.nf' script: '${folder.resolve('bar')}'" + e.message == "Include './bar' does not provide any module script -- the following path should contain a 'main.nf' script: '${folder.resolve('bar')}'" } def 'should check valid path' () { @@ -137,6 +137,16 @@ class IncludeDefTest extends Specification { when: include.checkValidPath('this/dir') then: + noExceptionThrown() // valid remote module reference (scope/name) + + when: + include.checkValidPath('nf-core/fastqc') + then: + noExceptionThrown() // valid remote module reference + + when: + include.checkValidPath('invalid!') + then: thrown(IllegalModulePath) when: diff --git a/modules/nf-lang/src/main/java/nextflow/module/spi/FallbackRemoteModuleResolver.java b/modules/nf-lang/src/main/java/nextflow/module/spi/FallbackRemoteModuleResolver.java new file mode 100644 index 0000000000..b438dbf7af --- /dev/null +++ b/modules/nf-lang/src/main/java/nextflow/module/spi/FallbackRemoteModuleResolver.java @@ -0,0 +1,45 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module.spi; + +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Fallback implementation of RemoteModuleResolver that is used when no other + * implementation is found via the SPI mechanism. + * + *

This implementation throws an exception with a helpful error message + * indicating that remote module resolution is not available. + * + * @author Jorge Ejarque + */ +public class FallbackRemoteModuleResolver implements RemoteModuleResolver { + + @Override + public Path resolve(String moduleName, Path baseDir) { + if (!Files.exists(baseDir.resolve(moduleName))) { + throw new IllegalStateException("Module '" + moduleName + "' not locally found at 'modules' folder - use 'nextflow install' to download module files"); + } + return baseDir.resolve(moduleName).resolve("main.nf"); + } + + @Override + public int getPriority() { + return Integer.MIN_VALUE; // Fallback has lowest possible priority + } +} \ No newline at end of file diff --git a/modules/nf-lang/src/main/java/nextflow/module/spi/RemoteModuleResolver.java b/modules/nf-lang/src/main/java/nextflow/module/spi/RemoteModuleResolver.java new file mode 100644 index 0000000000..5c6cf1fc9b --- /dev/null +++ b/modules/nf-lang/src/main/java/nextflow/module/spi/RemoteModuleResolver.java @@ -0,0 +1,68 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module.spi; + +import java.nio.file.Path; + +/** + * Service Provider Interface for resolving remote modules referenced with '@scope/name' syntax. + * + *

Implementations should handle: + *

    + *
  • Checking if a module is already installed locally
    • + *
  • Downloading modules from a registry if not present
    • + *
  • Version resolution and validation
    • + *
+ * + *

The interface follows the Java SPI pattern. Implementations should be registered + * in META-INF/services/nextflow.module.spi.RemoteModuleResolver + * + * @author Jorge Ejarque + */ +public interface RemoteModuleResolver { + + /** + * Resolve a remote module reference (e.g., '@scope/name') to a local path. + * + *

This method should: + *

    + *
  1. Parse the module reference
    2. + *
  2. Check if the module is already installed locally
    3. + *
  3. Download and install the module if not present (auto-install)
    4. + *
  4. Validate version constraints if specified
    5. + *
+ * + * @param moduleName The module reference string (e.g., '@scope/name' or '@scope/name@version') + * @param baseDir The base directory for the project (used to locate the modules directory) + * @return Path to the resolved module's main.nf file + * @throws IllegalArgumentException if the module reference is invalid or resolution fails + */ + Path resolve(String moduleName, Path baseDir); + + /** + * Get the priority of this resolver. Higher priority resolvers are tried first. + * + *

Use this to allow custom implementations to override the default resolver. + * The default implementation should return 0. Custom implementations can return + * positive values to take precedence. + * + * @return Priority value (higher = tried first), default should be 0 + */ + default int getPriority() { + return 0; + } +} \ No newline at end of file diff --git a/modules/nf-lang/src/main/java/nextflow/module/spi/RemoteModuleResolverProvider.java b/modules/nf-lang/src/main/java/nextflow/module/spi/RemoteModuleResolverProvider.java new file mode 100644 index 0000000000..da1b2da9e7 --- /dev/null +++ b/modules/nf-lang/src/main/java/nextflow/module/spi/RemoteModuleResolverProvider.java @@ -0,0 +1,92 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module.spi; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.ServiceLoader; + +/** + * Provider for accessing RemoteModuleResolver implementations via SPI. + * + *

This class uses the Java ServiceLoader mechanism to discover and load + * implementations of RemoteModuleResolver. It selects the implementation + * with the highest priority. + * + * @author Jorge Ejarque + */ +public class RemoteModuleResolverProvider { + + private static final Logger log = LoggerFactory.getLogger(RemoteModuleResolverProvider.class); + private static RemoteModuleResolver instance; + + /** + * Get the RemoteModuleResolver instance with the highest priority. + * + *

This method lazily loads and caches the resolver. It discovers all + * implementations via ServiceLoader and selects the one with the highest + * priority value. + * + *

If no implementations are found, returns the FallbackRemoteModuleResolver + * which throws an informative exception. + * + * @return The RemoteModuleResolver instance with highest priority + */ + public static synchronized RemoteModuleResolver getInstance() { + if (instance == null) { + instance = loadResolver(); + } + return instance; + } + + private static RemoteModuleResolver loadResolver() { + List resolvers = new ArrayList<>(); + ServiceLoader loader = ServiceLoader.load(RemoteModuleResolver.class); + + // Collect all available resolvers + for (RemoteModuleResolver resolver : loader) { + resolvers.add(resolver); + log.debug("Discovered RemoteModuleResolver: {} with priority {}", + resolver.getClass().getName(), resolver.getPriority()); + } + + // Sort by priority (highest first) + resolvers.sort(Comparator.comparingInt(RemoteModuleResolver::getPriority).reversed()); + + if (resolvers.isEmpty()) { + log.warn("No RemoteModuleResolver implementations found via SPI, using fallback"); + return new FallbackRemoteModuleResolver(); + } + + RemoteModuleResolver selected = resolvers.get(0); + log.debug("Selected RemoteModuleResolver: {} with priority {}", + selected.getClass().getName(), selected.getPriority()); + + return selected; + } + + /** + * Reset the cached instance. Used primarily for testing. + */ + public static synchronized void reset() { + instance = null; + } +} \ No newline at end of file diff --git a/modules/nf-lang/src/main/java/nextflow/script/control/ModuleResolver.java b/modules/nf-lang/src/main/java/nextflow/script/control/ModuleResolver.java index 5b76af4111..c52931142a 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/control/ModuleResolver.java +++ b/modules/nf-lang/src/main/java/nextflow/script/control/ModuleResolver.java @@ -23,6 +23,7 @@ import java.util.Set; import java.util.function.Function; +import nextflow.module.spi.RemoteModuleResolverProvider; import nextflow.script.ast.IncludeNode; import nextflow.script.ast.ScriptNode; import org.codehaus.groovy.control.SourceUnit; @@ -72,8 +73,18 @@ private SourceUnit resolveInclude(IncludeNode node, SourceUnit sourceUnit, Funct var source = node.source.getText(); if( source.startsWith("plugin/") ) return null; - var uri = sourceUnit.getSource().getURI(); - var includeUri = getIncludeUri(uri, source); + + var parent = Path.of(sourceUnit.getSource().getURI()).getParent(); + + // Resolve remote module paths (scope/name format, not starting with local prefixes) + if( isRemoteModule(source) ) { + var modules = Path.of("./modules"); + var resolver = RemoteModuleResolverProvider.getInstance(); + resolver.resolve(source, modules.getParent()); + parent = modules; + } + + var includeUri = getIncludeUri(parent, source); if( compiler.getSource(includeUri) != null ) return null; if( !Files.exists(Path.of(includeUri)) ) @@ -86,8 +97,15 @@ private SourceUnit resolveInclude(IncludeNode node, SourceUnit sourceUnit, Funct return includeSource; } - private static URI getIncludeUri(URI uri, String source) { - Path includePath = Path.of(uri).getParent().resolve(source); + static boolean isRemoteModule(String source) { + if( source.startsWith("/") || source.startsWith("./") || source.startsWith("../") ) + return false; + // Must match scope/name pattern: scope is lowercase alphanumeric with dots/underscores/hyphens + return source.matches("^[a-z0-9][a-z0-9._\\-]*/[a-z][a-z0-9._\\-]*(/[a-z][a-z0-9._\\-]*)*$"); + } + + private static URI getIncludeUri(Path parent, String source) { + Path includePath = parent.resolve(source); if( Files.isDirectory(includePath) ) includePath = includePath.resolve("main.nf"); else if( !source.endsWith(".nf") ) diff --git a/modules/nf-lang/src/main/java/nextflow/script/control/ResolveIncludeVisitor.java b/modules/nf-lang/src/main/java/nextflow/script/control/ResolveIncludeVisitor.java index 7dc9929bbe..b8bf711d93 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/control/ResolveIncludeVisitor.java +++ b/modules/nf-lang/src/main/java/nextflow/script/control/ResolveIncludeVisitor.java @@ -83,7 +83,10 @@ public void visitInclude(IncludeNode node) { setPlaceholderTargets(node); return; } - var includeUri = getIncludeUri(uri, source); + + var isRemoteModule = ModuleResolver.isRemoteModule(source); + var parent = isRemoteModule ? Path.of("modules") : Path.of(uri).getParent(); + var includeUri = getIncludeUri(parent, source); if( !isIncludeStale(node, includeUri) ) return; changed = true; @@ -121,8 +124,8 @@ private static void setPlaceholderTargets(IncludeNode node) { } } - private static URI getIncludeUri(URI uri, String source) { - Path includePath = Path.of(uri).getParent().resolve(source); + private static URI getIncludeUri(Path parent, String source) { + Path includePath = parent.resolve(source); if( Files.isDirectory(includePath) ) includePath = includePath.resolve("main.nf"); else if( !source.endsWith(".nf") ) diff --git a/settings.gradle b/settings.gradle index fe58cb245b..22284225f8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -50,3 +50,4 @@ include 'plugins:nf-k8s' include 'plugins:nf-seqera' //includeBuild '../sched' +//includeBuild('../plugin-registry') diff --git a/specs/251117-module-system/data-model.md b/specs/251117-module-system/data-model.md index 448509e52b..0d3ef4cf8a 100644 --- a/specs/251117-module-system/data-model.md +++ b/specs/251117-module-system/data-model.md @@ -164,7 +164,29 @@ registry { --- -### 5. PipelineSpec +### 5. DefaultRemoteModuleResolver (SPI) + +Bridges the DSL parser to the module resolution runtime. Class: `nextflow.module.DefaultRemoteModuleResolver`. + +```groovy +// Implements: nextflow.module.spi.RemoteModuleResolver (nf-lang) +class DefaultRemoteModuleResolver implements RemoteModuleResolver { + int getPriority() { return 0 } // Can be overridden by plugins with higher priority + + Path resolve(String moduleName, Path baseDir) { + // 1. Parse ModuleReference from "@scope/name" + // 2. Read version constraints from nextflow_spec.json / ModulesConfig + // 3. Call ModuleResolver.installModule(reference, version, autoInstall=true) + // 4. Return path to modules/@scope/name/main.nf + } +} +``` + +The SPI is loaded via Java `ServiceLoader` by `RemoteModuleResolverProvider` (in `nf-lang`), which selects the highest-priority implementation available. + +--- + +### 6. PipelineSpec Reads and writes `nextflow_spec.json` in the project root. Class: `nextflow.pipeline.PipelineSpec`. diff --git a/specs/251117-module-system/plan.md b/specs/251117-module-system/plan.md index 03ecb5cfff..e9d11d552f 100644 --- a/specs/251117-module-system/plan.md +++ b/specs/251117-module-system/plan.md @@ -77,17 +77,24 @@ modules/nextflow/src/main/groovy/nextflow/ │ └── RegistryConfig.groovy # registry{} config scope (fields: url, apiKey) ├── module/ │ ├── ModuleReference.groovy # @scope/name parser -│ ├── ModuleResolver.groovy # Core resolution logic +│ ├── ModuleResolver.groovy # Core resolution logic (version/integrity/install) │ ├── ModuleStorage.groovy # Local filesystem operations │ ├── ModuleRegistryClient.groovy # HTTP registry client │ ├── ModuleChecksum.groovy # SHA-256 integrity verification │ ├── ModuleSpec.groovy # Module manifest (meta.yaml) entity -│ └── InstalledModule.groovy # Installed module entity +│ ├── InstalledModule.groovy # Installed module entity +│ └── DefaultRemoteModuleResolver.groovy # SPI impl: bridges DSL parser → ModuleResolver └── pipeline/ └── PipelineSpec.groovy # nextflow_spec.json read/write modules/nf-lang/src/main/java/nextflow/script/ -└── ResolveIncludeVisitor.java # MODIFY: Add @scope/name detection +└── control/ResolveIncludeVisitor.java # MODIFIED: Delegates @scope/name to SPI resolver + +modules/nf-lang/src/main/java/nextflow/module/spi/ +├── RemoteModuleResolver.java # SPI interface (extensible by plugins) +├── RemoteModuleResolverProvider.java # ServiceLoader wrapper (singleton) +└── FallbackRemoteModuleResolver.java # Error fallback when no impl found + modules/nextflow/src/test/groovy/nextflow/ ├── cli/module/ @@ -105,7 +112,25 @@ tests/modules/ └── [other integration tests] ``` -**Structure Decision**: Implementation extends existing Nextflow core modules following modular architecture. New code in `modules/nextflow` for CLI and core logic. DSL parser extension in `modules/nf-lang`. No new plugins required. +**Structure Decision**: Implementation extends existing Nextflow core modules following modular architecture. New code in `modules/nextflow` for CLI and core logic. DSL parser extension in `modules/nf-lang` via SPI. No new plugins required. + +## Architecture Notes + +### Remote Module Inclusion — SPI Pattern + +The DSL parser (`ResolveIncludeVisitor`) detects the `@` prefix in `include` statements and delegates resolution to a `RemoteModuleResolver` SPI loaded via Java `ServiceLoader`. This keeps `nf-lang` decoupled from the runtime module resolution logic: + +``` +include { X } from '@nf-core/fastqc' + ↓ +ResolveIncludeVisitor (nf-lang) + source.startsWith("@") → RemoteModuleResolverProvider.getInstance().resolve(...) + ↓ +DefaultRemoteModuleResolver (nextflow module) + auto-installs via ModuleResolver if missing → returns Path to main.nf +``` + +The `RemoteModuleResolver` interface in `nf-lang` can be overridden by plugins with a higher priority value. ## Complexity Tracking diff --git a/specs/251117-module-system/research.md b/specs/251117-module-system/research.md index 40cb783e2b..d08005fc89 100644 --- a/specs/251117-module-system/research.md +++ b/specs/251117-module-system/research.md @@ -58,33 +58,40 @@ class CmdModule extends CmdBase implements UsageAware { **Research Question**: How to extend `include` statement parsing for registry modules? -**Decision**: Extend ResolveIncludeVisitor to detect `@` prefix and delegate to ModuleResolver +**Decision**: Extend `ResolveIncludeVisitor` to detect `@` prefix and delegate to a `RemoteModuleResolver` SPI loaded via Java `ServiceLoader` **Rationale**: -- IncludeNode already captures source path as string -- Detection: `source.startsWith('@')` distinguishes registry vs local paths -- Resolution happens at parse time (after plugin resolution) per ADR -- Preserves existing local file include behavior +- Keeps `nf-lang` decoupled from runtime module resolution (`nf-lang` has no dependency on `nextflow` module) +- SPI pattern allows plugins or custom implementations to override the default resolver +- Detection: `source.startsWith('@')` distinguishes registry vs local paths — preserves existing include behavior +- Resolution at parse time (after plugin resolution) per ADR -**Reference Implementation**: +**Implemented Architecture**: ``` -Location: modules/nf-lang/src/main/java/nextflow/script/ResolveIncludeVisitor.java -Extension Point: visitInclude() method -Pattern: - 1. Check if source starts with '@' - 2. If yes: call ModuleResolver.resolve(source, configuredVersion) - 3. ModuleResolver returns absolute path to modules/@scope/name/main.nf - 4. Continue with standard include processing +include { X } from '@scope/name' + ↓ +ResolveIncludeVisitor.visitInclude() [nf-lang] + source.startsWith("@") → RemoteModuleResolverProvider.getInstance().resolve(source, baseDir) + ↓ +RemoteModuleResolverProvider [nf-lang] + Java ServiceLoader discovers implementations; picks highest priority + ↓ +DefaultRemoteModuleResolver [nextflow module] + Calls ModuleResolver.installModule(reference, version, autoInstall=true) + Returns Path to modules/@scope/name/main.nf ``` **Key Files**: -- `IncludeNode.java` - AST representation -- `IncludeEntryNode.java` - Individual entries -- `ResolveIncludeVisitor.java` - Visitor for resolution +- `modules/nf-lang/src/main/java/nextflow/module/spi/RemoteModuleResolver.java` — SPI interface +- `modules/nf-lang/src/main/java/nextflow/module/spi/RemoteModuleResolverProvider.java` — ServiceLoader singleton +- `modules/nf-lang/src/main/java/nextflow/module/spi/FallbackRemoteModuleResolver.java` — error fallback +- `modules/nf-lang/src/main/java/nextflow/script/control/ResolveIncludeVisitor.java` — MODIFIED +- `modules/nextflow/src/main/groovy/nextflow/module/DefaultRemoteModuleResolver.groovy` — default impl **Alternatives Considered**: -- New ANTLR grammar token for `@`: Rejected - unnecessary parser complexity -- Dot file marker for local modules: Deferred to Open Questions in ADR +- New ANTLR grammar token for `@`: Rejected — unnecessary parser complexity +- Direct dependency from nf-lang to nextflow module: Rejected — circular dependency risk; SPI decouples cleanly +- Dot file marker for local modules: Deferred in ADR; current impl uses `@` for registry, `.`/`/` for local --- @@ -326,7 +333,7 @@ class ModuleChecksum { | Area | Decision | Key Reference | |------|----------|---------------| | CLI | JCommander subcommands; each extends CmdBase (ModuleRun extends CmdRun) | CmdModule.groovy | -| DSL Parser | Extend ResolveIncludeVisitor for `@scope/name` — pending | ResolveIncludeVisitor.java | +| DSL Parser | SPI pattern — ResolveIncludeVisitor delegates to RemoteModuleResolver; DefaultRemoteModuleResolver bridges to ModuleResolver | ResolveIncludeVisitor.java, RemoteModuleResolver.java | | Config | ModulesConfig + RegistryConfig (ConfigScope) | FusionConfig.groovy, ConfigScope.java | | Registry HTTP | ModuleRegistryClient using HxClient + npr-api models | HttpPluginRepository.groovy | | Authentication | `NXF_REGISTRY_TOKEN` env var or `registry.apiKey` config field (primary registry only) | RegistryConfig.groovy | @@ -342,4 +349,4 @@ class ModuleChecksum { 1. **Local vs managed module distinction**: Resolved — `@` prefix for registry modules only; local paths start with `.` or `/` 2. **Tool arguments**: Removed from ADR — not in scope 3. **Module version location**: Resolved — `nextflow_spec.json` (auto-managed by `module install`); `modules {}` block in `nextflow.config` supported as alternative -4. **DSL parser `@scope/name` include**: Pending (T017) \ No newline at end of file +4. **DSL parser `@scope/name` include**: ✅ Resolved — SPI pattern implemented (T017a-d) \ No newline at end of file From a2adf6e394d3153c0bbd0962b0948652925695d9 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Fri, 6 Mar 2026 09:14:55 -0600 Subject: [PATCH 20/23] update docs Signed-off-by: Ben Sherman --- docs/cli.md | 2 +- docs/module.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 0bc6655a9e..a6e7d6624c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -284,7 +284,7 @@ $ nextflow module install nf-core/fastqc $ nextflow module install nf-core/fastqc -version 1.0.0 ``` -After installation, module will be available in `modules/@nf-core/fastqc` and included in `nextflow_spec.json` +After installation, module will be available in `modules/nf-core/fastqc` and included in `nextflow_spec.json` Use the `-force` flag to reinstall a module even if local modifications exist. diff --git a/docs/module.md b/docs/module.md index 9f8afdd7b0..672e5733e1 100644 --- a/docs/module.md +++ b/docs/module.md @@ -304,10 +304,10 @@ $ nextflow module install nf-core/fastqc $ nextflow module install nf-core/fastqc -version 1.0.0 ``` -Installed modules are stored in the `modules/` directory and can be included using the registry syntax with the `@` prefix: +Installed modules are stored in the `modules/` directory and can be included by name instead of by relative path: ```nextflow -include { FASTQC } from '@nf-core/fastqc' +include { FASTQC } from 'nf-core/fastqc' workflow { reads = Channel.fromFilePairs('data/*_{1,2}.fastq.gz') @@ -332,8 +332,8 @@ Module versions are tracked in `nextflow_spec.json` in your project directory: ```json { "modules": { - "@nf-core/fastqc": "1.0.0", - "@nf-core/bwa-align": "1.2.0" + "nf-core/fastqc": "1.0.0", + "nf-core/bwa-align": "1.2.0" } } ``` @@ -361,7 +361,7 @@ Nextflow automatically verifies module integrity using checksums. If you modify ```console $ nextflow module install nf-core/fastqc -version 1.1.0 -Warning: Module @nf-core/fastqc has local modifications. Use -force to override. +Warning: Module nf-core/fastqc has local modifications. Use -force to override. ``` Use the `-force` flag to override local modifications when needed. @@ -442,7 +442,7 @@ Registry modules follow a standard directory structure: ``` modules/ -└── @scope/ +└── scope/ └── module-name/ ├── .checksum # Integrity checksum (generated automatically) ├── README.md # Documentation (required for publishing) From 292e4f2260f2b8a4e5a58ebe6b6a5b8fb7486a84 Mon Sep 17 00:00:00 2001 From: jorgee Date: Fri, 6 Mar 2026 20:10:49 +0100 Subject: [PATCH 21/23] change module remove behaviour to avoid unexpected removes Signed-off-by: jorgee --- docs/cli.md | 3 +- docs/module.md | 2 +- docs/reference/cli.md | 7 +- .../cli/module/CmdModuleRemove.groovy | 50 +++----- .../nextflow/module/ModuleChecksum.groovy | 3 +- .../nextflow/module/ModuleResolver.groovy | 6 +- .../nextflow/module/ModuleStorage.groovy | 41 ++++-- .../cli/module/CmdModuleRemoveTest.groovy | 120 +++++++++++++----- .../nextflow/module/ModuleResolverTest.groovy | 2 +- .../nextflow/module/ModuleStorageTest.groovy | 94 +++++++++++++- 10 files changed, 241 insertions(+), 87 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index a6e7d6624c..a8aba93cf3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -368,11 +368,10 @@ Use this to clean up unused modules, free disk space, or remove deprecated modul ```console $ nextflow module remove nf-core/fastqc -$ nextflow module remove nf-core/fastqc -keep-config $ nextflow module remove nf-core/fastqc -keep-files ``` -By default, both local files and configuration entries are removed. Use `-keep-config` to preserve version information in `nextflow_spec.json`, or `-keep-files` to remove only the configuration entry while keeping local files. +By default, both local files and configuration entries are removed. Use `-keep-files` to remove the configuration entry and `.module-info` while keeping local files. See {ref}`cli-module-remove` for more information. diff --git a/docs/module.md b/docs/module.md index 672e5733e1..4c03513916 100644 --- a/docs/module.md +++ b/docs/module.md @@ -377,7 +377,7 @@ $ nextflow module remove nf-core/fastqc By default, both the local module files and the entry in `nextflow_spec.json` are removed. Use the flags below to control this behaviour: - `-keep-files` — Remove the entry from `nextflow_spec.json` but keep the local module files -- `-keep-config` — Remove the local module files but keep the entry in `nextflow_spec.json` +- `-force` — Force removal even if the module has no `.module-info` file (i.e. not installed from a registry) or has local modifications ### Viewing module information diff --git a/docs/reference/cli.md b/docs/reference/cli.md index aad0ecceaf..05feb8d56b 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1284,8 +1284,8 @@ The `module` command provides a comprehensive system for managing reusable, regi : By default, removes both local files and configuration entries. Use options to control what gets removed. : The following options are available: - `-keep-config` - : Keep the version entry in `nextflow_spec.json` but delete local files from the `modules/` directory. + `-force` + : Force removal even if the module has no `.module-info` file (i.e. not installed from a registry) or has local modifications. `-keep-files` : Remove the version entry from `nextflow_spec.json` but keep local files in the `modules/` directory. @@ -1296,9 +1296,6 @@ The `module` command provides a comprehensive system for managing reusable, regi # Remove module completely $ nextflow module remove nf-core/fastqc - # Delete files but keep version config - $ nextflow module remove nf-core/fastqc -keep-config - # Remove from config but keep local files $ nextflow module remove nf-core/fastqc -keep-files ``` diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy index 3c3ab9d026..8b397bf0b8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy @@ -27,6 +27,7 @@ import nextflow.module.ModuleStorage import nextflow.pipeline.PipelineSpec import nextflow.util.TestOnly +import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths @@ -43,12 +44,12 @@ class CmdModuleRemove extends CmdBase { @Parameter(description = "", required = true) List args - @Parameter(names = ["-keep-config"], description = "Remove local files but keep the entry in nextflow_spec.json", arity = 0) - boolean keepConfig = false - @Parameter(names = ["-keep-files"], description = "Remove from config but keep local files", arity = 0) boolean keepFiles = false + @Parameter(names = ["-force"], description = "Force remove", arity = 0) + boolean force = false + @TestOnly protected Path root @@ -62,10 +63,8 @@ class CmdModuleRemove extends CmdBase { if( !args || args.size() != 1 ) { throw new AbortOperationException("Incorrect number of arguments") } - - // Validate flags - if( keepConfig && keepFiles ) { - throw new AbortOperationException("Cannot use both -keep-config and -keep-files flags together") + if( keepFiles && force ) { + throw new AbortOperationException("Cannot use both -keep-files and -force options") } def reference = ModuleReference.parse(args[0]) @@ -85,43 +84,32 @@ class CmdModuleRemove extends CmdBase { // Remove local files unless -keep-files is set if( !keepFiles ) { - println "Removing module files for ${reference}..." - filesRemoved = storage.removeModule(reference) + filesRemoved = storage.removeModule(reference, force) if( filesRemoved ) { - println "Module files removed successfully" + println "Module ${reference} files removed successfully" } else { - println "Module ${reference} was not installed locally" + println "Module ${reference} not found locally" } } else { - println "Keeping module files for ${reference} (due to -keep-files flag)" - } - - // Remove config entry unless -keep-config is set - if( !keepConfig ) { - println "Removing module entry from nextflow_spec.json..." - configRemoved = specFile.removeModuleEntry(reference.fullName) - if( configRemoved ) { - println "Module entry removed from configuration" - } else { - println "Module ${reference} was not configured in nextflow_spec.json" + println "Keeping module files for ${reference} (-keep-files flag)" + final moduleInfo = storage.getModuleInfo(reference) + if( Files.exists(moduleInfo) ) { + Files.delete(moduleInfo) } - } else { - println "Keeping module entry in nextflow_spec.json (due to -keep-config flag)" } - // Summary - if( filesRemoved || configRemoved ) { - println "\nModule ${reference} removal completed" - } else { - println "\nModule ${reference} was not found" + + configRemoved = specFile.removeModuleEntry(reference.fullName) + if( configRemoved ) { + println "Module ${reference} entry removed from spec file" } + } catch( AbortOperationException e ) { throw e } catch( Exception e ) { - log.error("Failed to remove module", e) - throw new AbortOperationException("Removal failed: ${e.message}", e) + throw new AbortOperationException("Failed to remove module $reference: ${e.message}", e) } } } diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy index 94235bc696..861020983b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy @@ -23,6 +23,8 @@ import java.nio.file.Files import java.nio.file.Path import java.security.MessageDigest +import static nextflow.module.ModuleStorage.MODULE_INFO_FILE + /** * Utility class for computing SHA-256 checksums of module directories * @@ -33,7 +35,6 @@ import java.security.MessageDigest class ModuleChecksum { public static final String CHECKSUM_ALGORITHM = "SHA-256" - public static final String MODULE_INFO_FILE = ".module-info" /** * Compute the SHA-256 checksum of a module directory diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy index a5a55c2fe8..b9818d85fa 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy @@ -142,13 +142,13 @@ class ModuleResolver { if( integrity == ModuleIntegrity.MODIFIED && !force ) { throw new AbortOperationException( "Module ${reference} has local modifications. " + - "Use --force to override, or save your changes first." + "Use '-force' to override, or save your changes first." ) } if( integrity == ModuleIntegrity.NO_REMOTE_MODULE && !force ) { throw new AbortOperationException( - " Folder 'modules/${reference}' already exists and is not a valid remote module. " + - "Use --force to override, or save your changes first." + "Folder 'modules/${reference}' already exists and is not a valid remote module. " + + "Use '-force' to override, or save your changes first." ) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy index 9cffaa0279..c59405b223 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy @@ -43,6 +43,7 @@ import java.util.zip.ZipInputStream class ModuleStorage { public static final String MODULE_MANIFEST_FILE = "meta.yml" public static final String MODULE_README_FILE = "README.md" + public static final String MODULE_INFO_FILE = ".module-info" private final Path modulesDir /** @@ -73,6 +74,16 @@ class ModuleStorage { return modulesDir.resolve(reference.scope).resolve(reference.name) } + /** + * Get the module info path for a specific module + * + * @param reference The module reference + * @return The module info path + */ + Path getModuleInfo(ModuleReference reference) { + return modulesDir.resolve(reference.scope).resolve(reference.name).resolve(MODULE_INFO_FILE) + } + /** * Check if a module is installed locally * @@ -101,7 +112,7 @@ class ModuleStorage { directory: moduleDir, mainFile: moduleDir.resolve(Const.DEFAULT_MAIN_FILE_NAME), manifestFile: moduleDir.resolve(MODULE_MANIFEST_FILE), - moduleInfoFile: moduleDir.resolve(ModuleChecksum.MODULE_INFO_FILE), + moduleInfoFile: moduleDir.resolve(MODULE_INFO_FILE), ) // Load checksum if available @@ -125,7 +136,7 @@ class ModuleStorage { try( final walkStream = Files.walk(modulesDir) ) { walkStream .filter { Path path -> Files.isDirectory(path) } - .filter { Path path -> Files.exists(path.resolve(ModuleChecksum.MODULE_INFO_FILE)) } + .filter { Path path -> Files.exists(path.resolve(MODULE_INFO_FILE)) } .each { Path moduleDir -> try { def rel = modulesDir.relativize(moduleDir) @@ -199,21 +210,33 @@ class ModuleStorage { * Remove an installed module * * @param reference The module reference + * @param force Force local module folder * @return true if module was removed, false if not installed */ - boolean removeModule(ModuleReference reference) { - def moduleDir = getModuleDir(reference) + boolean removeModule(ModuleReference reference, boolean force) { + final installed = getInstalledModule(reference) + if( !installed ) + return + final integrity = installed.integrity + if( integrity == ModuleIntegrity.NO_REMOTE_MODULE && !force ) { + throw new AbortOperationException( + "Folder 'modules/${reference}' already exists and is not a valid remote module ($MODULE_INFO_FILE missing). " + + "Use '-force' to remove, or save your changes first.") + } - if (!Files.exists(moduleDir)) { - return false + if( integrity == ModuleIntegrity.MODIFIED && !force ) { + throw new AbortOperationException( + "Module ${reference} has local modifications. " + + "Use '-force' to remove, or save your changes first." + ) } try { - FileHelper.deletePath(moduleDir) + FileHelper.deletePath(installed.directory) log.debug "Removed module: ${reference}" // Clean up empty scope directory - def scopeDir = moduleDir.parent + def scopeDir = installed.directory.parent if (Files.exists(scopeDir) && isEmpty(scopeDir)) { Files.delete(scopeDir) } @@ -377,7 +400,7 @@ class ModuleStorage { try ( def tarStream = Files.list(currentPath)) { tarStream.each { Path path -> // Skip .module-info file when creating bundle - if (path.fileName.toString() == ModuleChecksum.MODULE_INFO_FILE) { + if (path.fileName.toString() == MODULE_INFO_FILE) { return } diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRemoveTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRemoveTest.groovy index 0b4972c3ef..27c6b85242 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRemoveTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRemoveTest.groovy @@ -17,6 +17,7 @@ package nextflow.cli.module import nextflow.exception.AbortOperationException +import nextflow.module.ModuleChecksum import nextflow.module.ModuleReference import nextflow.module.ModuleStorage import nextflow.pipeline.PipelineSpec @@ -47,7 +48,7 @@ class CmdModuleRemoveTest extends Specification { given: def storage = new ModuleStorage(tempDir) def reference = new ModuleReference('nf-core', 'fastqc') - def moduleDir = createTestModule(storage, reference) + def moduleDir = createTestModule(storage, reference, true) // Create spec file with module entry def specFile = new PipelineSpec(tempDir) @@ -63,10 +64,8 @@ class CmdModuleRemoveTest extends Specification { def output = capture.toString() then: - output.contains('Removing module files') - output.contains('Module files removed successfully') - output.contains('Removing module entry from nextflow_spec.json') - output.contains('Module entry removed from configuration') + output.contains('Module nf-core/fastqc files removed successfully') + output.contains('Module nf-core/fastqc entry removed from spec') !Files.exists(moduleDir) and: @@ -74,11 +73,11 @@ class CmdModuleRemoveTest extends Specification { spec.getModules().get('nf-core/fastqc') == null } - def 'should keep config with -keep-config flag'() { + def 'should keep files with -keep-files flag'() { given: def storage = new ModuleStorage(tempDir) def reference = new ModuleReference('nf-core', 'fastqc') - def moduleDir = createTestModule(storage, reference) + def moduleDir = createTestModule(storage, reference, true) // Create spec file def specFile = new PipelineSpec(tempDir) @@ -87,7 +86,7 @@ class CmdModuleRemoveTest extends Specification { and: def cmd = new CmdModuleRemove() cmd.args = ['nf-core/fastqc'] - cmd.keepConfig = true + cmd.keepFiles = true cmd.root = tempDir when: @@ -95,29 +94,62 @@ class CmdModuleRemoveTest extends Specification { def output = capture.toString() then: - output.contains('Removing module files') - output.contains('Keeping module entry in nextflow_spec.json') - !Files.exists(moduleDir) + output.contains('Keeping module files for nf-core/fastqc ') + output.contains('Module nf-core/fastqc entry removed from spec file') + Files.exists(moduleDir) + Files.exists(moduleDir.resolve('main.nf')) + !Files.exists(moduleDir.resolve(ModuleStorage.MODULE_INFO_FILE)) and: def spec = new PipelineSpec(tempDir) - spec.getModules().get('nf-core/fastqc') == '1.0.0' + spec.getModules().get('nf-core/fastqc') == null } - def 'should keep files with -keep-files flag'() { + def 'should fail when both keep-files and force are set'() { + given: + def cmd = new CmdModuleRemove() + cmd.args = ['nf-core/fastqc'] + cmd.force = true + cmd.keepFiles = true + cmd.root = tempDir + + when: + cmd.run() + + then: + def e = thrown(AbortOperationException) + e.message.contains('Cannot use both -keep-files and -force options') + } + + def 'should fail to remove module without .module-info when force not set'() { given: def storage = new ModuleStorage(tempDir) def reference = new ModuleReference('nf-core', 'fastqc') - def moduleDir = createTestModule(storage, reference) + createTestModule(storage, reference) // no .module-info created - // Create spec file - def specFile = new PipelineSpec(tempDir) - specFile.addModuleEntry('nf-core/fastqc', '1.0.0') + and: + def cmd = new CmdModuleRemove() + cmd.args = ['nf-core/fastqc'] + cmd.root = tempDir + + when: + cmd.run() + + then: + def e = thrown(AbortOperationException) + e.message.contains('.module-info missing') + } + + def 'should force remove module without .module-info'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def moduleDir = createTestModule(storage, reference) // no .module-info created and: def cmd = new CmdModuleRemove() cmd.args = ['nf-core/fastqc'] - cmd.keepFiles = true + cmd.force = true cmd.root = tempDir when: @@ -125,30 +157,52 @@ class CmdModuleRemoveTest extends Specification { def output = capture.toString() then: - output.contains('Keeping module files') - output.contains('Removing module entry from nextflow_spec.json') - Files.exists(moduleDir) - Files.exists(moduleDir.resolve('main.nf')) + output.contains('Module nf-core/fastqc files removed successfully') + !Files.exists(moduleDir) + } + + def 'should fail to remove modified module when force not set'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def moduleDir = createTestModule(storage, reference, true) + // Modify a file to cause checksum mismatch + moduleDir.resolve('main.nf').text = 'process MODIFIED { }' and: - def spec = new PipelineSpec(tempDir) - spec.getModules().get('nf-core/fastqc') == null + def cmd = new CmdModuleRemove() + cmd.args = ['nf-core/fastqc'] + cmd.root = tempDir + + when: + cmd.run() + + then: + def e = thrown(AbortOperationException) + e.message.contains('local modifications') } - def 'should fail when both keep flags are set'() { + def 'should force remove modified module'() { given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def moduleDir = createTestModule(storage, reference, true) + // Modify a file to cause checksum mismatch + moduleDir.resolve('main.nf').text = 'process MODIFIED { }' + + and: def cmd = new CmdModuleRemove() cmd.args = ['nf-core/fastqc'] - cmd.keepConfig = true - cmd.keepFiles = true + cmd.force = true cmd.root = tempDir when: cmd.run() + def output = capture.toString() then: - def e = thrown(AbortOperationException) - e.message.contains('Cannot use both -keep-config and -keep-files') + output.contains('Module nf-core/fastqc files removed successfully') + !Files.exists(moduleDir) } def 'should handle removing non-existent module'() { @@ -162,7 +216,7 @@ class CmdModuleRemoveTest extends Specification { def output = capture.toString() then: - output.contains('was not installed locally') || output.contains('was not found') + output.contains('Module nf-core/nonexistent not found locally') } def 'should fail with no arguments'() { @@ -191,7 +245,7 @@ class CmdModuleRemoveTest extends Specification { thrown(AbortOperationException) } - private Path createTestModule(ModuleStorage storage, ModuleReference reference) { + private Path createTestModule(ModuleStorage storage, ModuleReference reference, boolean withModuleInfo = false) { def moduleDir = storage.getModuleDir(reference) Files.createDirectories(moduleDir) @@ -218,6 +272,10 @@ class CmdModuleRemoveTest extends Specification { description: FastQC quality control '''.stripIndent() + if( withModuleInfo ) { + ModuleChecksum.save(moduleDir, ModuleChecksum.compute(moduleDir)) + } + return moduleDir } } diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverTest.groovy index ab959dfb57..08aa6809f9 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverTest.groovy @@ -186,7 +186,7 @@ class ModuleResolverTest extends Specification { then: def e = thrown(AbortOperationException) e.message.contains('local modifications') - e.message.contains('--force') + e.message.contains('-force') cleanup: FileHelper.deletePath(moduleDir) diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy index 258c4c354b..06ecde6337 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy @@ -273,15 +273,17 @@ class ModuleStorageTest extends Specification { def reference = new ModuleReference('nf-core', 'fastqc') def moduleDir = storage.getModuleDir(reference) - // Create module + // Create valid module with correct checksum Files.createDirectories(moduleDir) moduleDir.resolve('main.nf').text = 'process TEST { }' + moduleDir.resolve('meta.yml').text = 'name: nf-core/fastqc\nversion: 1.0.0\n' + ModuleChecksum.save(moduleDir, ModuleChecksum.compute(moduleDir)) expect: Files.exists(moduleDir) when: - def removed = storage.removeModule(reference) + def removed = storage.removeModule(reference, false) then: removed @@ -294,12 +296,98 @@ class ModuleStorageTest extends Specification { def reference = new ModuleReference('nf-core', 'nonexistent') when: - def removed = storage.removeModule(reference) + def removed = storage.removeModule(reference, false) then: !removed } + def 'should throw when removing module without .module-info and force is false'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def moduleDir = storage.getModuleDir(reference) + + // Create module WITHOUT .module-info (but with required files for NO_REMOTE_MODULE integrity) + Files.createDirectories(moduleDir) + moduleDir.resolve('main.nf').text = 'process TEST { }' + moduleDir.resolve('meta.yml').text = 'name: nf-core/fastqc\nversion: 1.0.0\n' + + when: + storage.removeModule(reference, false) + + then: + def e = thrown(nextflow.exception.AbortOperationException) + e.message.contains('.module-info missing') + Files.exists(moduleDir) + } + + def 'should force remove module without .module-info'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def moduleDir = storage.getModuleDir(reference) + + // Create module WITHOUT .module-info (but with required files for NO_REMOTE_MODULE integrity) + Files.createDirectories(moduleDir) + moduleDir.resolve('main.nf').text = 'process TEST { }' + moduleDir.resolve('meta.yml').text = 'name: nf-core/fastqc\nversion: 1.0.0\n' + + expect: + Files.exists(moduleDir) + + when: + def removed = storage.removeModule(reference, true) + + then: + removed + !Files.exists(moduleDir) + } + + def 'should throw when removing modified module and force is false'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def moduleDir = storage.getModuleDir(reference) + + // Create module with mismatched checksum (simulates local modification) + Files.createDirectories(moduleDir) + moduleDir.resolve('main.nf').text = 'process TEST { }' + moduleDir.resolve('meta.yml').text = 'name: nf-core/fastqc\nversion: 1.0.0\n' + ModuleChecksum.save(moduleDir, 'stale-checksum-from-install') + // Now modify a file to cause checksum mismatch + moduleDir.resolve('main.nf').text = 'process TEST_MODIFIED { }' + + when: + storage.removeModule(reference, false) + + then: + def e = thrown(nextflow.exception.AbortOperationException) + e.message.contains('local modifications') + Files.exists(moduleDir) + } + + def 'should force remove modified module'() { + given: + def storage = new ModuleStorage(tempDir) + def reference = new ModuleReference('nf-core', 'fastqc') + def moduleDir = storage.getModuleDir(reference) + + // Create module with mismatched checksum (simulates local modification) + Files.createDirectories(moduleDir) + moduleDir.resolve('main.nf').text = 'process TEST { }' + moduleDir.resolve('meta.yml').text = 'name: nf-core/fastqc\nversion: 1.0.0\n' + ModuleChecksum.save(moduleDir, 'stale-checksum-from-install') + moduleDir.resolve('main.nf').text = 'process TEST_MODIFIED { }' + + when: + def removed = storage.removeModule(reference, true) + + then: + removed + !Files.exists(moduleDir) + } + def 'should compute and save checksum on install'() { given: def storage = new ModuleStorage(tempDir) From e96c6ef9d0b8827cad29b1ae9f9fd9e431238a56 Mon Sep 17 00:00:00 2001 From: Jorge Ejarque Date: Mon, 9 Mar 2026 09:30:11 +0100 Subject: [PATCH 22/23] Apply suggestions from code review [ci skip] Co-authored-by: Chris Hakkaart Signed-off-by: Jorge Ejarque --- docs/module.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/module.md b/docs/module.md index 4c03513916..bf2c07969a 100644 --- a/docs/module.md +++ b/docs/module.md @@ -304,7 +304,7 @@ $ nextflow module install nf-core/fastqc $ nextflow module install nf-core/fastqc -version 1.0.0 ``` -Installed modules are stored in the `modules/` directory and can be included by name instead of by relative path: +Installed modules are stored in the `modules/` directory and can be included by name instead of a relative path: ```nextflow include { FASTQC } from 'nf-core/fastqc' @@ -402,9 +402,9 @@ The argument can be either a `scope/name` reference (for an already-installed mo Your module directory must include: -- `main.nf` - The module entry point -- `meta.yaml` - Module metadata (name, description, version, etc.) -- `README.md` - Module documentation +- `main.nf`: The module entry point +- `meta.yaml`: Module metadata (name, description, version, etc.) +- `README.md`: Module documentation Authentication is required for publishing and can be provided via the `NXF_REGISTRY_TOKEN` environment variable or in your configuration: @@ -454,4 +454,4 @@ modules/ The `modules/` directory should be committed to your Git repository to ensure reproducibility. -See the {ref}`cli-page` documentation for complete details on all module commands. +See the {ref}`cli-page` documentation for more information about module commands. From 0b83a2c0a967ead6913829bc1169c1b192191361 Mon Sep 17 00:00:00 2001 From: jorgee Date: Tue, 10 Mar 2026 13:51:18 +0100 Subject: [PATCH 23/23] remove pipeline spec and modules config Signed-off-by: jorgee --- docs/cli.md | 4 +- docs/module.md | 18 +- docs/reference/cli.md | 5 +- .../cli/module/CmdModuleInstall.groovy | 13 +- .../nextflow/cli/module/CmdModuleList.groovy | 3 +- .../cli/module/CmdModulePublish.groovy | 6 +- .../cli/module/CmdModuleRemove.groovy | 15 +- .../nextflow/cli/module/CmdModuleRun.groovy | 10 +- .../nextflow/config/ModulesConfig.groovy | 95 --------- .../module/DefaultRemoteModuleResolver.groovy | 25 +-- .../nextflow/module/InstalledModule.groovy | 1 + .../nextflow/module/ModuleChecksum.groovy | 18 +- .../groovy/nextflow/module/ModuleInfo.groovy | 109 ++++++++++ .../module/ModuleRegistryClient.groovy | 8 +- .../nextflow/module/ModuleResolver.groovy | 31 ++- .../nextflow/module/ModuleStorage.groovy | 11 +- .../nextflow/pipeline/PipelineSpec.groovy | 146 ------------- .../cli/module/CmdModuleInstallTest.groovy | 38 ++-- .../cli/module/CmdModuleListTest.groovy | 3 +- .../cli/module/CmdModuleRemoveTest.groovy | 24 +-- .../nextflow/config/ModulesConfigTest.groovy | 197 ------------------ .../nextflow/module/ModuleInfoTest.groovy | 183 ++++++++++++++++ .../module/ModuleRegistryClientTest.groovy | 4 +- .../nextflow/module/ModuleResolverTest.groovy | 9 +- .../nextflow/module/ModuleStorageTest.groovy | 20 +- 25 files changed, 382 insertions(+), 614 deletions(-) delete mode 100644 modules/nextflow/src/main/groovy/nextflow/config/ModulesConfig.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/module/ModuleInfo.groovy delete mode 100644 modules/nextflow/src/main/groovy/nextflow/pipeline/PipelineSpec.groovy delete mode 100644 modules/nextflow/src/test/groovy/nextflow/config/ModulesConfigTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/module/ModuleInfoTest.groovy diff --git a/docs/cli.md b/docs/cli.md index a8aba93cf3..3391288f3a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -275,7 +275,7 @@ Use these commands to discover modules in registries, install them into your pro ### Installing modules -The `module install` command downloads modules from a registry and makes them available in your workflow. Modules are stored locally in the `modules/` directory and version information is tracked in `nextflow_spec.json`. +The `module install` command downloads modules from a registry and makes them available in your workflow. Modules are stored locally in the `modules/` directory. An additional `.module-info` file is created during to store installation information such as the module checksum at installation and the registry URL. Use this to add reusable modules to your pipeline, manage module versions, or update modules to newer versions. @@ -284,7 +284,7 @@ $ nextflow module install nf-core/fastqc $ nextflow module install nf-core/fastqc -version 1.0.0 ``` -After installation, module will be available in `modules/nf-core/fastqc` and included in `nextflow_spec.json` +After installation, module will be available in `modules/nf-core/fastqc`. Use the `-force` flag to reinstall a module even if local modifications exist. diff --git a/docs/module.md b/docs/module.md index bf2c07969a..747ae76395 100644 --- a/docs/module.md +++ b/docs/module.md @@ -325,20 +325,6 @@ $ nextflow module run nf-core/fastqc --input 'data/*.fastq.gz' This command accepts all standard Nextflow options (`-profile`, `-resume`, etc.) and automatically downloads the module if not already installed. -### Managing module versions - -Module versions are tracked in `nextflow_spec.json` in your project directory: - -```json -{ - "modules": { - "nf-core/fastqc": "1.0.0", - "nf-core/bwa-align": "1.2.0" - } -} -``` - -When you run your workflow, Nextflow automatically installs or updates modules to match the specified versions. ### Discovering modules @@ -374,9 +360,9 @@ Use the `module remove` command to uninstall a module: $ nextflow module remove nf-core/fastqc ``` -By default, both the local module files and the entry in `nextflow_spec.json` are removed. Use the flags below to control this behaviour: +By default, both the module files and the `.module-info` file are removed. Use the flags below to control this behaviour: -- `-keep-files` — Remove the entry from `nextflow_spec.json` but keep the local module files +- `-keep-files` — Remove the `.module-info` file created at install but keep the rest of files - `-force` — Force removal even if the module has no `.module-info` file (i.e. not installed from a registry) or has local modifications ### Viewing module information diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 05feb8d56b..b8fd201188 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1151,7 +1151,8 @@ The `module` command provides a comprehensive system for managing reusable, regi `install [options] [scope/name]` : Install a module from the registry into your project. -: Downloaded modules are stored in the `modules/` directory and version information is tracked in `nextflow_spec.json`. +: Downloaded modules are stored in the `modules/` directory. +: The `.module-info` file is created in the module directory during the installation to store additional information of the installed module, such as the checksum of the downloaded module files to detect if a module is locally modified and the URL of the registry used to download the module. : The following options are available: `-version` @@ -1288,7 +1289,7 @@ The `module` command provides a comprehensive system for managing reusable, regi : Force removal even if the module has no `.module-info` file (i.e. not installed from a registry) or has local modifications. `-keep-files` - : Remove the version entry from `nextflow_spec.json` but keep local files in the `modules/` directory. + : Remove the `.module-info` but keep local files in the `modules/` directory. : **Examples:** diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy index bcc52b6ed7..0a260c716f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy @@ -22,13 +22,13 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.cli.CmdBase import nextflow.config.ConfigBuilder -import nextflow.config.ModulesConfig + import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException import nextflow.module.ModuleReference import nextflow.module.ModuleRegistryClient import nextflow.module.ModuleResolver -import nextflow.pipeline.PipelineSpec + import nextflow.util.TestOnly import java.nio.file.Path @@ -80,19 +80,12 @@ class CmdModuleInstall extends CmdBase { .build() final registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig() - // Get modules versions from nextflow_spec.json. - final specFile = new PipelineSpec(baseDir) - final modulesConfig = new ModulesConfig(specFile.getModules()) - // Create resolver and install - def resolver = new ModuleResolver(baseDir, client ?: new ModuleRegistryClient(registryConfig), modulesConfig) + def resolver = new ModuleResolver(baseDir, client ?: new ModuleRegistryClient(registryConfig)) try { def installedMainFile = resolver.installModule(reference, version, force) - - // Update nextflow_spec.json with the installed module version def installedVersion = version ?: resolver.resolveVersion(reference) - specFile.addModuleEntry(reference.fullName, installedVersion) println "Module ${reference}@${installedVersion} installed and configured successfully" } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy index 5992196c2b..a9ecf87aca 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy @@ -122,7 +122,8 @@ class CmdModuleList extends CmdBase { name : module.reference.toString(), version : module.installedVersion ?: 'unknown', integrity: module.integrity.toString(), - directory: module.directory.toString() + directory: module.directory.toString(), + registry : module.registryUrl ?: 'unknown' ] } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy index 23cc9f74cc..4c95bc9539 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy @@ -26,6 +26,7 @@ import nextflow.config.ConfigBuilder import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException import nextflow.module.ModuleChecksum +import nextflow.module.ModuleInfo import nextflow.module.ModuleSpec import nextflow.module.ModuleReference import nextflow.module.ModuleRegistryClient @@ -135,14 +136,15 @@ class CmdModulePublish extends CmdBase { ] // Publish to registry + final registry = registryUrl ?: registryConfig.url log.info "Publishing module to registry: ${registryUrl ?: registryConfig.url}" def registryClient = new ModuleRegistryClient(registryConfig) - def response = registryClient.publishModule(manifest.name, request, registryUrl) + def response = registryClient.publishModule(manifest.name, request, registry) if (useModuleReference) { // If publish is performed using the module reference we should create/update the .module-info with the correct checksum try { - ModuleChecksum.save(moduleDir, ModuleChecksum.compute(moduleDir)) + ModuleInfo.save(moduleDir, [checksum: ModuleChecksum.compute(moduleDir), registryUrl: registry] ) }catch (Exception e){ log.warn("Unable to save the checksum - ${e.message}") } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy index 8b397bf0b8..cfcea1b1f4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy @@ -22,9 +22,10 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.cli.CmdBase import nextflow.exception.AbortOperationException +import nextflow.module.ModuleInfo import nextflow.module.ModuleReference import nextflow.module.ModuleStorage -import nextflow.pipeline.PipelineSpec + import nextflow.util.TestOnly import java.nio.file.Files @@ -44,7 +45,7 @@ class CmdModuleRemove extends CmdBase { @Parameter(description = "", required = true) List args - @Parameter(names = ["-keep-files"], description = "Remove from config but keep local files", arity = 0) + @Parameter(names = ["-keep-files"], description = "Remove only .module-info keeping the local files", arity = 0) boolean keepFiles = false @Parameter(names = ["-force"], description = "Force remove", arity = 0) @@ -72,15 +73,11 @@ class CmdModuleRemove extends CmdBase { // Get config def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize() - //Get module versions from nextflow_spec.json. - def specFile = new PipelineSpec(baseDir) - // Create resolver and spec file manager def storage = new ModuleStorage(baseDir) try { def filesRemoved = false - def configRemoved = false // Remove local files unless -keep-files is set if( !keepFiles ) { @@ -98,12 +95,6 @@ class CmdModuleRemove extends CmdBase { } } - - configRemoved = specFile.removeModuleEntry(reference.fullName) - if( configRemoved ) { - println "Module ${reference} entry removed from spec file" - } - } catch( AbortOperationException e ) { throw e diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy index 56f3cf18bf..c0f8fe8421 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy @@ -21,13 +21,13 @@ import com.beust.jcommander.Parameters import groovy.transform.CompileStatic import nextflow.cli.CmdRun import nextflow.config.ConfigBuilder -import nextflow.config.ModulesConfig + import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException import nextflow.module.ModuleReference import nextflow.module.ModuleRegistryClient import nextflow.module.ModuleResolver -import nextflow.pipeline.PipelineSpec + import nextflow.util.TestOnly import java.nio.file.Path @@ -78,11 +78,7 @@ class CmdModuleRun extends CmdRun { def registryConfig = config.navigate('registry') as RegistryConfig ?: new RegistryConfig() - //Get module version from nextflow_spec.json. - def specFile = new PipelineSpec(baseDir) - def modulesConfig = new ModulesConfig(specFile.getModules()) - - def resolver = new ModuleResolver(baseDir, client ?: new ModuleRegistryClient(registryConfig), modulesConfig) + def resolver = new ModuleResolver(baseDir, client ?: new ModuleRegistryClient(registryConfig)) Path moduleFile = resolver.installModule(reference, version) if( moduleFile ) { println "Executing module..." diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ModulesConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ModulesConfig.groovy deleted file mode 100644 index c1968c62ea..0000000000 --- a/modules/nextflow/src/main/groovy/nextflow/config/ModulesConfig.groovy +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2013-2026, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package nextflow.config - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import nextflow.config.spec.ConfigOption -import nextflow.config.spec.ConfigScope -import nextflow.config.spec.ScopeName -import nextflow.script.dsl.Description -import nextflow.util.TestOnly - -/** - * Configuration scope for module version declarations - * - * @author Jorge Ejarque - */ -@Slf4j -@ScopeName("modules") -@Description(""" - The `modules` scope provides module version declarations for the Nextflow module system. - Each entry maps a module reference to a specific version. -""") -@CompileStatic -class ModulesConfig implements ConfigScope { - - @ConfigOption - @Description("Module version mappings (module name -> version)") - private Map modules = [:] - - /* required by extension point -- do not remove */ - ModulesConfig() {} - - ModulesConfig(Map opts) { - if (opts) { - opts.each { key, value -> - this.modules.put(key.toString(), value.toString()) - } - } - } - - /** - * Get the configured version for a module - * - * @param moduleName The module name (e.g., "@nf-core/fastqc") - * @return The configured version, or null if not configured - */ - String getVersion(String moduleName) { - return this.modules.get(moduleName) - } - - /** - * Get all configured modules - * - * @return Map of module name to version - */ - @TestOnly - Map getAllModules() { - return Collections.unmodifiableMap(modules) - } - - /** - * Set a module version - * - * @param moduleName The module name - * @param version The version to set - */ - void setVersion(String moduleName, String version) { - this.modules.put(moduleName, version) - } - - /** - * Check if a module version is configured - * - * @param moduleName The module name - * @return true if version is configured - */ - boolean hasVersion(String moduleName) { - return modules.containsKey(moduleName) - } -} diff --git a/modules/nextflow/src/main/groovy/nextflow/module/DefaultRemoteModuleResolver.groovy b/modules/nextflow/src/main/groovy/nextflow/module/DefaultRemoteModuleResolver.groovy index e1db3366f0..ef3d0efb45 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/DefaultRemoteModuleResolver.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/DefaultRemoteModuleResolver.groovy @@ -19,14 +19,11 @@ package nextflow.module import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.Global -import nextflow.NF -import nextflow.Session import nextflow.config.ConfigBuilder -import nextflow.config.ModulesConfig + import nextflow.config.RegistryConfig import nextflow.exception.IllegalModulePath import nextflow.module.spi.RemoteModuleResolver -import nextflow.pipeline.PipelineSpec import java.nio.file.Path @@ -50,13 +47,11 @@ class DefaultRemoteModuleResolver implements RemoteModuleResolver { @Override Path resolve(String moduleName, Path baseDir) { - final modulesConfig = getModuleConfig(baseDir) - final config = Global.config ?: new ConfigBuilder().setBaseDir(baseDir).build() final registryConfig = config.navigate('registry') as RegistryConfig // Create module resolver - def resolver = new ModuleResolver(baseDir, modulesConfig, registryConfig) + def resolver = new ModuleResolver(baseDir, registryConfig) try { log.debug "Resolving remote module: ${moduleName}" @@ -81,20 +76,4 @@ class DefaultRemoteModuleResolver implements RemoteModuleResolver { int getPriority() { return 0 // Default implementation has lowest priority } - - private ModulesConfig getModuleConfig(Path baseDir) { - def specFile = new PipelineSpec(baseDir) - - if (!specFile.exists()) { - log.warn1("Remote module specified and 'nextflow_spec.json' not found") - return new ModulesConfig() - } - - def modules = specFile.getModules() - if (!modules || modules.isEmpty()) { - log.warn1("Remote module specified and no modules configured in 'nextflow_spec.json'") - return new ModulesConfig() - } - return new ModulesConfig(modules) - } } diff --git a/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy b/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy index 88fddb02a6..e951df7c59 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy @@ -42,6 +42,7 @@ class InstalledModule { Path moduleInfoFile String installedVersion String expectedChecksum + String registryUrl /** * Get the integrity status of this installed module diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy index 861020983b..0693b3155c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy @@ -23,7 +23,7 @@ import java.nio.file.Files import java.nio.file.Path import java.security.MessageDigest -import static nextflow.module.ModuleStorage.MODULE_INFO_FILE +import static nextflow.module.ModuleInfo.MODULE_INFO_FILE /** * Utility class for computing SHA-256 checksums of module directories @@ -92,13 +92,7 @@ class ModuleChecksum { * @param checksum The checksum to save */ static void save(Path moduleDir, String checksum) { - def moduleInfoFile = moduleDir.resolve(MODULE_INFO_FILE) - def props = new Properties() - // If file exists loads to update current just checksum property - if( Files.exists( moduleInfoFile)) - moduleInfoFile.withInputStream { is -> props.load(is) } - props.setProperty('checksum', checksum) - moduleInfoFile.withOutputStream { os -> props.store(os, null) } + ModuleInfo.save(moduleDir,'checksum', checksum) } /** @@ -108,13 +102,7 @@ class ModuleChecksum { * @return The checksum, or null if file doesn't exist */ static String load(Path moduleDir) { - def moduleInfoFile = moduleDir.resolve(MODULE_INFO_FILE) - if( !Files.exists(moduleInfoFile) ) { - return null - } - def props = new Properties() - moduleInfoFile.withInputStream { is -> props.load(is) } - return props.getProperty('checksum') + ModuleInfo.load(moduleDir, 'checksum') } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleInfo.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleInfo.groovy new file mode 100644 index 0000000000..a0fd26511f --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleInfo.groovy @@ -0,0 +1,109 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j + +import java.nio.file.Files +import java.nio.file.Path + +/** + * Utility class for managing .module-info + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +class ModuleInfo { + + public static final String MODULE_INFO_FILE = ".module-info" + + + /** + * Save a property to the .module-info file in the module directory + * + * @param moduleDir The module directory path + * @param property The property to save + * @param value The property value to save + */ + static void save(Path moduleDir, String property, String value) { + def moduleInfoFile = moduleDir.resolve(MODULE_INFO_FILE) + def props = new Properties() + // If file exists loads to update current just checksum property + if( Files.exists( moduleInfoFile)) + moduleInfoFile.withInputStream { is -> props.load(is) } + props.setProperty(property, value) + moduleInfoFile.withOutputStream { os -> props.store(os, null) } + } + + /** + * Save a property to the .module-info file in the module directory + * + * @param moduleDir The module directory path + * @param properties Map with properties to save + */ + static void save(Path moduleDir, Map properties) { + if( properties ) { + def moduleInfoFile = moduleDir.resolve(MODULE_INFO_FILE) + def props = new Properties() + // If file exists loads to update current just checksum property + if( Files.exists( moduleInfoFile ) ) + moduleInfoFile.withInputStream { is -> props.load(is) } + + for( final property : properties.entrySet() ) { + props.setProperty(property.key, property.value) + } + moduleInfoFile.withOutputStream { os -> props.store(os, null) } + } + } + + /** + * Return the value of property from the .module-info file in the module directory + * + * @param moduleDir The module directory path + * @param moduleDir The module directory path + * @return The checksum, or null if file doesn't exist + */ + static String load(Path moduleDir, String property) { + def moduleInfoFile = moduleDir.resolve(MODULE_INFO_FILE) + if( !Files.exists(moduleInfoFile) ) { + log.debug("Module file $moduleInfoFile not found") + return null + } + def props = new Properties() + moduleInfoFile.withInputStream { is -> props.load(is) } + return props.getProperty(property) + } + + /** + * Load all properties from the .module-info file in the module directory + * + * @param moduleDir The module directory path + * @return The checksum, or null if file doesn't exist + */ + static Map load(Path moduleDir) { + def moduleInfoFile = moduleDir.resolve(MODULE_INFO_FILE) + if( !Files.exists(moduleInfoFile) ) { + log.debug("Module file $moduleInfoFile not found") + return [:] + } + def props = new Properties() + moduleInfoFile.withInputStream { is -> props.load(is) } + return props as Map + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy index 4b2fc225ec..0528b17609 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleRegistryClient.groovy @@ -214,9 +214,9 @@ class ModuleRegistryClient { * @param name The module name * @param version The module version * @param targetPath The target path to download to - * @return Path with the downloaded file path + * @return URL of the repository used to downloaded file path */ - Path downloadModule(String name, String version, Path targetPath) { + String downloadModule(String name, String version, Path targetPath) { def registryUrls = config.allUrls if( targetPath.exists() ) { targetPath.delete() @@ -240,7 +240,7 @@ class ModuleRegistryClient { /** * Download module from a specific registry URL */ - private Path downloadModuleFromRegistry(String registryUrl, String name, String version, Path targetPath) { + private String downloadModuleFromRegistry(String registryUrl, String name, String version, Path targetPath) { def endpoint = "${registryUrl}/v1/modules/${encodeName(name)}/${version}/download" def uri = URI.create(endpoint) @@ -281,7 +281,7 @@ class ModuleRegistryClient { validateDownloadIntegrity(response, uri, targetPath, name, version) - return targetPath + return registryUrl } catch( AbortOperationException e ) { throw e diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy index b9818d85fa..8e8626a0b9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleResolver.groovy @@ -18,7 +18,7 @@ package nextflow.module import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import nextflow.config.ModulesConfig + import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException @@ -36,16 +36,14 @@ class ModuleResolver { private final ModuleRegistryClient registryClient private final ModuleStorage storage - private final ModulesConfig modulesConfig - ModuleResolver(Path baseDir, ModuleRegistryClient registryClient, ModulesConfig modulesConfig = null) { + ModuleResolver(Path baseDir, ModuleRegistryClient registryClient) { this.registryClient = registryClient this.storage = new ModuleStorage(baseDir) - this.modulesConfig = modulesConfig ?: new ModulesConfig() } - ModuleResolver(Path baseDir, ModulesConfig modulesConfig = null, RegistryConfig registryConfig = null) { - this(baseDir, new ModuleRegistryClient(registryConfig ?: new RegistryConfig()), modulesConfig) + ModuleResolver(Path baseDir, RegistryConfig registryConfig = null) { + this(baseDir, new ModuleRegistryClient(registryConfig ?: new RegistryConfig())) } @@ -58,8 +56,6 @@ class ModuleResolver { * @return Path to the module's main.nf file */ Path resolve(ModuleReference reference, String version = null, boolean autoInstall = false) { - // Determine version: explicit > config > latest - def targetVersion = version ?: modulesConfig.getVersion(reference.fullName) // Check if module is already installed def installed = storage.getInstalledModule(reference) @@ -81,15 +77,15 @@ class ModuleResolver { } // Check if version matches - if( targetVersion && installed.installedVersion != targetVersion ) { + if( version && installed.installedVersion != version ) { if( autoInstall ) { - log.info "Upgrading module ${reference} from ${installed.installedVersion} to ${targetVersion}" - return installModule(reference, targetVersion) + log.info "Upgrading module ${reference} from ${installed.installedVersion} to ${version}" + return installModule(reference, version) } else { throw new AbortOperationException( "Module ${reference} version mismatch: " + - "installed=${installed.installedVersion}, required=${targetVersion}. " + - "Run 'nextflow module install ${reference}@${targetVersion}' to update." + "installed=${installed.installedVersion}, required=${version}. " + + "Run 'nextflow module install ${reference}@${version}' to update." ) } } @@ -100,7 +96,7 @@ class ModuleResolver { // Module not installed if( autoInstall ) { - return installModule(reference, targetVersion) + return installModule(reference, version) } else { throw new AbortOperationException( "Module ${reference} is not installed. " + @@ -110,8 +106,7 @@ class ModuleResolver { } String resolveVersion(ModuleReference reference) { - final version = modulesConfig.getVersion(reference.fullName) - ?: registryClient.fetchModule(reference.fullName)?.latest?.version + final version = registryClient.fetchModule(reference.fullName)?.latest?.version if( !version ) { throw new AbortOperationException("Module ${reference} has no published versions") } @@ -160,10 +155,10 @@ class ModuleResolver { Path tempFile = Files.createTempFile("nf-module-", ".tgz") try { // Download and validate integrity using server checksum - def downloadResult = registryClient.downloadModule(reference.fullName, version, tempFile) + def downloadUrl = registryClient.downloadModule(reference.fullName, version, tempFile) // Install to modules directory (will compute directory checksum for future integrity checks) - InstalledModule installed = storage.installModule(reference, version, tempFile) + InstalledModule installed = storage.installModule(reference, version, tempFile, downloadUrl) log.info "Module ${reference}@${version} installed successfully at ${installed.mainFile.parent}" return installed.mainFile diff --git a/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy b/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy index c59405b223..6c9a90ca87 100644 --- a/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/module/ModuleStorage.groovy @@ -33,6 +33,8 @@ import java.util.stream.Stream import java.util.zip.ZipEntry import java.util.zip.ZipInputStream +import static nextflow.module.ModuleInfo.MODULE_INFO_FILE + /** * Manages local filesystem storage for modules * @@ -43,7 +45,6 @@ import java.util.zip.ZipInputStream class ModuleStorage { public static final String MODULE_MANIFEST_FILE = "meta.yml" public static final String MODULE_README_FILE = "README.md" - public static final String MODULE_INFO_FILE = ".module-info" private final Path modulesDir /** @@ -116,7 +117,9 @@ class ModuleStorage { ) // Load checksum if available - installed.expectedChecksum = ModuleChecksum.load(moduleDir) + Map infoProps = ModuleInfo.load(moduleDir) + installed.expectedChecksum = infoProps?.checksum + installed.registryUrl = infoProps?.registryUrl installed.installedVersion = ModuleSpec.load(installed.manifestFile).version return installed } @@ -164,7 +167,7 @@ class ModuleStorage { * @param packageFile The downloaded package file (zip or tar.gz) * @return The InstalledModule object */ - InstalledModule installModule(ModuleReference reference, String version, Path packageFile) { + InstalledModule installModule(ModuleReference reference, String version, Path packageFile, String downloadUrl) { def moduleDir = getModuleDir(reference) try { @@ -187,7 +190,7 @@ class ModuleStorage { // Compute and save checksum of extracted directory contents // This checksum is used to detect local modifications def checksum = ModuleChecksum.compute(moduleDir) - ModuleChecksum.save(moduleDir, checksum) + ModuleInfo.save(moduleDir, [checksum: checksum, registryUrl: downloadUrl]) log.debug "Installed module ${reference}@${version} to ${moduleDir}" diff --git a/modules/nextflow/src/main/groovy/nextflow/pipeline/PipelineSpec.groovy b/modules/nextflow/src/main/groovy/nextflow/pipeline/PipelineSpec.groovy deleted file mode 100644 index 2e109cdc4f..0000000000 --- a/modules/nextflow/src/main/groovy/nextflow/pipeline/PipelineSpec.groovy +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright 2013-2026, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package nextflow.pipeline - -import groovy.json.JsonOutput -import groovy.json.JsonSlurper -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j - -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.StandardOpenOption - -/** - * Manages the nextflow_spec.json file for module version declarations - * - * @author Jorge Ejarque - */ -@Slf4j -@CompileStatic -class PipelineSpec { - - private static final String SPEC_FILE_NAME = 'nextflow_spec.json' - - private final Path baseDir - private final Path specFile - - PipelineSpec(Path baseDir) { - this.baseDir = baseDir - this.specFile = baseDir.resolve(SPEC_FILE_NAME) - } - - /** - * Add or update a module entry in the spec file - * - * @param moduleName The module name (e.g., "@nf-core/fastqc" or "nf-core/fastqc") - * @param version The module version - */ - void addModuleEntry(String moduleName, String version) { - // Normalize module name (strip leading @ if present) - def normalizedName = moduleName.startsWith('@') ? moduleName.substring(1) : moduleName - - def spec = readSpecFile() - - // Ensure modules map exists - if (!spec.modules) { - spec.modules = [:] - } - - // Check if already configured with same version - if (spec.modules[normalizedName] == version) { - log.info "Module ${normalizedName} already configured with version ${version} in ${SPEC_FILE_NAME}" - return - } - - // Add or update entry - spec.modules[normalizedName] = version - writeSpecFile(spec) - log.info "Added ${normalizedName}@${version} to ${SPEC_FILE_NAME}" - } - - /** - * Remove a module entry from the spec file - * - * @param moduleName The module name (e.g., "@nf-core/fastqc" or "nf-core/fastqc") - * @return true if entry was removed, false if it didn't exist - */ - boolean removeModuleEntry(String moduleName) { - - def spec = readSpecFile() - - if (!spec.modules) { - return false - } - final modules = spec.modules as Map - if( modules.remove(moduleName) == null ) - return false - writeSpecFile(spec) - log.info "Removed ${moduleName} from ${SPEC_FILE_NAME}" - return true - } - /** - * @return Modules Map stored in the spec file - */ - Map getModules() { - def spec = readSpecFile() - - if (!spec.modules) { - return [:] - } - return spec.modules as Map - } - - /** - * Check if the spec file exists - * - * @return true if the file exists - */ - boolean exists() { - return Files.exists(specFile) - } - - private Map readSpecFile() { - if (!Files.exists(specFile)) { - return [:] - } - - try { - def content = Files.readString(specFile) - if (content.trim().isEmpty()) { - return [:] - } - return new JsonSlurper().parseText(content) as Map - } catch (Exception e) { - throw new RuntimeException("Failed to read spec file ${specFile}: ${e.message}", e) - } - } - - private void writeSpecFile(Map spec) { - try { - // Create directory if it doesn't exist - if (!Files.exists(specFile.parent)) { - Files.createDirectories(specFile.parent) - } - - def jsonContent = JsonOutput.prettyPrint(JsonOutput.toJson(spec)) - Files.writeString(specFile, jsonContent, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING) - } catch (Exception e) { - throw new RuntimeException("Failed to write spec file ${specFile}: ${e.message}", e) - } - } -} diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInstallTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInstallTest.groovy index a2f088b544..6f3e9d08e9 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInstallTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleInstallTest.groovy @@ -19,10 +19,12 @@ package nextflow.cli.module import io.seqera.npr.api.schema.v1.Module import io.seqera.npr.api.schema.v1.ModuleRelease import nextflow.cli.Launcher +import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException import nextflow.module.ModuleChecksum +import nextflow.module.ModuleInfo import nextflow.module.ModuleRegistryClient -import nextflow.pipeline.PipelineSpec +import nextflow.module.ModuleStorage import org.apache.commons.compress.archivers.tar.TarArchiveEntry import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream import org.junit.Rule @@ -85,11 +87,8 @@ class CmdModuleInstallTest extends Specification { def moduleDir = tempDir.resolve('modules/nf-core/fastqc') Files.exists(moduleDir) Files.exists(moduleDir.resolve('main.nf')) - Files.exists(moduleDir.resolve('meta.yml')) - - and: - def spec = new PipelineSpec(tempDir) - spec.getModules().get('nf-core/fastqc') == '1.0.0' + Files.exists(moduleDir.resolve(ModuleStorage.MODULE_MANIFEST_FILE)) + Files.exists(moduleDir.resolve(ModuleInfo.MODULE_INFO_FILE)) } def 'should install module with specific version'() { @@ -108,7 +107,7 @@ class CmdModuleInstallTest extends Specification { def mockClient = Mock(ModuleRegistryClient) mockClient.downloadModule(_, _, _) >> { String name, String version, Path dest -> Files.write(dest, modulePackage) - return dest + return 'http://registry.com' } cmd.client = mockClient @@ -120,10 +119,12 @@ class CmdModuleInstallTest extends Specification { output.contains('Installing') output.contains('nf-core/fastqc') output.contains('2.0.0') - - and: - def spec = new PipelineSpec(tempDir) - spec.getModules().get('nf-core/fastqc') == '2.0.0' + def moduleDir = tempDir.resolve('modules/nf-core/fastqc') + Files.exists(moduleDir) + Files.exists(moduleDir.resolve('main.nf')) + Files.exists(moduleDir.resolve(ModuleStorage.MODULE_MANIFEST_FILE)) + Files.exists(moduleDir.resolve(ModuleInfo.MODULE_INFO_FILE)) + ModuleInfo.load(moduleDir, 'registryUrl') == "http://registry.com" } @@ -139,9 +140,6 @@ class CmdModuleInstallTest extends Specification { description: Test module """.stripIndent() - def spec = new PipelineSpec(tempDir) - spec.addModuleEntry('nf-core/fastqc', '1.0.0') - and: def cmd = new CmdModuleInstall() cmd.launcher = Mock(Launcher) { @@ -158,7 +156,7 @@ class CmdModuleInstallTest extends Specification { def mockClient = Mock(ModuleRegistryClient) mockClient.downloadModule('nf-core/fastqc', '2.0.0', _) >> { String name, String version, Path dest -> Files.write(dest, modulePackage) - return dest + return 'registry' } cmd.client = mockClient @@ -170,10 +168,6 @@ class CmdModuleInstallTest extends Specification { output.contains('Installing') output.contains('2.0.0') - and: - def updatedSpec = new PipelineSpec(tempDir) - updatedSpec.getModules().get('nf-core/fastqc') == '2.0.0' - and: moduleDir.resolve('main.nf').text.contains('FASTQC') // New content } @@ -190,8 +184,6 @@ class CmdModuleInstallTest extends Specification { description: Test module """.stripIndent() ModuleChecksum.save(moduleDir, 'wrong-checksum') - def spec = new PipelineSpec(tempDir) - spec.addModuleEntry('nf-core/fastqc', '1.0.0') and: def cmd = new CmdModuleInstall() @@ -247,10 +239,6 @@ class CmdModuleInstallTest extends Specification { def moduleDir = tempDir.resolve('modules/myorg/custom-module') Files.exists(moduleDir) Files.exists(moduleDir.resolve('main.nf')) - - and: - def spec = new PipelineSpec(tempDir) - spec.getModules().get('myorg/custom-module') == '1.0.0' } def 'should create modules directory if it does not exist'() { diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy index 00bafc75b8..dace2c8ba0 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleListTest.groovy @@ -18,6 +18,7 @@ package nextflow.cli.module import groovy.json.JsonSlurper import nextflow.module.ModuleChecksum +import nextflow.module.ModuleInfo import nextflow.module.ModuleReference import nextflow.module.ModuleStorage import org.junit.Rule @@ -178,7 +179,7 @@ class CmdModuleListTest extends Specification { """.stripIndent() // Create .module-info - ModuleChecksum.save(moduleDir, ModuleChecksum.compute(moduleDir)) + ModuleInfo.save(moduleDir, [checksum: ModuleChecksum.compute(moduleDir), registryUrl: 'http://registry.com']) return moduleDir } diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRemoveTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRemoveTest.groovy index 27c6b85242..bdc306bdb5 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRemoveTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/module/CmdModuleRemoveTest.groovy @@ -18,9 +18,9 @@ package nextflow.cli.module import nextflow.exception.AbortOperationException import nextflow.module.ModuleChecksum +import nextflow.module.ModuleInfo import nextflow.module.ModuleReference import nextflow.module.ModuleStorage -import nextflow.pipeline.PipelineSpec import org.junit.Rule import spock.lang.Specification import spock.lang.TempDir @@ -44,16 +44,12 @@ class CmdModuleRemoveTest extends Specification { // No setup needed - using root field directly - def 'should remove module files and config entry'() { + def 'should remove all module files'() { given: def storage = new ModuleStorage(tempDir) def reference = new ModuleReference('nf-core', 'fastqc') def moduleDir = createTestModule(storage, reference, true) - // Create spec file with module entry - def specFile = new PipelineSpec(tempDir) - specFile.addModuleEntry('nf-core/fastqc', '1.0.0') - and: def cmd = new CmdModuleRemove() cmd.args = ['nf-core/fastqc'] @@ -65,12 +61,7 @@ class CmdModuleRemoveTest extends Specification { then: output.contains('Module nf-core/fastqc files removed successfully') - output.contains('Module nf-core/fastqc entry removed from spec') !Files.exists(moduleDir) - - and: - def spec = new PipelineSpec(tempDir) - spec.getModules().get('nf-core/fastqc') == null } def 'should keep files with -keep-files flag'() { @@ -79,10 +70,6 @@ class CmdModuleRemoveTest extends Specification { def reference = new ModuleReference('nf-core', 'fastqc') def moduleDir = createTestModule(storage, reference, true) - // Create spec file - def specFile = new PipelineSpec(tempDir) - specFile.addModuleEntry('nf-core/fastqc', '1.0.0') - and: def cmd = new CmdModuleRemove() cmd.args = ['nf-core/fastqc'] @@ -95,14 +82,9 @@ class CmdModuleRemoveTest extends Specification { then: output.contains('Keeping module files for nf-core/fastqc ') - output.contains('Module nf-core/fastqc entry removed from spec file') Files.exists(moduleDir) Files.exists(moduleDir.resolve('main.nf')) - !Files.exists(moduleDir.resolve(ModuleStorage.MODULE_INFO_FILE)) - - and: - def spec = new PipelineSpec(tempDir) - spec.getModules().get('nf-core/fastqc') == null + !Files.exists(moduleDir.resolve(ModuleInfo.MODULE_INFO_FILE)) } def 'should fail when both keep-files and force are set'() { diff --git a/modules/nextflow/src/test/groovy/nextflow/config/ModulesConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/config/ModulesConfigTest.groovy deleted file mode 100644 index f320aa472d..0000000000 --- a/modules/nextflow/src/test/groovy/nextflow/config/ModulesConfigTest.groovy +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright 2013-2026, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package nextflow.config - -import spock.lang.Specification - -/** - * Tests for ModulesConfig - * - * @author Jorge Ejarque - */ -class ModulesConfigTest extends Specification { - - def 'should create empty config'() { - when: - def config = new ModulesConfig() - - then: - config.getAllModules().isEmpty() - } - - def 'should set and get module version'() { - given: - def config = new ModulesConfig() - - when: - config.setVersion('nf-core/fastqc', '1.0.0') - - then: - config.getVersion('nf-core/fastqc') == '1.0.0' - config.hasVersion('nf-core/fastqc') - } - - def 'should return null for unconfigured module'() { - given: - def config = new ModulesConfig() - - when: - def version = config.getVersion('nf-core/bwa') - - then: - version == null - !config.hasVersion('nf-core/bwa') - } - - def 'should override existing version'() { - given: - def config = new ModulesConfig() - config.setVersion('nf-core/fastqc', '1.0.0') - - when: - config.setVersion('nf-core/fastqc', '2.0.0') - - then: - config.getVersion('nf-core/fastqc') == '2.0.0' - } - - def 'should return unmodifiable map from getModules'() { - given: - def config = new ModulesConfig() - config.setVersion('nf-core/fastqc', '1.0.0') - - when: - def modules = config.getAllModules() - modules.put('nf-core/bwa', '2.0.0') - - then: - thrown(UnsupportedOperationException) - } - - def 'should return all configured modules'() { - given: - def config = new ModulesConfig() - config.setVersion('nf-core/fastqc', '1.0.0') - config.setVersion('nf-core/bwa', '2.0.0') - config.setVersion('myorg/custom', '0.5.0') - - when: - def modules = config.getAllModules() - - then: - modules.size() == 3 - modules['nf-core/fastqc'] == '1.0.0' - modules['nf-core/bwa'] == '2.0.0' - modules['myorg/custom'] == '0.5.0' - } - - def 'should handle empty initialization'() { - when: - def config = new ModulesConfig(null) - - then: - config.getAllModules().isEmpty() - !config.hasVersion('nf-core/fastqc') - } - - def 'should store multiple versions independently'() { - given: - def config = new ModulesConfig() - - when: - config.setVersion('nf-core/fastqc', '1.0.0') - config.setVersion('nf-core/bwa', '2.0.0') - config.setVersion('myorg/custom', '0.5.0') - - then: - config.getVersion('nf-core/fastqc') == '1.0.0' - config.getVersion('nf-core/bwa') == '2.0.0' - config.getVersion('myorg/custom') == '0.5.0' - config.allModules.size() == 3 - } - - def 'should handle module names with special characters'() { - given: - def config = new ModulesConfig() - - when: - config.setVersion('org-name/module-name', '1.0.0') - config.setVersion('org_name/module_name', '2.0.0') - config.setVersion('simple-module', '3.0.0') - - then: - config.getVersion('org-name/module-name') == '1.0.0' - config.getVersion('org_name/module_name') == '2.0.0' - config.getVersion('simple-module') == '3.0.0' - } - - def 'should handle version strings with various formats'() { - given: - def config = new ModulesConfig() - - when: - config.setVersion('nf-core/fastqc', '1.0.0') - config.setVersion('nf-core/bwa', 'v2.0.0') - config.setVersion('nf-core/samtools', '1.0.0-beta') - config.setVersion('nf-core/bowtie', '1.0.0-rc.1') - - then: - config.getVersion('nf-core/fastqc') == '1.0.0' - config.getVersion('nf-core/bwa') == 'v2.0.0' - config.getVersion('nf-core/samtools') == '1.0.0-beta' - config.getVersion('nf-core/bowtie') == '1.0.0-rc.1' - } - - def 'should check if multiple modules have versions'() { - given: - def config = new ModulesConfig() - config.setVersion('nf-core/fastqc', '1.0.0') - config.setVersion('nf-core/bwa', '2.0.0') - - expect: - config.hasVersion('nf-core/fastqc') - config.hasVersion('nf-core/bwa') - !config.hasVersion('nf-core/samtools') - } - - def 'should handle version updates'() { - given: - def config = new ModulesConfig() - config.setVersion('nf-core/fastqc', '1.0.0') - - when: - config.setVersion('nf-core/fastqc', '1.1.0') - config.setVersion('nf-core/fastqc', '2.0.0') - - then: - config.getVersion('nf-core/fastqc') == '2.0.0' - } - - def 'should maintain separate versions for different modules'() { - given: - def config = new ModulesConfig() - - when: - config.setVersion('nf-core/fastqc', '1.0.0') - config.setVersion('nf-core/bwa', '2.0.0') - - then: - config.getVersion('nf-core/fastqc') == '1.0.0' - config.getVersion('nf-core/bwa') == '2.0.0' - config.getVersion('nf-core/fastqc') != config.getVersion('nf-core/bwa') - } -} diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleInfoTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleInfoTest.groovy new file mode 100644 index 0000000000..0df03532e7 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleInfoTest.groovy @@ -0,0 +1,183 @@ +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.module + +import java.nio.file.Files +import java.nio.file.Path + +import spock.lang.Specification + +/** + * Test suite for ModuleInfo + * + * @author Jorge Ejarque + */ +class ModuleInfoTest extends Specification { + + Path tempDir + + def setup() { + tempDir = Files.createTempDirectory('nf-module-info-test-') + } + + def cleanup() { + tempDir?.deleteDir() + } + + def 'should save and load a single property'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + when: + ModuleInfo.save(moduleDir, 'checksum', 'abc123') + + then: + ModuleInfo.load(moduleDir, 'checksum') == 'abc123' + } + + def 'should create .module-info file on save'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + when: + ModuleInfo.save(moduleDir, 'version', '1.0.0') + + then: + Files.exists(moduleDir.resolve(ModuleInfo.MODULE_INFO_FILE)) + } + + def 'should update existing property without affecting others'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + ModuleInfo.save(moduleDir, 'checksum', 'original') + ModuleInfo.save(moduleDir, 'registryUrl', 'http://registry.com') + + when: + ModuleInfo.save(moduleDir, 'checksum', 'updated') + + then: + ModuleInfo.load(moduleDir, 'checksum') == 'updated' + ModuleInfo.load(moduleDir, 'registryUrl') == 'http://registry.com' + } + + def 'should return null when loading property from non-existent file'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + when: + def result = ModuleInfo.load(moduleDir, 'checksum') + + then: + result == null + } + + def 'should return null when loading non-existent property'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + ModuleInfo.save(moduleDir, 'registryUrl', 'http://registry.com') + + when: + def result = ModuleInfo.load(moduleDir, 'missing-property') + + then: + result == null + } + + def 'should save and load multiple properties via map'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + def props = [checksum: 'abc123', registryUrl: 'http://registry.com', author: 'test'] + + when: + ModuleInfo.save(moduleDir, props) + + then: + ModuleInfo.load(moduleDir, 'checksum') == 'abc123' + ModuleInfo.load(moduleDir, 'registryUrl') == 'http://registry.com' + ModuleInfo.load(moduleDir, 'author') == 'test' + } + + def 'should merge map properties with existing ones'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + ModuleInfo.save(moduleDir, 'existing', 'value') + + when: + ModuleInfo.save(moduleDir, [newprop: 'newval']) + + then: + ModuleInfo.load(moduleDir, 'existing') == 'value' + ModuleInfo.load(moduleDir, 'newprop') == 'newval' + } + + def 'should do nothing when saving null map'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + when: + ModuleInfo.save(moduleDir, (Map) null) + + then: + !Files.exists(moduleDir.resolve(ModuleInfo.MODULE_INFO_FILE)) + } + + def 'should do nothing when saving empty map'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + when: + ModuleInfo.save(moduleDir, [:]) + + then: + !Files.exists(moduleDir.resolve(ModuleInfo.MODULE_INFO_FILE)) + } + + def 'should load all properties as map'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + ModuleInfo.save(moduleDir, [checksum: 'abc123', registryUrl: 'http://registry.com']) + + when: + def result = ModuleInfo.load(moduleDir) + + then: + result['checksum'] == 'abc123' + result['registryUrl'] == 'http://registry.com' + } + + def 'should return empty map when loading all from non-existent file'() { + given: + def moduleDir = tempDir.resolve('module') + Files.createDirectories(moduleDir) + + when: + def result = ModuleInfo.load(moduleDir) + + then: + result == [:] + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleRegistryClientTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleRegistryClientTest.groovy index 16e44ec9b1..66fbee0ff6 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleRegistryClientTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleRegistryClientTest.groovy @@ -169,7 +169,7 @@ class ModuleRegistryClientTest extends Specification { def result = client.downloadModule('nf-core/fastqc', '1.0.0', destFile) then: - result == destFile + result == url Files.exists(destFile) Files.size(destFile) == modulePackage.length @@ -351,7 +351,7 @@ class ModuleRegistryClientTest extends Specification { def result = client.downloadModule('nf-core/fastqc', '1.0.0', destFile) then: - result == destFile + result == url Files.exists(destFile) and: 'verify checksum header was present' diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverTest.groovy index 08aa6809f9..2c87bdde8e 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleResolverTest.groovy @@ -16,7 +16,7 @@ package nextflow.module -import nextflow.config.ModulesConfig +import nextflow.config.RegistryConfig import nextflow.exception.AbortOperationException import nextflow.file.FileHelper import spock.lang.Specification @@ -103,8 +103,7 @@ class ModuleResolverTest extends Specification { def 'should throw exception when version mismatch without auto-install'() { given: - def modulesConfig = new ModulesConfig(['nf-core/fastqc': '2.0.0']) - def resolver = new ModuleResolver(tempDir, modulesConfig, null) + def resolver = new ModuleResolver(tempDir, new RegistryConfig()) def reference = new ModuleReference('nf-core', 'fastqc') def storage = new ModuleStorage(tempDir) def moduleDir = storage.getModuleDir(reference) @@ -122,7 +121,7 @@ class ModuleResolverTest extends Specification { ModuleChecksum.save(moduleDir, checksum) when: - resolver.resolve(reference, null, false) + resolver.resolve(reference, '2.0.0', false) then: def e = thrown(AbortOperationException) @@ -131,7 +130,7 @@ class ModuleResolverTest extends Specification { e.message.contains('required=2.0.0') cleanup: - FileHelper.deletePath(moduleDir) + if( moduleDir ) FileHelper.deletePath(moduleDir) } def 'should resolve installed module with matching version'() { diff --git a/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy b/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy index 06ecde6337..bb44f8fa94 100644 --- a/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/module/ModuleStorageTest.groovy @@ -22,6 +22,8 @@ import java.util.zip.GZIPOutputStream import org.apache.commons.compress.archivers.tar.TarArchiveEntry import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream +import nextflow.exception.AbortOperationException + import spock.lang.Specification /** @@ -221,13 +223,14 @@ class ModuleStorageTest extends Specification { def storage = new ModuleStorage(tempDir) def reference = new ModuleReference('nf-core', 'fastqc') def version = '1.0.0' + def url = "http://registry.com" // Create a gzip package file def packageFile = Files.createTempFile('module-', '.tgz') createTestPackage(packageFile) when: - def installed = storage.installModule(reference, version, packageFile) + def installed = storage.installModule(reference, version, packageFile, url) then: installed != null @@ -235,6 +238,9 @@ class ModuleStorageTest extends Specification { installed.installedVersion == '1.0.0' Files.exists(installed.mainFile) Files.exists(installed.moduleInfoFile) + Files.exists(installed.directory) + installed.registryUrl == url + installed.expectedChecksum == ModuleChecksum.compute(installed.directory) cleanup: packageFile?.delete() @@ -245,6 +251,7 @@ class ModuleStorageTest extends Specification { def storage = new ModuleStorage(tempDir) def reference = new ModuleReference('nf-core', 'fastqc') def moduleDir = storage.getModuleDir(reference) + def url = "http://registry.com" // Create existing installation Files.createDirectories(moduleDir) @@ -256,7 +263,7 @@ class ModuleStorageTest extends Specification { createTestPackage(packageFile) when: - def installed = storage.installModule(reference, '2.0.0', packageFile) + def installed = storage.installModule(reference, '2.0.0', packageFile, url) then: installed != null @@ -317,7 +324,7 @@ class ModuleStorageTest extends Specification { storage.removeModule(reference, false) then: - def e = thrown(nextflow.exception.AbortOperationException) + def e = thrown(AbortOperationException) e.message.contains('.module-info missing') Files.exists(moduleDir) } @@ -362,7 +369,7 @@ class ModuleStorageTest extends Specification { storage.removeModule(reference, false) then: - def e = thrown(nextflow.exception.AbortOperationException) + def e = thrown(AbortOperationException) e.message.contains('local modifications') Files.exists(moduleDir) } @@ -394,9 +401,10 @@ class ModuleStorageTest extends Specification { def reference = new ModuleReference('nf-core', 'fastqc') def packageFile = Files.createTempFile('module-', '.tgz') createTestPackage(packageFile) + def url = "http://registry.com" when: - def installed = storage.installModule(reference, '1.0.0', packageFile) + def installed = storage.installModule(reference, '1.0.0', packageFile, url) then: installed.expectedChecksum != null @@ -415,7 +423,7 @@ class ModuleStorageTest extends Specification { invalidPackage.text = 'not a valid gzip file' when: - storage.installModule(reference, '1.0.0', invalidPackage) + storage.installModule(reference, '1.0.0', invalidPackage, null) then: thrown(Exception)