From 6530e26dbb9d83fdb0dfcd52d7cc45791753c034 Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 26 Jun 2026 13:51:30 +0200 Subject: [PATCH 01/24] debug facet_imaging --- steps/wsclean.cwl | 4 ++-- workflows/facet_imaging.cwl | 5 +++++ workflows/subworkflows/image_and_trim.cwl | 10 ++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/steps/wsclean.cwl b/steps/wsclean.cwl index e48305db..3d3edc65 100644 --- a/steps/wsclean.cwl +++ b/steps/wsclean.cwl @@ -53,7 +53,7 @@ inputs: prefix: '-minuv-l' - id: weight type: string? - default: briggs -1.5 + default: briggs -1.4 inputBinding: position: 1 shellQuote: false @@ -193,7 +193,7 @@ inputs: prefix: '-join-channels' - id: fit-spectral-pol type: int? - default: 9 + default: 5 inputBinding: position: 1 shellQuote: false diff --git a/workflows/facet_imaging.cwl b/workflows/facet_imaging.cwl index bb0f0143..1717a43f 100644 --- a/workflows/facet_imaging.cwl +++ b/workflows/facet_imaging.cwl @@ -41,6 +41,9 @@ inputs: If mosaic is true, a final mosaic will be made of the trimmed facet images using this configuration. + - id: tmpdir + type: string? + doc: Temporary directory to run I/O heavy jobs. steps: - id: sort_mses @@ -72,6 +75,8 @@ steps: source: pixel_scale - id: resolution source: resolution + - id: tmpdir + source: tmpdir out: - id: MFS_image_pb - id: MFS_image diff --git a/workflows/subworkflows/image_and_trim.cwl b/workflows/subworkflows/image_and_trim.cwl index a1538182..54a04be7 100644 --- a/workflows/subworkflows/image_and_trim.cwl +++ b/workflows/subworkflows/image_and_trim.cwl @@ -36,6 +36,10 @@ inputs: doc: | Restoring beam to use for every facet following the WSClean order of major axis, minor axis, position angle. + - id: tmpdir + type: string? + doc: Temporary directory to run I/O heavy jobs. + steps: - id: find_image_size label: image_size @@ -72,6 +76,12 @@ steps: valueFrom: $(self.toString() + "asec") - id: beam-shape source: restoring_beam + - id: tmpdir + source: tmpdir + - id: apply-facet-beam + default: false + - id: apply-primary-beam + default: true out: - id: MFS_image_pb - id: MFS_image From cb1640bad7e0b339b7ace7f41f7582ad87533fc7 Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 26 Jun 2026 13:53:28 +0200 Subject: [PATCH 02/24] add inputs.config --- workflows/facet_imaging.cwl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/facet_imaging.cwl b/workflows/facet_imaging.cwl index 1717a43f..2bd43e06 100644 --- a/workflows/facet_imaging.cwl +++ b/workflows/facet_imaging.cwl @@ -102,7 +102,7 @@ steps: out: - id: output_image run: ../steps/swarp.cwl - when: $(config != null) + when: $(inputs.config != null) outputs: - id: MFS_images_pb From 7cec14a4d75bb185908238df9c1bce57582104a4 Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 26 Jun 2026 13:54:34 +0200 Subject: [PATCH 03/24] add InlineJavascriptRequirement --- workflows/facet_imaging.cwl | 1 + 1 file changed, 1 insertion(+) diff --git a/workflows/facet_imaging.cwl b/workflows/facet_imaging.cwl index 2bd43e06..e59b65a6 100644 --- a/workflows/facet_imaging.cwl +++ b/workflows/facet_imaging.cwl @@ -9,6 +9,7 @@ doc: | requirements: - class: ScatterFeatureRequirement - class: SubworkflowFeatureRequirement + - class: InlineJavascriptRequirement inputs: - id: msin From 680d11bf0c6c0231a5e58d246b16450fc7e5bd1c Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 26 Jun 2026 14:24:08 +0200 Subject: [PATCH 04/24] ncpu bug --- steps/wsclean.cwl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/steps/wsclean.cwl b/steps/wsclean.cwl index 3d3edc65..9794c990 100644 --- a/steps/wsclean.cwl +++ b/steps/wsclean.cwl @@ -349,7 +349,7 @@ requirements: listing: - entry: $(inputs.msin) - class: ResourceRequirement - coresMin: $(inputs.cores) + coresMin: $(inputs.ncpu) stdout: wsclean.log stderr: wsclean_err.log From b1838a17ac7d15c5933e63e992fc060efc60aa9c Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 26 Jun 2026 14:29:11 +0200 Subject: [PATCH 05/24] dockerrequirement --- steps/estimate_image_size.cwl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/steps/estimate_image_size.cwl b/steps/estimate_image_size.cwl index 27439d88..01dcad39 100644 --- a/steps/estimate_image_size.cwl +++ b/steps/estimate_image_size.cwl @@ -56,3 +56,7 @@ outputs: requirements: - class: InlineJavascriptRequirement + +hints: + - class: DockerRequirement + dockerPull: vlbi-cwl From 9f7832eb3f55d0bb7b5677d34633aae72aa13f6d Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 26 Jun 2026 14:44:32 +0200 Subject: [PATCH 06/24] scalar visibilities off --- workflows/subworkflows/image_and_trim.cwl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workflows/subworkflows/image_and_trim.cwl b/workflows/subworkflows/image_and_trim.cwl index 54a04be7..988358ac 100644 --- a/workflows/subworkflows/image_and_trim.cwl +++ b/workflows/subworkflows/image_and_trim.cwl @@ -82,6 +82,8 @@ steps: default: false - id: apply-primary-beam default: true + - id: scalar-visibilities + default: false out: - id: MFS_image_pb - id: MFS_image From b0091068082739ee2d01d3006847c61e243d00c4 Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 26 Jun 2026 15:36:00 +0200 Subject: [PATCH 07/24] scalar vis --- steps/wsclean.cwl | 2 +- workflows/image_intermediate_resolution.cwl | 2 ++ workflows/subworkflows/image_and_trim.cwl | 2 -- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/steps/wsclean.cwl b/steps/wsclean.cwl index 9794c990..b8308b3f 100644 --- a/steps/wsclean.cwl +++ b/steps/wsclean.cwl @@ -255,7 +255,7 @@ inputs: prefix: '-dd-psf-grid' - id: scalar-visibilities type: boolean? - default: true + default: false inputBinding: position: 1 shellQuote: false diff --git a/workflows/image_intermediate_resolution.cwl b/workflows/image_intermediate_resolution.cwl index 07772dbb..218c0ce2 100644 --- a/workflows/image_intermediate_resolution.cwl +++ b/workflows/image_intermediate_resolution.cwl @@ -112,6 +112,8 @@ steps: "facet-solutions": inputs.dd_solutions, "soltabs": ["amplitude000", "phase000"] }) + - id: scalar-visibilities + default: true out: - id: MFS_image_pb - id: MFS_image diff --git a/workflows/subworkflows/image_and_trim.cwl b/workflows/subworkflows/image_and_trim.cwl index 988358ac..54a04be7 100644 --- a/workflows/subworkflows/image_and_trim.cwl +++ b/workflows/subworkflows/image_and_trim.cwl @@ -82,8 +82,6 @@ steps: default: false - id: apply-primary-beam default: true - - id: scalar-visibilities - default: false out: - id: MFS_image_pb - id: MFS_image From ea21824d39efcd3e201d1372c1ea8de55adfca46 Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Thu, 6 Aug 2026 15:49:37 +0200 Subject: [PATCH 08/24] add dp3 avg --- steps/dp3_avg_step.cwl | 57 +++++++++++++++++++++++ workflows/facet_imaging.cwl | 27 ++++++++++- workflows/subworkflows/image_and_trim.cwl | 4 +- 3 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 steps/dp3_avg_step.cwl diff --git a/steps/dp3_avg_step.cwl b/steps/dp3_avg_step.cwl new file mode 100644 index 00000000..30fc716d --- /dev/null +++ b/steps/dp3_avg_step.cwl @@ -0,0 +1,57 @@ +class: CommandLineTool +cwlVersion: v1.2 +id: prediction_avg +label: DP3 averaging for prediction +doc: Average MeasurementSet in time and frequency for faster prediction. + +baseCommand: DP3 + +inputs: + - id: msin + type: Directory + doc: Input MeasurementSet + inputBinding: + position: 0 + prefix: msin= + separate: false + + - id: avgstep + type: int? + doc: Averaging factor + +outputs: + - id: ms_avg + doc: MeasurementSet at lower time/freq resolution + type: Directory + outputBinding: + glob: "$( inputs.msin.basename + '.avg.ms')" + + - id: logfile + type: File[] + outputBinding: + glob: predict_avg*.log + doc: | + The files containing the stdout + and stderr from the step. + +arguments: + - steps=[avg] + - avg.type=averager + - valueFrom: $("avg.timestep=" + inputs.avgstep) + - valueFrom: $("avg.freqstep=" + inputs.avgstep) + - msout.storagemanager='dysco' + - msout.storagemanager.databitrate=6 + - msout=$( inputs.msin.basename + '.avg.ms') + +requirements: + - class: InlineJavascriptRequirement + +hints: + - class: DockerRequirement + dockerPull: vlbi-cwl + - class: ResourceRequirement + coresMin: 8 + +stdout: dp3_avg_step.log +stderr: dp3_avg_step_err.log + diff --git a/workflows/facet_imaging.cwl b/workflows/facet_imaging.cwl index e59b65a6..86c037e9 100644 --- a/workflows/facet_imaging.cwl +++ b/workflows/facet_imaging.cwl @@ -24,6 +24,10 @@ inputs: type: string doc: Angular resolution that will be passed to WSClean's taper argument. Its syntax follows that of WSClean. + - id: avgstep + type: int? + doc: Extra averaging step over time and frequency + - id: facet_polygons type: File[] doc: | @@ -47,11 +51,28 @@ inputs: doc: Temporary directory to run I/O heavy jobs. steps: + - id: average_ms + label: Apply extra averaging of MS + in: + - id: msin + source: msin + - id: avgstep + source: avgstep + out: + - ms_avg + run: ../steps/dp3_avg_step.cwl + scatter: msin + when: $(inputs.avgstep != null) + - id: sort_mses label: Trim facets in: - id: input_entry - source: msin + source: + - average_ms/ms_avg + - msin + pickValue: first_non_null + linkMerge: merge_flattened out: - id: sorted_entries run: ../steps/sort_by_name.cwl @@ -128,7 +149,9 @@ outputs: outputSource: image_and_trim/MFS_model - id: MFS_psfs - type: File[] + type: + - File? + - File[]? outputSource: image_and_trim/MFS_psf - id: MFS_mosaic diff --git a/workflows/subworkflows/image_and_trim.cwl b/workflows/subworkflows/image_and_trim.cwl index 54a04be7..2458c171 100644 --- a/workflows/subworkflows/image_and_trim.cwl +++ b/workflows/subworkflows/image_and_trim.cwl @@ -201,5 +201,7 @@ outputs: outputSource: trim_residual/trimmed_image - id: MFS_psf - type: File + type: + - File? + - File[]? outputSource: make_facet_image/MFS_psf From 42d8d84584982e6193303b178b86f672f9edc823 Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Thu, 6 Aug 2026 16:27:18 +0200 Subject: [PATCH 09/24] rename step --- steps/dp3_avg_step.cwl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/steps/dp3_avg_step.cwl b/steps/dp3_avg_step.cwl index 30fc716d..914f00bf 100644 --- a/steps/dp3_avg_step.cwl +++ b/steps/dp3_avg_step.cwl @@ -1,6 +1,6 @@ class: CommandLineTool cwlVersion: v1.2 -id: prediction_avg +id: dp3_avg_step label: DP3 averaging for prediction doc: Average MeasurementSet in time and frequency for faster prediction. From f414df3f771e506b9264170efc1b944150a23194 Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Thu, 6 Aug 2026 16:39:15 +0200 Subject: [PATCH 10/24] fix order linkmerge --- workflows/facet_imaging.cwl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/facet_imaging.cwl b/workflows/facet_imaging.cwl index 86c037e9..6bcad437 100644 --- a/workflows/facet_imaging.cwl +++ b/workflows/facet_imaging.cwl @@ -71,8 +71,8 @@ steps: source: - average_ms/ms_avg - msin - pickValue: first_non_null linkMerge: merge_flattened + pickValue: first_non_null out: - id: sorted_entries run: ../steps/sort_by_name.cwl From c69a1b02b7ac4c541695e40c952f6a83ba66389e Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Thu, 6 Aug 2026 16:51:31 +0200 Subject: [PATCH 11/24] remove linkmerge --- workflows/facet_imaging.cwl | 1 - 1 file changed, 1 deletion(-) diff --git a/workflows/facet_imaging.cwl b/workflows/facet_imaging.cwl index 6bcad437..8820741c 100644 --- a/workflows/facet_imaging.cwl +++ b/workflows/facet_imaging.cwl @@ -71,7 +71,6 @@ steps: source: - average_ms/ms_avg - msin - linkMerge: merge_flattened pickValue: first_non_null out: - id: sorted_entries From 552f3d3da3e5e0f90466e81bc6bed383de65bd59 Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Thu, 6 Aug 2026 17:03:53 +0200 Subject: [PATCH 12/24] multiple inputs --- workflows/facet_imaging.cwl | 1 + 1 file changed, 1 insertion(+) diff --git a/workflows/facet_imaging.cwl b/workflows/facet_imaging.cwl index 8820741c..e19f73e7 100644 --- a/workflows/facet_imaging.cwl +++ b/workflows/facet_imaging.cwl @@ -10,6 +10,7 @@ requirements: - class: ScatterFeatureRequirement - class: SubworkflowFeatureRequirement - class: InlineJavascriptRequirement + - class: MultipleInputFeatureRequirement inputs: - id: msin From 1daf14fdd9ecccef222364b279a6d849ff7cc263 Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 7 Aug 2026 10:52:08 +0200 Subject: [PATCH 13/24] fix error from sortms --- workflows/facet_imaging.cwl | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/workflows/facet_imaging.cwl b/workflows/facet_imaging.cwl index e19f73e7..669bfd38 100644 --- a/workflows/facet_imaging.cwl +++ b/workflows/facet_imaging.cwl @@ -72,7 +72,18 @@ steps: source: - average_ms/ms_avg - msin - pickValue: first_non_null + linkMerge: merge_nested + pickValue: all_non_null + valueFrom: | + ${ + var avg = self[0]; + if (avg === null || avg === undefined) { return self[1]; } + var kept = []; + for (var i = 0; i < avg.length; i++) { + if (avg[i] !== null && avg[i] !== undefined) { kept.push(avg[i]); } + } + return kept.length === 0 ? self[1] : kept; + } out: - id: sorted_entries run: ../steps/sort_by_name.cwl From 6075fe826234f5035af3a829fbc7edcff02deb78 Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 7 Aug 2026 11:05:52 +0200 Subject: [PATCH 14/24] add number of cores for imaging --- workflows/facet_imaging.cwl | 7 +++++++ workflows/subworkflows/image_and_trim.cwl | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/workflows/facet_imaging.cwl b/workflows/facet_imaging.cwl index 669bfd38..961f77cb 100644 --- a/workflows/facet_imaging.cwl +++ b/workflows/facet_imaging.cwl @@ -51,6 +51,11 @@ inputs: type: string? doc: Temporary directory to run I/O heavy jobs. + - id: ncpu + type: int? + default: 24 + doc: The minimum number of cores that should be available for steps that require high I/O. + steps: - id: average_ms label: Apply extra averaging of MS @@ -110,6 +115,8 @@ steps: source: resolution - id: tmpdir source: tmpdir + - id: ncpu + source: ncpu out: - id: MFS_image_pb - id: MFS_image diff --git a/workflows/subworkflows/image_and_trim.cwl b/workflows/subworkflows/image_and_trim.cwl index 2458c171..201d4ce5 100644 --- a/workflows/subworkflows/image_and_trim.cwl +++ b/workflows/subworkflows/image_and_trim.cwl @@ -13,7 +13,7 @@ inputs: type: Directory doc: MeasurementSet that will be imaged. - - id: number_cores + - id: ncpu type: int? default: 24 doc: The number of cores that WSClean will use. @@ -60,7 +60,7 @@ steps: label: make_facet_image in: - id: ncpu - source: number_cores + source: ncpu - id: msin source: msin - id: name From 52ff586e4cfd9c519a52002c335884c876f50ffb Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 7 Aug 2026 11:07:24 +0200 Subject: [PATCH 15/24] relabel fix --- workflows/facet_imaging.cwl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/facet_imaging.cwl b/workflows/facet_imaging.cwl index 961f77cb..48d51ec6 100644 --- a/workflows/facet_imaging.cwl +++ b/workflows/facet_imaging.cwl @@ -71,7 +71,7 @@ steps: when: $(inputs.avgstep != null) - id: sort_mses - label: Trim facets + label: Sort MS based on name in: - id: input_entry source: @@ -94,7 +94,7 @@ steps: run: ../steps/sort_by_name.cwl - id: sort_facet_regions - label: Trim facets + label: Sort facets based on name in: - id: input_entry source: facet_polygons From 0cb949dcf16c48f1074fd427e59996fd3948a9d6 Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 7 Aug 2026 11:08:20 +0200 Subject: [PATCH 16/24] relabel fix --- workflows/subworkflows/image_and_trim.cwl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/workflows/subworkflows/image_and_trim.cwl b/workflows/subworkflows/image_and_trim.cwl index 201d4ce5..b5843a87 100644 --- a/workflows/subworkflows/image_and_trim.cwl +++ b/workflows/subworkflows/image_and_trim.cwl @@ -94,7 +94,7 @@ steps: run: ../../steps/wsclean.cwl - id: trim_image_pb - label: Trim facets + label: Trim facet image-pb in: - id: image source: @@ -108,7 +108,7 @@ steps: run: ../../steps/trim_facet.cwl - id: trim_image - label: Trim facets + label: Trim facet image in: - id: image source: @@ -122,7 +122,7 @@ steps: run: ../../steps/trim_facet.cwl - id: trim_model_pb - label: Trim facets + label: Trim model image-pb in: - id: image source: @@ -136,7 +136,7 @@ steps: run: ../../steps/trim_facet.cwl - id: trim_model - label: Trim facets + label: Trim model image in: - id: image source: @@ -150,7 +150,7 @@ steps: run: ../../steps/trim_facet.cwl - id: trim_residual_pb - label: Trim facets + label: Trim residual image-pb in: - id: image source: @@ -164,7 +164,7 @@ steps: run: ../../steps/trim_facet.cwl - id: trim_residual - label: Trim facets + label: Trim residual image in: - id: image source: From 49241d1f0299c35e15d003d0b9a567046fb19738 Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 7 Aug 2026 11:24:45 +0200 Subject: [PATCH 17/24] lower multiscal-scale-bias for lower chance on diverging imaging --- steps/wsclean.cwl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/steps/wsclean.cwl b/steps/wsclean.cwl index c345dde3..09ebe0fe 100644 --- a/steps/wsclean.cwl +++ b/steps/wsclean.cwl @@ -137,7 +137,7 @@ inputs: prefix: '-niter' - id: multiscale-scale-bias type: float? - default: 0.7 + default: 0.6 inputBinding: position: 1 shellQuote: false From a01edae5a1869582a6002c213aa0c50794105493 Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 7 Aug 2026 11:30:28 +0200 Subject: [PATCH 18/24] add briggs option --- steps/wsclean.cwl | 9 +++++---- workflows/facet_imaging.cwl | 9 ++++++++- workflows/subworkflows/image_and_trim.cwl | 7 +++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/steps/wsclean.cwl b/steps/wsclean.cwl index 09ebe0fe..dd1906e1 100644 --- a/steps/wsclean.cwl +++ b/steps/wsclean.cwl @@ -51,13 +51,14 @@ inputs: position: 1 shellQuote: false prefix: '-minuv-l' - - id: weight - type: string? - default: briggs -1.4 + - id: briggs + type: float? + default: -1.4 inputBinding: position: 1 shellQuote: false - prefix: '-weight' + prefix: '-weight briggs' + itemSeparator: ' ' - id: weighting-rank-filter type: int? default: 3 diff --git a/workflows/facet_imaging.cwl b/workflows/facet_imaging.cwl index 48d51ec6..13f2e5be 100644 --- a/workflows/facet_imaging.cwl +++ b/workflows/facet_imaging.cwl @@ -40,6 +40,11 @@ inputs: doc: | Restoring beam to use for every facet following the WSClean order of major axis, minor axis, position angle. + - id: briggs + type: float? + default: -1.4 + doc: Briggs weighting for WSClean. + - id: swarp_config type: File? doc: | @@ -54,7 +59,7 @@ inputs: - id: ncpu type: int? default: 24 - doc: The minimum number of cores that should be available for steps that require high I/O. + doc: The number of cores that WSClean will use. steps: - id: average_ms @@ -113,6 +118,8 @@ steps: source: pixel_scale - id: resolution source: resolution + - id: briggs + source: briggs - id: tmpdir source: tmpdir - id: ncpu diff --git a/workflows/subworkflows/image_and_trim.cwl b/workflows/subworkflows/image_and_trim.cwl index b5843a87..694bf338 100644 --- a/workflows/subworkflows/image_and_trim.cwl +++ b/workflows/subworkflows/image_and_trim.cwl @@ -27,6 +27,11 @@ inputs: default: 0.3asec doc: Angular resolution that will be passed to WSClean's taper argument. Its syntax follows that of WSClean. + - id: briggs + type: float? + default: -1.4 + doc: Briggs weighting for WSClean. + - id: facet_polygon type: File doc: DS9 region file that will be used to trim the facet. @@ -76,6 +81,8 @@ steps: valueFrom: $(self.toString() + "asec") - id: beam-shape source: restoring_beam + - id: briggs + source: briggs - id: tmpdir source: tmpdir - id: apply-facet-beam From 109f79899cfde23bfbadb365588e94a6b4db21f7 Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 7 Aug 2026 11:43:15 +0200 Subject: [PATCH 19/24] remove item sep --- steps/wsclean.cwl | 1 - 1 file changed, 1 deletion(-) diff --git a/steps/wsclean.cwl b/steps/wsclean.cwl index dd1906e1..ac3e29f0 100644 --- a/steps/wsclean.cwl +++ b/steps/wsclean.cwl @@ -58,7 +58,6 @@ inputs: position: 1 shellQuote: false prefix: '-weight briggs' - itemSeparator: ' ' - id: weighting-rank-filter type: int? default: 3 From ef167bff93c51a568497f0efcd6f7414c5619fcb Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 7 Aug 2026 12:16:57 +0200 Subject: [PATCH 20/24] calculate ncpu --- steps/wsclean.cwl | 17 ++++++++++++++--- workflows/facet_imaging.cwl | 5 +++-- workflows/subworkflows/image_and_trim.cwl | 5 +++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/steps/wsclean.cwl b/steps/wsclean.cwl index ac3e29f0..ce99d2ac 100644 --- a/steps/wsclean.cwl +++ b/steps/wsclean.cwl @@ -26,11 +26,12 @@ inputs: prefix: '-temp-dir' - id: ncpu type: int? - default: 24 inputBinding: position: 1 shellQuote: false prefix: '-j' + valueFrom: | + ${ return self !== null ? self : runtime.cores; } - id: size type: int[]? default: [22500, 22500] @@ -347,11 +348,21 @@ hints: requirements: - class: ShellCommandRequirement + - class: InlineJavascriptRequirement + expressionLib: + - | + function wsclean_cores(size) { + var imsize = Math.max(size[0], size[1]); + var raw = imsize / 512.0; + var cores = Math.round(raw / 4.0) * 4; // nearest multiple of 4 + return Math.max(8, Math.min(64, cores)); + } + - class: ResourceRequirement + coresMin: | + ${ return inputs.ncpu !== null ? inputs.ncpu : wsclean_cores(inputs.image_size); } - class: InitialWorkDirRequirement listing: - entry: $(inputs.msin) - - class: ResourceRequirement - coresMin: $(inputs.ncpu) stdout: wsclean.log stderr: wsclean_err.log diff --git a/workflows/facet_imaging.cwl b/workflows/facet_imaging.cwl index 13f2e5be..8b89a35e 100644 --- a/workflows/facet_imaging.cwl +++ b/workflows/facet_imaging.cwl @@ -58,8 +58,9 @@ inputs: - id: ncpu type: int? - default: 24 - doc: The number of cores that WSClean will use. + doc: | + The number of cores that WSClean will use. + Default is to calculate it internally based on image size steps: - id: average_ms diff --git a/workflows/subworkflows/image_and_trim.cwl b/workflows/subworkflows/image_and_trim.cwl index 694bf338..7a15fc52 100644 --- a/workflows/subworkflows/image_and_trim.cwl +++ b/workflows/subworkflows/image_and_trim.cwl @@ -15,8 +15,9 @@ inputs: - id: ncpu type: int? - default: 24 - doc: The number of cores that WSClean will use. + doc: | + The number of cores that WSClean will use. + Default is to calculate it internally based on image size - id: pixel_scale type: float From 0a22eab16827cbcf23423c259fc15f42bcd6312c Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 7 Aug 2026 12:17:50 +0200 Subject: [PATCH 21/24] calculate ncpu --- steps/wsclean.cwl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/steps/wsclean.cwl b/steps/wsclean.cwl index ce99d2ac..32016cfa 100644 --- a/steps/wsclean.cwl +++ b/steps/wsclean.cwl @@ -359,7 +359,7 @@ requirements: } - class: ResourceRequirement coresMin: | - ${ return inputs.ncpu !== null ? inputs.ncpu : wsclean_cores(inputs.image_size); } + ${ return inputs.ncpu !== null ? inputs.ncpu : wsclean_cores(inputs.size); } - class: InitialWorkDirRequirement listing: - entry: $(inputs.msin) From 3a9704470e97caf41a3de650f652d72770d31d10 Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 7 Aug 2026 15:28:11 +0200 Subject: [PATCH 22/24] avgstep to averaging_factor --- steps/dp3_avg_step.cwl | 6 +++--- workflows/facet_imaging.cwl | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/steps/dp3_avg_step.cwl b/steps/dp3_avg_step.cwl index 914f00bf..14b7b642 100644 --- a/steps/dp3_avg_step.cwl +++ b/steps/dp3_avg_step.cwl @@ -15,7 +15,7 @@ inputs: prefix: msin= separate: false - - id: avgstep + - id: averaging_factor type: int? doc: Averaging factor @@ -37,8 +37,8 @@ outputs: arguments: - steps=[avg] - avg.type=averager - - valueFrom: $("avg.timestep=" + inputs.avgstep) - - valueFrom: $("avg.freqstep=" + inputs.avgstep) + - valueFrom: $("avg.timestep=" + inputs.averaging_factor) + - valueFrom: $("avg.freqstep=" + inputs.averaging_factor) - msout.storagemanager='dysco' - msout.storagemanager.databitrate=6 - msout=$( inputs.msin.basename + '.avg.ms') diff --git a/workflows/facet_imaging.cwl b/workflows/facet_imaging.cwl index 8b89a35e..2b804aeb 100644 --- a/workflows/facet_imaging.cwl +++ b/workflows/facet_imaging.cwl @@ -25,7 +25,7 @@ inputs: type: string doc: Angular resolution that will be passed to WSClean's taper argument. Its syntax follows that of WSClean. - - id: avgstep + - id: averaging_factor type: int? doc: Extra averaging step over time and frequency @@ -68,13 +68,13 @@ steps: in: - id: msin source: msin - - id: avgstep - source: avgstep + - id: averaging_factor + source: averaging_factor out: - ms_avg run: ../steps/dp3_avg_step.cwl scatter: msin - when: $(inputs.avgstep != null) + when: $(inputs.averaging_factor != null) - id: sort_mses label: Sort MS based on name From e09ad5074f4888b240c177d474c38c4c5bacd7bb Mon Sep 17 00:00:00 2001 From: Jurjen de Jong Date: Fri, 7 Aug 2026 15:28:48 +0200 Subject: [PATCH 23/24] Update workflows/facet_imaging.cwl Co-authored-by: Frits Sweijen --- workflows/facet_imaging.cwl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/facet_imaging.cwl b/workflows/facet_imaging.cwl index 2b804aeb..a74d91ca 100644 --- a/workflows/facet_imaging.cwl +++ b/workflows/facet_imaging.cwl @@ -27,7 +27,7 @@ inputs: - id: averaging_factor type: int? - doc: Extra averaging step over time and frequency + doc: Additional factor to average the data with in both time and frequency before imaging. - id: facet_polygons type: File[] From 58ed4f3bd1a4aea8db9d7744ab37a5349a5fc977 Mon Sep 17 00:00:00 2001 From: jurjen93 Date: Fri, 7 Aug 2026 15:30:25 +0200 Subject: [PATCH 24/24] make averaging factor > 1 --- workflows/facet_imaging.cwl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflows/facet_imaging.cwl b/workflows/facet_imaging.cwl index a74d91ca..82e26c76 100644 --- a/workflows/facet_imaging.cwl +++ b/workflows/facet_imaging.cwl @@ -27,6 +27,7 @@ inputs: - id: averaging_factor type: int? + default: 1 doc: Additional factor to average the data with in both time and frequency before imaging. - id: facet_polygons @@ -74,7 +75,7 @@ steps: - ms_avg run: ../steps/dp3_avg_step.cwl scatter: msin - when: $(inputs.averaging_factor != null) + when: $(inputs.averaging_factor > 1) - id: sort_mses label: Sort MS based on name