From d75b705eee80c0ff91da3f87d834cbdbee61d348 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 27 May 2026 11:24:31 +0100 Subject: [PATCH 01/25] Remove faceselfcal run from split-directions --- workflows/split-directions.cwl | 39 ---------------------------------- 1 file changed, 39 deletions(-) diff --git a/workflows/split-directions.cwl b/workflows/split-directions.cwl index d5e57353..6cdb322d 100644 --- a/workflows/split-directions.cwl +++ b/workflows/split-directions.cwl @@ -49,10 +49,6 @@ inputs: type: float default: 0.0 doc: Peak flux (Jy/beam) cut to pre-select sources from catalogue. Default at 0.0 is no peak flux selection. - - id: configfile - type: File? - default: null - doc: The configuration file to be used to run facetselfcal.py during the target_selfcal step. - id: frequency_resolution type: string? default: '390.56kHz' @@ -144,25 +140,6 @@ steps: when: $(inputs.dd_selection) run: ./subworkflows/phasediff_selection.cwl - - id: target_selfcal - label: Target Selfcal - in: - - id: msin - source: - - phasediff_selection/best_ms - - flatten_msout/flattenedarray - pickValue: first_non_null - - id: configfile - source: configfile - - id: do_selfcal - source: do_selfcal - out: - - id: images - - id: h5parm - when: $(inputs.do_selfcal) - run: ../steps/facet_selfcal.cwl - scatter: msin - outputs: - id: msout_concat type: Directory[] @@ -170,22 +147,6 @@ outputs: - phasediff_selection/best_ms - flatten_msout/flattenedarray pickValue: first_non_null - - id: images - type: - type: array - items: - type: array - items: File - outputSource: - - target_selfcal/images - pickValue: the_only_non_null - id: phasediff_score_csv type: File? outputSource: phasediff_selection/phasediff_score_csv - - id: h5parm - type: - - File - - File[] - outputSource: - - target_selfcal/h5parm - pickValue: the_only_non_null From 36e1e8ded382ba1988dcadca44ec118fd18619a3 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 27 May 2026 11:29:56 +0100 Subject: [PATCH 02/25] Tweak docstring --- workflows/split-directions.cwl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/workflows/split-directions.cwl b/workflows/split-directions.cwl index 6cdb322d..bec931bc 100644 --- a/workflows/split-directions.cwl +++ b/workflows/split-directions.cwl @@ -7,8 +7,7 @@ doc: | * Splits a LOFAR MeasurementSet into various target directions * Applies delay calibrator solutions * Optionally (for wide-field imaging) performs direction-dependent calibrator selection - * Optionally performs self-calibration on the target directions - This step should be run after the delay calibration workflow. + This step should be run after the delay calibration workflow if run manually, but is typically run automatically by dd-calibration. requirements: - class: SubworkflowFeatureRequirement From 8bd762d658f201bcb28f1b8cfd2a0fdc72468ebc Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 27 May 2026 12:03:32 +0100 Subject: [PATCH 03/25] Remove do_selfcal parameter --- workflows/split-directions.cwl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/workflows/split-directions.cwl b/workflows/split-directions.cwl index bec931bc..188c9048 100644 --- a/workflows/split-directions.cwl +++ b/workflows/split-directions.cwl @@ -26,10 +26,6 @@ inputs: type: File doc: The image catalogue (in FITS or CSV format) containing the target directions. default: lotss_catalogue.csv - - id: do_selfcal - type: boolean? - default: false - doc: Whether to do selfcal on the direction concat MSs. - id: dd_selection type: boolean? default: false From f707fe73c677b7c3981cc224aa6a9db2722c93bc Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 27 May 2026 12:06:18 +0100 Subject: [PATCH 04/25] Update workflows/split-directions.cwl Co-authored-by: Matthijs van der Wild --- workflows/split-directions.cwl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/split-directions.cwl b/workflows/split-directions.cwl index 188c9048..5f42eff6 100644 --- a/workflows/split-directions.cwl +++ b/workflows/split-directions.cwl @@ -7,7 +7,7 @@ doc: | * Splits a LOFAR MeasurementSet into various target directions * Applies delay calibrator solutions * Optionally (for wide-field imaging) performs direction-dependent calibrator selection - This step should be run after the delay calibration workflow if run manually, but is typically run automatically by dd-calibration. + This workflow should be run on data processed by the delay calibration workflow if run manually, but is typically run automatically by dd-calibration. requirements: - class: SubworkflowFeatureRequirement From 5cd1c0b2545b4945059bb123fcf785ed41cc38b6 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 19 Feb 2026 14:17:43 +0000 Subject: [PATCH 05/25] Don't pass config to split-directions in auto selection --- workflows/subworkflows/find-best-delay-calibrator.cwl | 2 -- 1 file changed, 2 deletions(-) diff --git a/workflows/subworkflows/find-best-delay-calibrator.cwl b/workflows/subworkflows/find-best-delay-calibrator.cwl index df68640b..2a0e59a5 100644 --- a/workflows/subworkflows/find-best-delay-calibrator.cwl +++ b/workflows/subworkflows/find-best-delay-calibrator.cwl @@ -69,8 +69,6 @@ steps: source: delay_calibrator - id: select_best_n source: select_best_n_delay_calibrators - - id: configfile - source: configfile - id: frequency_resolution source: frequency_resolution - id: time_resolution From b56d51560265e35e32911cab8bf62837d4d24702 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 1 Apr 2026 17:03:10 +0200 Subject: [PATCH 06/25] Update auto delay calibration with automatic config generation --- steps/lofar_vlbi_plot.cwl | 5 +++ workflows/delay-calibration.cwl | 7 +++ .../find-best-delay-calibrator.cwl | 43 +++++++++++++------ 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/steps/lofar_vlbi_plot.cwl b/steps/lofar_vlbi_plot.cwl index a67950ac..9f7a106f 100644 --- a/steps/lofar_vlbi_plot.cwl +++ b/steps/lofar_vlbi_plot.cwl @@ -22,6 +22,11 @@ outputs: doc: Delay Calibrator CSV outputBinding: glob: delay_calibrators.csv + - id: image_catalogue_pf + type: File + doc: Image catalogue CSV + outputBinding: + glob: image_catalogue.csv arguments: - --continue_no_lotss diff --git a/workflows/delay-calibration.cwl b/workflows/delay-calibration.cwl index 1a39986a..1f2a04a7 100644 --- a/workflows/delay-calibration.cwl +++ b/workflows/delay-calibration.cwl @@ -206,6 +206,7 @@ steps: source: delay_calibrator out: - id: delay_calibrator_pf + - id: image_catalogue_pf run: ../steps/lofar_vlbi_plot.cwl when: $(inputs.delay_calibrator == null) @@ -328,6 +329,12 @@ steps: - lofar_vlbi_plot/delay_calibrator_pf pickValue: first_non_null valueFrom: $(self) + - id: image_catalogue + source: + - image_catalogue + - lofar_vlbi_plot/image_catalogue_pf + pickValue: first_non_null + valueFrom: $(self) - id: select_best_n_delay_calibrators source: select_best_n_delay_calibrators - id: do_auto_delay_selection diff --git a/workflows/subworkflows/find-best-delay-calibrator.cwl b/workflows/subworkflows/find-best-delay-calibrator.cwl index 2a0e59a5..bdf97f1c 100644 --- a/workflows/subworkflows/find-best-delay-calibrator.cwl +++ b/workflows/subworkflows/find-best-delay-calibrator.cwl @@ -19,14 +19,19 @@ inputs: - id: delay_calibrator type: File doc: A delay calibrator catalogue in CSV format. + - id: image_catalogue + type: File + doc: A catalogue with other sources in the field (e.g. a LoTSS catalogue). - id: configfile type: File doc: Settings for the delay calibration in delay_solve. - id: starting_skymodel type: - - File[]? + - File[] + # This default avoids validation warnings for the starting_skymodels output. + default: [] doc: | - Optional starting models in BBS-compatible text format used to kickstart the delay calibration. If given, the number of skymodels must be equal to `select_best_n_delay_calibrators`. Additionally, they should be named in such a way that when sorted by name, the delay calibrator MSes and skymodels end up in the same order. + Optional starting models in FITS format used to kickstart the delay calibration. If given, the number of skymodels must be equal to `select_best_n_delay_calibrators`. Additionally, they should be named in such a way that when sorted by name, the delay calibrator MSes and skymodels end up in the same order. - id: select_best_n_delay_calibrators type: int? default: 1 @@ -52,10 +57,13 @@ steps: source: delay_calibrator - id: process_all default: true + - id: starting_skymodel + default: starting_skymodel out: - id: skymodel - id: logfile run: ../../steps/delay_cal_model.cwl + when: $(inputs.starting_skymodel.length == 0) - id: select_best_delay_cal in: @@ -84,7 +92,7 @@ steps: - id: sort_skymodels in: - id: input_entry - source: + source: - starting_skymodel - generate_skymodels/skymodel pickValue: all_non_null @@ -106,21 +114,26 @@ steps: in: - id: msin source: sort_ms/sorted_entries - - id: configfile - source: configfile - - id: skymodel + - id: delay_calibrator + source: delay_calibrator + - id: image_catalogue + source: image_catalogue + - id: model_image source: sort_skymodels/sorted_entries out: - - id: images - - id: h5parm - run: ../../steps/facet_selfcal.cwl - scatter: [msin, skymodel] + - id: solutions + - id: starting_skymodel + - id: config + - id: pictures + - id: logfile + run: ./delay_cal_run.cwl + scatter: [msin, model_image] scatterMethod: dotproduct - id: flatten_delay_images in: - id: nestedarray - source: delay_selfcal/images + source: delay_selfcal/pictures out: - id: flattenedarray run: ../../steps/flatten.cwl @@ -137,7 +150,11 @@ outputs: format, phase-shifted to the delay calibrator. - id: starting_skymodels - outputSource: generate_skymodels/skymodel + outputSource: + - starting_skymodel + - generate_skymodels/skymodel + pickValue: all_non_null + linkMerge: merge_flattened type: File[] doc: Starting models that were used to kickstart the delay calibration. @@ -157,7 +174,7 @@ outputs: A CSV file containing the phasediff scores for each of the calibrators that were split out. - id: solutions - outputSource: delay_selfcal/h5parm + outputSource: delay_selfcal/solutions type: - File - File[] From a54486442071073e8eefcc5e45192bb32d6596d2 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 3 Apr 2026 11:27:26 +0200 Subject: [PATCH 07/25] Don't run selfcal in split-directions --- workflows/subworkflows/find-best-delay-calibrator.cwl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/subworkflows/find-best-delay-calibrator.cwl b/workflows/subworkflows/find-best-delay-calibrator.cwl index bdf97f1c..df23b6c6 100644 --- a/workflows/subworkflows/find-best-delay-calibrator.cwl +++ b/workflows/subworkflows/find-best-delay-calibrator.cwl @@ -72,7 +72,7 @@ steps: - id: dd_selection default: true - id: do_selfcal - default: true + default: false - id: image_cat source: delay_calibrator - id: select_best_n From 11783efb5eddbc88e936a2906f8bbd61f67ef688 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 8 Apr 2026 12:03:46 +0100 Subject: [PATCH 08/25] Fix skymodel trigger and validation --- workflows/delay-calibration.cwl | 6 +++--- .../subworkflows/find-best-delay-calibrator.cwl | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/workflows/delay-calibration.cwl b/workflows/delay-calibration.cwl index 1f2a04a7..d857594d 100644 --- a/workflows/delay-calibration.cwl +++ b/workflows/delay-calibration.cwl @@ -467,9 +467,9 @@ outputs: outputSource: - phaseup/starting_skymodel - select_best_delay_cal/starting_skymodels - type: File[] - pickValue: all_non_null - linkMerge: merge_flattened + type: File[]? + pickValue: first_non_null + linkMerge: merge_nested doc: | The starting model(s) that were used to kick start the delay calibration. diff --git a/workflows/subworkflows/find-best-delay-calibrator.cwl b/workflows/subworkflows/find-best-delay-calibrator.cwl index df23b6c6..e180f237 100644 --- a/workflows/subworkflows/find-best-delay-calibrator.cwl +++ b/workflows/subworkflows/find-best-delay-calibrator.cwl @@ -27,9 +27,7 @@ inputs: doc: Settings for the delay calibration in delay_solve. - id: starting_skymodel type: - - File[] - # This default avoids validation warnings for the starting_skymodels output. - default: [] + - File[]? doc: | Optional starting models in FITS format used to kickstart the delay calibration. If given, the number of skymodels must be equal to `select_best_n_delay_calibrators`. Additionally, they should be named in such a way that when sorted by name, the delay calibrator MSes and skymodels end up in the same order. - id: select_best_n_delay_calibrators @@ -63,7 +61,7 @@ steps: - id: skymodel - id: logfile run: ../../steps/delay_cal_model.cwl - when: $(inputs.starting_skymodel.length == 0) + when: $(inputs.starting_skymodel == null) - id: select_best_delay_cal in: @@ -153,9 +151,11 @@ outputs: outputSource: - starting_skymodel - generate_skymodels/skymodel - pickValue: all_non_null - linkMerge: merge_flattened - type: File[] + pickValue: first_non_null + linkMerge: merge_nested + #pickValue: all_non_null + #linkMerge: merge_flattened + type: File[]? doc: Starting models that were used to kickstart the delay calibration. - id: pictures From 97b42b37ba9ac8d02d4991792631be790066f3bd Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 23 Apr 2026 18:01:53 +0100 Subject: [PATCH 09/25] Remove commented code --- workflows/subworkflows/find-best-delay-calibrator.cwl | 2 -- 1 file changed, 2 deletions(-) diff --git a/workflows/subworkflows/find-best-delay-calibrator.cwl b/workflows/subworkflows/find-best-delay-calibrator.cwl index e180f237..71341244 100644 --- a/workflows/subworkflows/find-best-delay-calibrator.cwl +++ b/workflows/subworkflows/find-best-delay-calibrator.cwl @@ -153,8 +153,6 @@ outputs: - generate_skymodels/skymodel pickValue: first_non_null linkMerge: merge_nested - #pickValue: all_non_null - #linkMerge: merge_flattened type: File[]? doc: Starting models that were used to kickstart the delay calibration. From 50f6d10f01f560d2aba91b72a9ae20d42e368fdb Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 23 Apr 2026 18:07:38 +0100 Subject: [PATCH 10/25] Return the skymodels that were generated instead of the input images --- .../subworkflows/find-best-delay-calibrator.cwl | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/workflows/subworkflows/find-best-delay-calibrator.cwl b/workflows/subworkflows/find-best-delay-calibrator.cwl index 71341244..dcc2c7c3 100644 --- a/workflows/subworkflows/find-best-delay-calibrator.cwl +++ b/workflows/subworkflows/find-best-delay-calibrator.cwl @@ -137,6 +137,15 @@ steps: run: ../../steps/flatten.cwl label: flatten_delay_images + - id: flatten_delay_models + in: + - id: nestedarray + source: delay_selfcal/starting_skymodel + out: + - id: flattenedarray + run: ../../steps/flatten.cwl + label: flatten_delay_images + outputs: - id: msout outputSource: @@ -149,12 +158,12 @@ outputs: - id: starting_skymodels outputSource: - - starting_skymodel + - flatten_delay_models/flattenedarray - generate_skymodels/skymodel pickValue: first_non_null linkMerge: merge_nested type: File[]? - doc: Starting models that were used to kickstart the delay calibration. + doc: Starting skymodels that were used to kickstart the delay calibration. - id: pictures outputSource: From 05350e5cc8486f5815c80fcc64c9bea42ade4f6e Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 24 Apr 2026 14:55:38 +0100 Subject: [PATCH 11/25] Improve output types per review --- workflows/delay-calibration.cwl | 3 +-- workflows/subworkflows/find-best-delay-calibrator.cwl | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/workflows/delay-calibration.cwl b/workflows/delay-calibration.cwl index d857594d..fc644adb 100644 --- a/workflows/delay-calibration.cwl +++ b/workflows/delay-calibration.cwl @@ -467,9 +467,8 @@ outputs: outputSource: - phaseup/starting_skymodel - select_best_delay_cal/starting_skymodels - type: File[]? + type: File[] pickValue: first_non_null - linkMerge: merge_nested doc: | The starting model(s) that were used to kick start the delay calibration. diff --git a/workflows/subworkflows/find-best-delay-calibrator.cwl b/workflows/subworkflows/find-best-delay-calibrator.cwl index dcc2c7c3..b77e3e0b 100644 --- a/workflows/subworkflows/find-best-delay-calibrator.cwl +++ b/workflows/subworkflows/find-best-delay-calibrator.cwl @@ -162,7 +162,7 @@ outputs: - generate_skymodels/skymodel pickValue: first_non_null linkMerge: merge_nested - type: File[]? + type: File[] doc: Starting skymodels that were used to kickstart the delay calibration. - id: pictures From fac3eef96b30b66c06d48fce8e67b6367465ade0 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 24 Apr 2026 14:56:03 +0100 Subject: [PATCH 12/25] Remove unused configfile input --- workflows/subworkflows/find-best-delay-calibrator.cwl | 3 --- 1 file changed, 3 deletions(-) diff --git a/workflows/subworkflows/find-best-delay-calibrator.cwl b/workflows/subworkflows/find-best-delay-calibrator.cwl index b77e3e0b..12f8a7e6 100644 --- a/workflows/subworkflows/find-best-delay-calibrator.cwl +++ b/workflows/subworkflows/find-best-delay-calibrator.cwl @@ -22,9 +22,6 @@ inputs: - id: image_catalogue type: File doc: A catalogue with other sources in the field (e.g. a LoTSS catalogue). - - id: configfile - type: File - doc: Settings for the delay calibration in delay_solve. - id: starting_skymodel type: - File[]? From 4ac363a43d4f294e026b2eb9b2684de6faa59437 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 24 Apr 2026 15:03:51 +0100 Subject: [PATCH 13/25] Remove remaining configfile options and simplify skymodel output further --- workflows/delay-calibration.cwl | 6 ------ workflows/subworkflows/find-best-delay-calibrator.cwl | 1 - 2 files changed, 7 deletions(-) diff --git a/workflows/delay-calibration.cwl b/workflows/delay-calibration.cwl index fc644adb..79cca5a8 100644 --- a/workflows/delay-calibration.cwl +++ b/workflows/delay-calibration.cwl @@ -71,10 +71,6 @@ inputs: default: TGSSphase doc: The name of the target solution table to use from the solset input. - - id: configfile - type: File - doc: Settings for the delay calibration in delay_solve. - - id: reference_stationSB type: int? default: 104 @@ -321,8 +317,6 @@ steps: linkMerge: merge_nested pickValue: first_non_null valueFrom: $(self) - - id: configfile - source: configfile - id: delay_calibrator source: - delay_calibrator diff --git a/workflows/subworkflows/find-best-delay-calibrator.cwl b/workflows/subworkflows/find-best-delay-calibrator.cwl index 12f8a7e6..2685b8b2 100644 --- a/workflows/subworkflows/find-best-delay-calibrator.cwl +++ b/workflows/subworkflows/find-best-delay-calibrator.cwl @@ -158,7 +158,6 @@ outputs: - flatten_delay_models/flattenedarray - generate_skymodels/skymodel pickValue: first_non_null - linkMerge: merge_nested type: File[] doc: Starting skymodels that were used to kickstart the delay calibration. From 3472234c598cfe6a5a05d14e2c79e5ba3bb2b067 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 24 Apr 2026 15:05:12 +0100 Subject: [PATCH 14/25] Add lofar-vlbi-plot as a dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 27ddf305..1aff56f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ dependencies = [ "LINC @ git+https://git.astron.nl/RD/LINC", "lofar_facet_selfcal @ git+https://github.com/rvweeren/lofar_facet_selfcal", "lofar_helpers @ git+https://github.com/jurjen93/lofar_helpers", + "lofar-vlbi-plot", "losoto", "numpy", "pandas", From 292a503db3785309e42c15c0b425fc6a4807c03d Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 6 May 2026 13:14:16 +0100 Subject: [PATCH 15/25] Return configs for each delay calibrator --- .../find-best-delay-calibrator.cwl | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/workflows/subworkflows/find-best-delay-calibrator.cwl b/workflows/subworkflows/find-best-delay-calibrator.cwl index 2685b8b2..ca417629 100644 --- a/workflows/subworkflows/find-best-delay-calibrator.cwl +++ b/workflows/subworkflows/find-best-delay-calibrator.cwl @@ -141,7 +141,16 @@ steps: out: - id: flattenedarray run: ../../steps/flatten.cwl - label: flatten_delay_images + label: flatten_delay_models + + - id: flatten_delay_configs + in: + - id: nestedarray + source: delay_selfcal/config + out: + - id: flattenedarray + run: ../../steps/flatten.cwl + label: flatten_delay_configs outputs: - id: msout @@ -169,6 +178,14 @@ outputs: linkMerge: merge_flattened doc: Inspection plots generated by lofar_facet_selfcal. + - id: configs + outputSource: + - flatten_delay_configs/flattenedarray + type: File[] + pickValue: all_non_null + linkMerge: merge_flattened + doc: Facetselfcal config files used for the calibration. + - id: phasediff_score_csv outputSource: select_best_delay_cal/phasediff_score_csv type: From b7d04a11e49e6cf009f361812085bff59381d34a Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 7 May 2026 12:56:01 +0100 Subject: [PATCH 16/25] Fix config output --- workflows/subworkflows/find-best-delay-calibrator.cwl | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/workflows/subworkflows/find-best-delay-calibrator.cwl b/workflows/subworkflows/find-best-delay-calibrator.cwl index ca417629..48d84fcf 100644 --- a/workflows/subworkflows/find-best-delay-calibrator.cwl +++ b/workflows/subworkflows/find-best-delay-calibrator.cwl @@ -143,15 +143,6 @@ steps: run: ../../steps/flatten.cwl label: flatten_delay_models - - id: flatten_delay_configs - in: - - id: nestedarray - source: delay_selfcal/config - out: - - id: flattenedarray - run: ../../steps/flatten.cwl - label: flatten_delay_configs - outputs: - id: msout outputSource: @@ -180,7 +171,7 @@ outputs: - id: configs outputSource: - - flatten_delay_configs/flattenedarray + - delay_selfcal/config type: File[] pickValue: all_non_null linkMerge: merge_flattened From fe7a7120be7b580e83933eebcb20b0b92726e6a3 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 7 May 2026 13:03:15 +0100 Subject: [PATCH 17/25] Make image_catalogue optional --- workflows/delay-calibration.cwl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/delay-calibration.cwl b/workflows/delay-calibration.cwl index 79cca5a8..f36aa4c2 100644 --- a/workflows/delay-calibration.cwl +++ b/workflows/delay-calibration.cwl @@ -43,7 +43,7 @@ inputs: If not provided, lofar-vlbi-plot will be run to generate the CSV (requires internet access). - id: image_catalogue - type: File + type: File? doc: An image catalogue file in CSV format. - id: Ateam_skymodel From 3784ae9b2b9a042c4df384b41372f23285945898 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 7 May 2026 14:42:51 +0100 Subject: [PATCH 18/25] Correct input key for model generation --- workflows/subworkflows/find-best-delay-calibrator.cwl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/subworkflows/find-best-delay-calibrator.cwl b/workflows/subworkflows/find-best-delay-calibrator.cwl index 48d84fcf..fb5336ee 100644 --- a/workflows/subworkflows/find-best-delay-calibrator.cwl +++ b/workflows/subworkflows/find-best-delay-calibrator.cwl @@ -52,7 +52,7 @@ steps: source: delay_calibrator - id: process_all default: true - - id: starting_skymodel + - id: model_image default: starting_skymodel out: - id: skymodel From ef7650faf0b2868121ad5e6dc395b0d0aef2544d Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Thu, 7 May 2026 20:35:11 +0100 Subject: [PATCH 19/25] Remove unnecessary step --- steps/delay_cal_model.cwl | 3 --- .../find-best-delay-calibrator.cwl | 19 ------------------- 2 files changed, 22 deletions(-) diff --git a/steps/delay_cal_model.cwl b/steps/delay_cal_model.cwl index e82bfc65..3517b2e6 100644 --- a/steps/delay_cal_model.cwl +++ b/steps/delay_cal_model.cwl @@ -29,9 +29,6 @@ inputs: position: 2 prefix: --model-image separate: true - valueFrom: $(self.basename) - - - id: process_all type: boolean? default: false diff --git a/workflows/subworkflows/find-best-delay-calibrator.cwl b/workflows/subworkflows/find-best-delay-calibrator.cwl index fb5336ee..9d514ee8 100644 --- a/workflows/subworkflows/find-best-delay-calibrator.cwl +++ b/workflows/subworkflows/find-best-delay-calibrator.cwl @@ -43,23 +43,6 @@ inputs: Time resolution to average the split off delay calibrators to. steps: - - id: generate_skymodels - in: - - id: msin - source: msin - valueFrom: $(self[0]) - - id: delay_calibrator - source: delay_calibrator - - id: process_all - default: true - - id: model_image - default: starting_skymodel - out: - - id: skymodel - - id: logfile - run: ../../steps/delay_cal_model.cwl - when: $(inputs.starting_skymodel == null) - - id: select_best_delay_cal in: - id: msin @@ -89,7 +72,6 @@ steps: - id: input_entry source: - starting_skymodel - - generate_skymodels/skymodel pickValue: all_non_null linkMerge: merge_flattened out: @@ -156,7 +138,6 @@ outputs: - id: starting_skymodels outputSource: - flatten_delay_models/flattenedarray - - generate_skymodels/skymodel pickValue: first_non_null type: File[] doc: Starting skymodels that were used to kickstart the delay calibration. From 3b3f9d597360f695654bf04c586922424c473d98 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Mon, 11 May 2026 14:13:22 +0100 Subject: [PATCH 20/25] Tweak cores for PyBDSF in skynet --- scripts/skynet.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/skynet.py b/scripts/skynet.py index d36e6031..d2a07048 100755 --- a/scripts/skynet.py +++ b/scripts/skynet.py @@ -38,12 +38,16 @@ def write_skymodel(model: str, outname: str): skymodel.write( '{:s}\n'.format(ss_to_write) ) def model_from_image(modelImage: str, smodel: float, opt_coords: SkyCoord, astroSearchRadius: float = 3.0, outdir: str = '.'): - img = bdsf.process_image(modelImage, mean_map='zero', rms_map=True, rms_box = (100,10), outdir=outdir) + # ncores=1 is set to disable multiprocessing as this sometimes triggers an + # issue with socket path lengths cause by deeply nested temporary directories + # that can occur during a pipeline run. + # The images processed here are small, so it shouldn't affect performance meaningfully. + img = bdsf.process_image(modelImage, mean_map='zero', rms_map=True, rms_box = (100,10), outdir=outdir, ncores=1) sources = img.sources maxval = 0. for src in sources: maxval = np.max( (maxval, src.total_flux) ) - img = bdsf.process_image(modelImage, mean_map='zero', rms_map=True, rms_box = (100,10), advanced_opts=True, blank_limit=0.01*maxval, outdir=outdir) + img = bdsf.process_image(modelImage, mean_map='zero', rms_map=True, rms_box = (100,10), advanced_opts=True, blank_limit=0.01*maxval, outdir=outdir, ncores=1) sources = img.sources # Scale model flux density to the provided value. tot_flux = 0. From 66fb8d417d5f9065fdb3ec956311ee747c0175b4 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Mon, 11 May 2026 15:18:54 +0100 Subject: [PATCH 21/25] Revert "Tweak cores for PyBDSF in skynet" This reverts commit aee623c5d52e556a6e90dee86ee51803d81eaeff. --- scripts/skynet.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/skynet.py b/scripts/skynet.py index d2a07048..d36e6031 100755 --- a/scripts/skynet.py +++ b/scripts/skynet.py @@ -38,16 +38,12 @@ def write_skymodel(model: str, outname: str): skymodel.write( '{:s}\n'.format(ss_to_write) ) def model_from_image(modelImage: str, smodel: float, opt_coords: SkyCoord, astroSearchRadius: float = 3.0, outdir: str = '.'): - # ncores=1 is set to disable multiprocessing as this sometimes triggers an - # issue with socket path lengths cause by deeply nested temporary directories - # that can occur during a pipeline run. - # The images processed here are small, so it shouldn't affect performance meaningfully. - img = bdsf.process_image(modelImage, mean_map='zero', rms_map=True, rms_box = (100,10), outdir=outdir, ncores=1) + img = bdsf.process_image(modelImage, mean_map='zero', rms_map=True, rms_box = (100,10), outdir=outdir) sources = img.sources maxval = 0. for src in sources: maxval = np.max( (maxval, src.total_flux) ) - img = bdsf.process_image(modelImage, mean_map='zero', rms_map=True, rms_box = (100,10), advanced_opts=True, blank_limit=0.01*maxval, outdir=outdir, ncores=1) + img = bdsf.process_image(modelImage, mean_map='zero', rms_map=True, rms_box = (100,10), advanced_opts=True, blank_limit=0.01*maxval, outdir=outdir) sources = img.sources # Scale model flux density to the provided value. tot_flux = 0. From 1fb8f834fbed8b2ecab095a417ba8c16b765439d Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 15 May 2026 14:40:03 +0100 Subject: [PATCH 22/25] Tweak doc and remove unnecessary input type (won't run on single sources) --- workflows/delay-calibration.cwl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/workflows/delay-calibration.cwl b/workflows/delay-calibration.cwl index f36aa4c2..67ea0a22 100644 --- a/workflows/delay-calibration.cwl +++ b/workflows/delay-calibration.cwl @@ -154,10 +154,11 @@ inputs: - id: starting_skymodel type: - - File? - File[]? doc: | - Optional starting model(s) in BBS-compatible text format used to kickstart the delay calibration. If given and `do_auto_delay_selection` is enabled, the number of skymodels must be equal to `select_best_n_delay_calibrators`. Additionally, they should be named in such a way that when sorted by name, the delay calibrator MSes and skymodels end up in the same order. + Optional starting model(s) in FITS format used to kickstart the delay calibration when automatic selection from multiple delay candidates is used. + If given and `do_auto_delay_selection` is enabled, the number of skymodels must be equal to `select_best_n_delay_calibrators`. + Additionally, they should be named in such a way that when sorted by name, the delay calibrator MSes and skymodels end up in the same order. - id: do_auto_delay_selection type: boolean? From 55aa540f15db25150e6ab64416bea13a76123dd7 Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 22 May 2026 15:36:40 +0100 Subject: [PATCH 23/25] Output the configs used in the delay cal --- workflows/delay-calibration.cwl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/workflows/delay-calibration.cwl b/workflows/delay-calibration.cwl index 67ea0a22..b4806bad 100644 --- a/workflows/delay-calibration.cwl +++ b/workflows/delay-calibration.cwl @@ -347,6 +347,7 @@ steps: - id: pictures - id: phasediff_score_csv - id: solutions + - id: configs run: ./subworkflows/find-best-delay-calibrator.cwl when: $(inputs.do_auto_delay_selection) @@ -454,7 +455,10 @@ outputs: - id: facetselfcal_config outputSource: - phaseup/facetselfcal_config - type: File + - select_best_delay_cal/configs + type: + - File + - File[] pickValue: the_only_non_null doc: Config file with settings used for the delay calibration. From 018670983ab2f26583dac2ee093867f1e7d8c17c Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Fri, 22 May 2026 15:52:16 +0100 Subject: [PATCH 24/25] Fix return of skymodels --- workflows/subworkflows/find-best-delay-calibrator.cwl | 1 - 1 file changed, 1 deletion(-) diff --git a/workflows/subworkflows/find-best-delay-calibrator.cwl b/workflows/subworkflows/find-best-delay-calibrator.cwl index 9d514ee8..1e06c0ad 100644 --- a/workflows/subworkflows/find-best-delay-calibrator.cwl +++ b/workflows/subworkflows/find-best-delay-calibrator.cwl @@ -138,7 +138,6 @@ outputs: - id: starting_skymodels outputSource: - flatten_delay_models/flattenedarray - pickValue: first_non_null type: File[] doc: Starting skymodels that were used to kickstart the delay calibration. From 66979f98a3868a92db75ac272adf8e77ea48cf9a Mon Sep 17 00:00:00 2001 From: Frits Sweijen Date: Wed, 27 May 2026 10:47:47 +0100 Subject: [PATCH 25/25] Remove unneeded outputs --- workflows/subworkflows/find-best-delay-calibrator.cwl | 2 -- 1 file changed, 2 deletions(-) diff --git a/workflows/subworkflows/find-best-delay-calibrator.cwl b/workflows/subworkflows/find-best-delay-calibrator.cwl index 1e06c0ad..2807e3cd 100644 --- a/workflows/subworkflows/find-best-delay-calibrator.cwl +++ b/workflows/subworkflows/find-best-delay-calibrator.cwl @@ -61,8 +61,6 @@ steps: source: time_resolution out: - id: msout_concat - - id: images - - id: h5parm - id: phasediff_score_csv run: ../split-directions.cwl label: select_best_delay_cal