Facet-imaging updates

steps/dp3_avg_step.cwl

@@ -0,0 +1,57 @@
+class: CommandLineTool
+cwlVersion: v1.2
+id: dp3_avg_step
+label: DP3 averaging for prediction
+doc: Average MeasurementSet in time and frequency for faster prediction.
+
+baseCommand: DP3
+
+inputs:
+  - id: msin
+    type: Directory
+    doc: Input MeasurementSet
+    inputBinding:
+      position: 0
+      prefix: msin=
+      separate: false
+
+  - id: averaging_factor
+    type: int?
+    doc: Averaging factor
+
+outputs:
+  - id: ms_avg
+    doc: MeasurementSet at lower time/freq resolution
+    type: Directory
+    outputBinding:
+      glob: "$( inputs.msin.basename + '.avg.ms')"
+
+  - id: logfile
+    type: File[]
+    outputBinding:
+      glob: predict_avg*.log
+    doc: |
+        The files containing the stdout
+        and stderr from the step.
+
+arguments:
+  - steps=[avg]
+  - avg.type=averager
+  - valueFrom: $("avg.timestep=" + inputs.averaging_factor)
+  - valueFrom: $("avg.freqstep=" + inputs.averaging_factor)
+  - msout.storagemanager='dysco'
+  - msout.storagemanager.databitrate=6
+  - msout=$( inputs.msin.basename + '.avg.ms')
+
+requirements:
+  - class: InlineJavascriptRequirement
+
+hints:
+  - class: DockerRequirement
+    dockerPull: vlbi-cwl
+  - class: ResourceRequirement
+    coresMin: 8
+
+stdout: dp3_avg_step.log
+stderr: dp3_avg_step_err.log
+

steps/wsclean.cwl

@@ -26,11 +26,12 @@ inputs:
       prefix: '-temp-dir'
   - id: ncpu
     type: int?
-    default: 24
     inputBinding:
       position: 1
       shellQuote: false
       prefix: '-j'
+      valueFrom: |
+        ${ return self !== null ? self : runtime.cores; }
   - id: size
     type: int[]?
     default: [22500, 22500]
@@ -51,13 +52,13 @@ inputs:
       position: 1
       shellQuote: false
       prefix: '-minuv-l'
-  - id: weight
-    type: string?
-    default: briggs -1.4
+  - id: briggs
+    type: float?
+    default: -1.4
     inputBinding:
       position: 1
       shellQuote: false
-      prefix: '-weight'
+      prefix: '-weight briggs'
   - id: weighting-rank-filter
     type: int?
     default: 3
@@ -137,7 +138,7 @@ inputs:
       prefix: '-niter'
   - id: multiscale-scale-bias
     type: float?
-    default: 0.7
+    default: 0.6
     inputBinding:
       position: 1
       shellQuote: false
@@ -347,11 +348,21 @@ hints:
 
 requirements:
   - class: ShellCommandRequirement
+  - class: InlineJavascriptRequirement
+    expressionLib:
+      - |
+        function wsclean_cores(size) {
+          var imsize = Math.max(size[0], size[1]);
+          var raw = imsize / 512.0;
+          var cores = Math.round(raw / 4.0) * 4;   // nearest multiple of 4
+          return Math.max(8, Math.min(64, cores));
+        }
+  - class: ResourceRequirement
+    coresMin: |
+      ${ return inputs.ncpu !== null ? inputs.ncpu : wsclean_cores(inputs.size); }
   - class: InitialWorkDirRequirement
     listing:
       - entry: $(inputs.msin)
-  - class: ResourceRequirement
-    coresMin: $(inputs.ncpu)
 
 stdout: wsclean.log
 stderr: wsclean_err.log

workflows/facet_imaging.cwl

@@ -10,6 +10,7 @@ requirements:
     - class: ScatterFeatureRequirement
     - class: SubworkflowFeatureRequirement
     - class: InlineJavascriptRequirement
+    - class: MultipleInputFeatureRequirement
 
 inputs:
     - id: msin
@@ -24,6 +25,11 @@ inputs:
       type: string
       doc: Angular resolution that will be passed to WSClean's taper argument. Its syntax follows that of WSClean.
 
+    - id: averaging_factor
+      type: int?
+      default: 1
+      doc: Additional factor to average the data with in both time and frequency before imaging.
+
     - id: facet_polygons
       type: File[]
       doc: |
@@ -35,6 +41,11 @@ inputs:
       doc: |
         Restoring beam to use for every facet following the WSClean order of major axis, minor axis, position angle.
 
+    - id: briggs
+      type: float?
+      default: -1.4
+      doc: Briggs weighting for WSClean.
+
     - id: swarp_config
       type: File?
       doc: |
@@ -46,18 +57,51 @@ inputs:
       type: string?
       doc: Temporary directory to run I/O heavy jobs.
 
+    - id: ncpu
+      type: int?
+      doc: |
+        The number of cores that WSClean will use.
+        Default is to calculate it internally based on image size
+
 steps:
+    - id: average_ms
+      label: Apply extra averaging of MS
+      in:
+        - id: msin
+          source: msin
+        - id: averaging_factor
+          source: averaging_factor
+      out:
+        - ms_avg
+      run: ../steps/dp3_avg_step.cwl
+      scatter: msin
+      when: $(inputs.averaging_factor > 1)
+
     - id: sort_mses
-      label: Trim facets
+      label: Sort MS based on name
       in:
         - id: input_entry
-          source: msin
+          source:
+            - average_ms/ms_avg
+            - msin
+          linkMerge: merge_nested
+          pickValue: all_non_null
+          valueFrom: |
+            ${
+              var avg = self[0];
+              if (avg === null || avg === undefined) { return self[1]; }
+              var kept = [];
+              for (var i = 0; i < avg.length; i++) {
+                if (avg[i] !== null && avg[i] !== undefined) { kept.push(avg[i]); }
+              }
+              return kept.length === 0 ? self[1] : kept;
+            }
       out:
         - id: sorted_entries
       run: ../steps/sort_by_name.cwl
 
     - id: sort_facet_regions
-      label: Trim facets
+      label: Sort facets based on name
       in:
         - id: input_entry
           source: facet_polygons
@@ -76,8 +120,12 @@ steps:
           source: pixel_scale
         - id: resolution
           source: resolution
+        - id: briggs
+          source: briggs
         - id: tmpdir
           source: tmpdir
+        - id: ncpu
+          source: ncpu
       out:
         - id: MFS_image_pb
         - id: MFS_image
@@ -128,7 +176,9 @@ outputs:
     outputSource: image_and_trim/MFS_model
 
   - id: MFS_psfs
-    type: File[]
+    type:
+      - File?
+      - File[]?
     outputSource: image_and_trim/MFS_psf
 
   - id: MFS_mosaic

workflows/subworkflows/image_and_trim.cwl

@@ -13,10 +13,11 @@ inputs:
       type: Directory
       doc: MeasurementSet that will be imaged.
 
-    - id: number_cores
+    - id: ncpu
       type: int?
-      default: 24
-      doc: The number of cores that WSClean will use.
+      doc: |
+        The number of cores that WSClean will use.
+        Default is to calculate it internally based on image size
 
     - id: pixel_scale
       type: float
@@ -27,6 +28,11 @@ inputs:
       default: 0.3asec
       doc: Angular resolution that will be passed to WSClean's taper argument. Its syntax follows that of WSClean.
 
+    - id: briggs
+      type: float?
+      default: -1.4
+      doc: Briggs weighting for WSClean.
+
     - id: facet_polygon
       type: File
       doc: DS9 region file that will be used to trim the facet.
@@ -60,7 +66,7 @@ steps:
       label: make_facet_image
       in:
         - id: ncpu
-          source: number_cores
+          source: ncpu
         - id: msin
           source: msin
         - id: name
@@ -76,6 +82,8 @@ steps:
           valueFrom: $(self.toString() + "asec")
         - id: beam-shape
           source: restoring_beam
+        - id: briggs
+          source: briggs
         - id: tmpdir
           source: tmpdir
         - id: apply-facet-beam
@@ -94,7 +102,7 @@ steps:
       run: ../../steps/wsclean.cwl
 
     - id: trim_image_pb
-      label: Trim facets
+      label: Trim facet image-pb
       in:
         - id: image
           source:
@@ -108,7 +116,7 @@ steps:
       run: ../../steps/trim_facet.cwl
 
     - id: trim_image
-      label: Trim facets
+      label: Trim facet image
       in:
         - id: image
           source:
@@ -122,7 +130,7 @@ steps:
       run: ../../steps/trim_facet.cwl
 
     - id: trim_model_pb
-      label: Trim facets
+      label: Trim model image-pb
       in:
         - id: image
           source:
@@ -136,7 +144,7 @@ steps:
       run: ../../steps/trim_facet.cwl
 
     - id: trim_model
-      label: Trim facets
+      label: Trim model image
       in:
         - id: image
           source:
@@ -150,7 +158,7 @@ steps:
       run: ../../steps/trim_facet.cwl
 
     - id: trim_residual_pb
-      label: Trim facets
+      label: Trim residual image-pb
       in:
         - id: image
           source:
@@ -164,7 +172,7 @@ steps:
       run: ../../steps/trim_facet.cwl
 
     - id: trim_residual
-      label: Trim facets
+      label: Trim residual image
       in:
         - id: image
           source:
@@ -201,5 +209,7 @@ outputs:
       outputSource: trim_residual/trimmed_image
 
     - id: MFS_psf
-      type: File
+      type:
+        - File?
+        - File[]?
       outputSource: make_facet_image/MFS_psf