Allow all fields in the values section and make output optional

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.5.0
+
+1. Made the `output` section of the worksheet optional. This is especially useful for pipelines that perform quality checking.
+2. It's now possible to use all data fields in the `values` section of the worksheet. The name of the `input` object has also been changed to `data`
+
 ## 0.4.1
 
 1. Removed tag based include filters since that doesn't make sense anymore after splitting samplesheets per tag

README.md

@@ -6,7 +6,7 @@ To use the plugin add the following to your `nextflow.config`:
 
 ```groovy
 plugins {
-   id 'nf-cmgg@0.4.1'
+   id 'nf-cmgg@0.5.0'
 }
 ```
 

build.gradle

@@ -6,7 +6,7 @@ dependencies {
     implementation 'com.squareup.okhttp3:okhttp:5.3.2'
 }
 
-version = '0.4.1'
+version = '0.5.0'
 
 nextflowPlugin {
     nextflowVersion = '25.10.0'

src/main/groovy/nfcmgg/plugin/worksheet/Worksheet.groovy

@@ -19,6 +19,8 @@ class Worksheet {
     final WorksheetMetrics metrics
     final WorksheetSamplesheets samplesheets
 
+    final Set<String> dataFields = [] as Set<String>
+
     Worksheet(Path worksheet) {
         final WorksheetErrors errors = new WorksheetErrors()
         final Map worksheetMap = loadWorksheetMap(worksheet)
@@ -28,88 +30,68 @@ class Worksheet {
         }
         name = worksheetMap.name as String
 
-        WorksheetInput parsedInput = null
         if (worksheetMap.input == null) {
             errors.error("Worksheet ${worksheet.name} is missing the required 'input' field")
         } else {
             try {
-                parsedInput = new WorksheetInput(worksheetMap.input as Map)
+                input = new WorksheetInput(worksheetMap.input as Map)
+                dataFields.addAll(input.fields.keySet())
             } catch (WorksheetException e) {
                 errors.record(e.message)
             }
         }
-        input = parsedInput
 
         if (worksheetMap.id_field == null) {
             errors.error("Worksheet ${worksheet.name} is missing the required 'id_field' field")
-        } else if (parsedInput != null && !parsedInput.hasSource(worksheetMap.id_field.toString())) {
+        } else if (input != null && !input.hasSource(worksheetMap.id_field.toString())) {
             errors.error(
                 "Worksheet ${worksheet.name} 'id_field' '${worksheetMap.id_field}' is not defined" +
                 ' as a source or key (when source is missing) in the input block'
             )
         }
         idField = worksheetMap.id_field as String
 
-        final Set<String> inputFields = parsedInput != null ? parsedInput.fields.keySet() : ([] as Set<String>)
-        WorksheetValues parsedValues = null
-        if (worksheetMap.values != null) {
+        if (worksheetMap.output != null) {
             try {
-                parsedValues = new WorksheetValues(worksheetMap.values as Map, inputFields)
+                output = new WorksheetOutput(worksheetMap.output as Map)
+                dataFields.addAll(output.fields.keySet())
             } catch (WorksheetException e) {
                 errors.record(e.message)
             }
         }
-        values = parsedValues
 
-        WorksheetOutput parsedOutput = null
-        if (worksheetMap.output == null) {
-            errors.error("Worksheet ${worksheet.name} is missing the required 'output' field")
-        } else {
+        if (worksheetMap.metrics != null) {
             try {
-                parsedOutput = new WorksheetOutput(worksheetMap.output as Map)
+                metrics = new WorksheetMetrics(worksheetMap.metrics as Map)
+                dataFields.addAll(metrics.fields.keySet())
             } catch (WorksheetException e) {
                 errors.record(e.message)
             }
         }
-        output = parsedOutput
 
-        WorksheetMetrics parsedMetrics = null
-        if (worksheetMap.metrics != null) {
+        if (worksheetMap.values != null) {
             try {
-                parsedMetrics = new WorksheetMetrics(worksheetMap.metrics as Map)
+                values = new WorksheetValues(worksheetMap.values as Map, dataFields)
+                dataFields.addAll(values.fields.keySet())
             } catch (WorksheetException e) {
                 errors.record(e.message)
             }
         }
-        metrics = parsedMetrics
 
-        WorksheetSamplesheets parsedSamplesheets = null
         if (worksheetMap.samplesheets == null) {
             errors.error("Worksheet ${worksheet.name} is missing the required 'samplesheets' field")
         } else {
-            final Set<String> dataFields = [] as Set
-            dataFields.addAll(inputFields)
-            if (parsedValues != null) {
-                dataFields.addAll(parsedValues.fields.keySet())
-            }
-            if (parsedOutput != null) {
-                dataFields.addAll(parsedOutput.fields.keySet())
-            }
-            if (parsedMetrics != null) {
-                dataFields.addAll(parsedMetrics.fields.keySet())
-            }
             try {
                 final WorksheetSamplesheetsSettings samplesheetSettings = new WorksheetSamplesheetsSettings(
                     worksheetMap.samplesheets_settings as Map, dataFields
                 )
-                parsedSamplesheets = new WorksheetSamplesheets(
+                samplesheets = new WorksheetSamplesheets(
                     worksheetMap.samplesheets as List<Map>, dataFields, samplesheetSettings
                 )
             } catch (WorksheetException e) {
                 errors.record(e.message)
             }
         }
-        samplesheets = parsedSamplesheets
 
         errors.throwIfAny("Worksheet ${worksheet.name} is invalid")
     }

src/main/groovy/nfcmgg/plugin/worksheet/WorksheetValues.groovy

@@ -22,7 +22,7 @@ class WorksheetValues {
      */
     final Map<String, Field> fields
 
-    WorksheetValues(Map values, Set<String> inputFields) {
+    WorksheetValues(Map values, Set<String> dataFields) {
         final WorksheetErrors errors = new WorksheetErrors()
         if (values == null || values.isEmpty()) {
             errors.error('Worksheet values is missing or empty')
@@ -32,15 +32,15 @@ class WorksheetValues {
         values.each { key, value ->
             final String fieldName = key as String
             final Map fieldMap = (value != null ? value as Map : [:]) as Map
-            parsed[fieldName] = new Field(fieldName, fieldMap, inputFields, errors)
+            parsed[fieldName] = new Field(fieldName, fieldMap, dataFields, errors)
         }
         fields = parsed.asImmutable()
         errors.throwIfAny('Invalid worksheet values')
     }
 
-    Map<String, Object> convert(Map<String, Object> inputData, Map<String, Object> params) {
-        Map<String, Object> convertedData = inputData
-        Binding inputBinding = new Binding([input: inputData, params: params])
+    Map<String, Object> convert(Map<String, Object> dataFields, Map<String, Object> params) {
+        Map<String, Object> convertedData = dataFields
+        Binding inputBinding = new Binding([data: dataFields, params: params])
         GroovyShell inputShell = SafeGroovy.shell(inputBinding)
         fields.each { key, field ->
             convertedData[key] = field.define(inputShell)
@@ -58,7 +58,7 @@ class WorksheetValues {
         final Object value
         final String func
 
-        Field(String key, Map fieldMap, Set<String> inputFields, WorksheetErrors errors) {
+        Field(String key, Map fieldMap, Set<String> dataFields, WorksheetErrors errors) {
             this.key = key
             value = fieldMap.containsKey('value') ? fieldMap.value : null
             func = fieldMap.func != null ? fieldMap.func as String : null
@@ -69,7 +69,7 @@ class WorksheetValues {
                 errors.error("Values field '${key}' cannot define both 'value' and 'func'")
             }
             if (func != null) {
-                validateFunc(key, func, inputFields, errors)
+                validateFunc(key, func, dataFields, errors)
             }
         }
 
@@ -82,7 +82,7 @@ class WorksheetValues {
             }
         }
 
-        private static void validateFunc(String key, String func, Set<String> inputFields, WorksheetErrors errors) {
+        private static void validateFunc(String key, String func, Set<String> dataFields, WorksheetErrors errors) {
             try {
                 SafeGroovy.parse(func)
             } catch (CompilationFailedException e) {
@@ -91,10 +91,10 @@ class WorksheetValues {
             final Matcher matcher = INPUT_FIELD_PATTERN.matcher(func)
             while (matcher.find()) {
                 final String referenced = matcher.group(1)
-                if (!inputFields.contains(referenced)) {
+                if (!dataFields.contains(referenced)) {
                     errors.error(
-                        "Values field '${key}' references unknown input field 'input.${referenced}'. " +
-                        "Available input fields: ${inputFields.sort().join(', ')}"
+                        "Values field '${key}' references unknown data field 'data.${referenced}'. " +
+                        "Available data fields: ${dataFields.sort().join(', ')}"
                     )
                 }
             }

src/main/resources/worksheets/nfcmgg_preprocessing.yml

@@ -39,15 +39,15 @@ values:
   strandedness:
     value: unknown
   normdup:
-    func: input.tag.toLowerCase() == "copgt-m"
+    func: data.tag.toLowerCase() == "copgt-m"
   nipt:
-    func: input.tag.toLowerCase() == "cfdnaseq"
+    func: data.tag.toLowerCase() == "cfdnaseq"
   binsize:
     func: |
-      (input.organism?.toLowerCase() == 'mus musculus' || input.genome?.toLowerCase() == 'mm10')
-      && ![100, 500].contains(input.binsize) ? 100 : input.binsize
+      (data.organism?.toLowerCase() == 'mus musculus' || data.genome?.toLowerCase() == 'mm10')
+      && ![100, 500].contains(data.binsize) ? 100 : data.binsize
   exomecnv_batch:
-    func: 'input.library + "_" + input.sex'
+    func: 'data.library + "_" + data.sex'
 # Define which output files that should be remembered for samplesheet generation.
 # Key-value pairs should be used here where the key is the name used by the plugin
 # All fields can take the following key-value pairs:

src/test/groovy/nfcmgg/plugin/worksheet/WorksheetTest.groovy

@@ -77,7 +77,7 @@ samplesheets:
 
         then:
         WorksheetException e = thrown()
-        e.message.contains("unknown input field 'input.missing'")
+        e.message.contains("unknown data field 'data.missing'")
     }
 
     void 'missing output pattern aborts the plugin'() {

worksheet-schema.json

@@ -2,7 +2,7 @@
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "description": "Schema for the worksheet configuration YAML file",
   "type": "object",
-  "required": ["name", "id_field", "input", "output", "samplesheets"],
+  "required": ["name", "id_field", "input", "samplesheets"],
   "properties": {
     "name": {
       "type": "string",