Refactor Conda and Spack support for Wave to Java

Signed-off-by: Paolo Di Tommaso <[email protected]>

Author: pditommaso

plugins/nf-wave/build.gradle

@@ -33,6 +33,7 @@ dependencies {
     compileOnly 'org.slf4j:slf4j-api:2.0.7'
     compileOnly 'org.pf4j:pf4j:3.4.1'
     api 'org.apache.commons:commons-compress:1.21'
+    api 'org.apache.commons:commons-lang3:3.12.0'
     api 'com.google.code.gson:gson:2.10.1'
 
     testImplementation(testFixtures(project(":nextflow")))

plugins/nf-wave/src/main/io/seqera/wave/plugin/config/CondaOpts.groovy renamed to plugins/nf-wave/src/main/io/seqera/wave/config/CondaOpts.java

@@ -15,25 +15,27 @@
  *
  */
 
-package io.seqera.wave.plugin.config
+package io.seqera.wave.config;
+
+import java.util.List;
+import java.util.Map;
 
 /**
  * Conda build options
  *
  * @author Paolo Di Tommaso <[email protected]>
  */
-class CondaOpts {
-
-    final public static String DEFAULT_MAMBA_IMAGE = 'mambaorg/micromamba:1.4.2'
+public class CondaOpts {
+    final public static String DEFAULT_MAMBA_IMAGE = "mambaorg/micromamba:1.4.2";
 
-    final String mambaImage
-    final List<String> commands
-    final String basePackages
+    final public String mambaImage;
+    final public List<String> commands;
+    final public String basePackages;
 
-    CondaOpts(Map opts) {
-        this.mambaImage = opts.mambaImage ?: DEFAULT_MAMBA_IMAGE
-        this.commands = opts.commands as List<String>
-        this.basePackages = opts.basePackages
+    public CondaOpts(Map<String,?> opts) {
+        this.mambaImage = opts.containsKey("mambaImage") ? opts.get("mambaImage").toString(): DEFAULT_MAMBA_IMAGE;
+        this.commands = opts.containsKey("commands") ? (List<String>)opts.get("commands") : null;
+        this.basePackages = (String)opts.get("basePackages");
     }
 
 }

plugins/nf-wave/src/main/io/seqera/wave/config/SpackOpts.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package io.seqera.wave.config;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Spack build options
+ *
+ * @author Marco De La Pierre <[email protected]>
+ */
+public class SpackOpts {
+
+    final static public String DEFAULT_SPACK_BUILDER_IMAGE = "spack/ubuntu-jammy:v0.19.2";
+    final static public String DEFAULT_SPACK_RUNNER_IMAGE = "ubuntu:22.04";
+    final static public String DEFAULT_SPACK_OSPACKAGES = "";
+    final static public String DEFAULT_SPACK_FLAGS = "-O3";
+
+    public final Boolean checksum;
+    public final String builderImage;
+    public final String runnerImage;
+    public final String osPackages;
+    public final String cFlags;
+    public final String cxxFlags;
+    public final String fFlags;
+    public final List<String> commands;
+
+    public SpackOpts() {
+        this(Map.of());
+    }
+    public SpackOpts(Map<String,?> opts) {
+        this.checksum = opts.get("checksum") == null || Boolean.parseBoolean(opts.get("checksum").toString());
+        this.builderImage = opts.containsKey("builderImage") ? opts.get("builderImage").toString() : DEFAULT_SPACK_BUILDER_IMAGE;
+        this.runnerImage = opts.containsKey("runnerImage") ? opts.get("runnerImage").toString() : DEFAULT_SPACK_RUNNER_IMAGE;
+        this.osPackages = opts.containsKey("osPackages") ? opts.get("osPackages").toString() : DEFAULT_SPACK_OSPACKAGES;
+        this.cFlags = opts.containsKey("cFlags") ? opts.get("cFlags").toString() : DEFAULT_SPACK_FLAGS;
+        this.cxxFlags = opts.containsKey("cxxFlags") ? opts.get("cxxFlags").toString() : DEFAULT_SPACK_FLAGS;
+        this.fFlags = opts.containsKey("fFlags") ? opts.get("fFlags").toString() : DEFAULT_SPACK_FLAGS;
+        this.commands = opts.containsKey("commands") ? (List<String>)opts.get("commands") : null;
+    }
+
+}

plugins/nf-wave/src/main/io/seqera/wave/plugin/DescribeContainerResponse.groovy

@@ -1,3 +1,20 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
 package io.seqera.wave.plugin
 
 import java.time.Instant

plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy

@@ -17,6 +17,7 @@
 
 package io.seqera.wave.plugin
 
+import static io.seqera.wave.util.DockerHelper.*
 
 import java.net.http.HttpClient
 import java.net.http.HttpRequest
@@ -54,12 +55,10 @@ import io.seqera.wave.plugin.packer.Packer
 import nextflow.Session
 import nextflow.SysEnv
 import nextflow.container.resolver.ContainerInfo
-import nextflow.executor.BashTemplateEngine
 import nextflow.fusion.FusionConfig
 import nextflow.processor.Architecture
 import nextflow.processor.TaskRun
 import nextflow.script.bundle.ResourcesBundle
-import nextflow.util.MustacheTemplateEngine
 import nextflow.util.SysHelper
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
@@ -380,10 +379,11 @@ class WaveClient {
             // map the recipe to a dockerfile
             if( isCondaLocalFile(attrs.conda) ) {
                 condaFile = Path.of(attrs.conda)
-                dockerScript = condaFileToDockerFile()
+                dockerScript = condaFileToDockerFile(config.condaOpts())
             }
+            // 'conda' attributes is resolved as the conda packages to be used
             else {
-                dockerScript = condaRecipeToDockerFile(attrs.conda)
+                dockerScript = condaPackagesToDockerFile(attrs.conda, condaChannels, config.condaOpts())
             }
         }
 
@@ -399,10 +399,10 @@ class WaveClient {
             // map the recipe to a dockerfile
             if( isSpackFile(attrs.spack) ) {
                 spackFile = Path.of(attrs.spack)
-                dockerScript = spackFileToDockerFile(spackArch)
+                dockerScript = spackFileToDockerFile(spackArch, config.spackOpts())
             }
             else {
-                dockerScript = spackRecipeToDockerFile(attrs.spack, spackArch)
+                dockerScript = spackPackagesToDockerFile(attrs.spack, spackArch, config.spackOpts())
             }
         }
 
@@ -464,105 +464,7 @@ class WaveClient {
         }
     }
 
-    protected String condaFileToDockerFile() {
-        final template = """\
-        FROM {{base_image}}
-        COPY --chown=\$MAMBA_USER:\$MAMBA_USER conda.yml /tmp/conda.yml
-        RUN micromamba install -y -n base -f /tmp/conda.yml && \\
-            {{base_packages}}
-            micromamba clean -a -y
-        """.stripIndent(true)
-        final image = config.condaOpts().mambaImage
-
-        final basePackage =  config.condaOpts().basePackages ? "micromamba install -y -n base ${config.condaOpts().basePackages} && \\".toString() : null
-        final binding = ['base_image': image, 'base_packages': basePackage]
-        final result = new MustacheTemplateEngine().render(template, binding)
-
-        return addCommands(result)
-    }
-
-    // Dockerfile template adpated from the Spack package manager
-    // https://github.com/spack/spack/blob/develop/share/spack/templates/container/Dockerfile
-    // LICENSE APACHE 2.0
-    protected String spackFileToDockerFile(String spackArch) {
-
-        String cmd_template = ''
-        final binding = [
-            'builder_image': config.spackOpts().builderImage,
-            'c_flags': config.spackOpts().cFlags,
-            'cxx_flags': config.spackOpts().cxxFlags,
-            'f_flags': config.spackOpts().fFlags,
-            'spack_arch': spackArch,
-            'checksum_string': config.spackOpts().checksum ? '' : '-n ',
-            'runner_image': config.spackOpts().runnerImage,
-            'os_packages': config.spackOpts().osPackages,
-            'add_commands': addCommands(cmd_template),
-        ]
-        final template = WaveClient.class.getResource('/templates/spack/dockerfile-spack-file.txt')
-        try(final reader = template.newReader()) {
-            final result = new BashTemplateEngine().render(reader, binding)
-            return result
-        }
-    }
 
-    protected String addCommands(String result) {
-        if( config.condaOpts().commands )
-            for( String cmd : config.condaOpts().commands ) {
-                result += cmd + "\n"
-            }
-        if( config.spackOpts().commands )
-            for( String cmd : config.spackOpts().commands ) {
-                result += cmd + "\n"
-        }
-        return result
-    }
-
-    protected String condaRecipeToDockerFile(String recipe) {
-        final template = """\
-        FROM {{base_image}}
-        RUN \\
-            micromamba install -y -n base {{channel_opts}} \\
-            {{target}} \\
-            {{base_packages}}
-            && micromamba clean -a -y
-        """.stripIndent(true)
-
-        final channelsOpts = condaChannels.collect(it -> "-c $it").join(' ')
-        final image = config.condaOpts().mambaImage
-        final target = recipe.startsWith('http://') || recipe.startsWith('https://')
-                ? "-f $recipe".toString()
-                : recipe
-        final basePackage =  config.condaOpts().basePackages ? "&& micromamba install -y -n base ${config.condaOpts().basePackages} \\".toString() : null
-        final binding = [base_image: image, channel_opts: channelsOpts, target:target, base_packages: basePackage]
-        final result = new MustacheTemplateEngine().render(template, binding)
-        return addCommands(result)
-    }
-
-    // Dockerfile template adpated from the Spack package manager
-    // https://github.com/spack/spack/blob/develop/share/spack/templates/container/Dockerfile
-    // LICENSE APACHE 2.0
-    protected String spackRecipeToDockerFile(String recipe, String spackArch) {
-
-        String cmd_template = ''
-        final binding = [
-            'recipe': recipe,
-            'builder_image': config.spackOpts().builderImage,
-            'c_flags': config.spackOpts().cFlags,
-            'cxx_flags': config.spackOpts().cxxFlags,
-            'f_flags': config.spackOpts().fFlags,
-            'spack_arch': spackArch,
-            'checksum_string': config.spackOpts().checksum ? '' : '-n ',
-            'runner_image': config.spackOpts().runnerImage,
-            'os_packages': config.spackOpts().osPackages,
-            'add_commands': addCommands(cmd_template),
-        ]
-        final template = WaveClient.class.getResource('/templates/spack/dockerfile-spack-recipe.txt')
-
-        try(final reader = template.newReader()) {
-            final result = new BashTemplateEngine().render(reader, binding)
-            return result
-        }
-    }
 
     static protected boolean isCondaLocalFile(String value) {
         if( value.contains('\n') )

plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveObserver.groovy

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020-2022, Seqera Labs
+ * Copyright 2013-2023, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

plugins/nf-wave/src/main/io/seqera/wave/plugin/adapter/InstantAdapter.groovy

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020-2022, Seqera Labs
+ * Copyright 2013-2023, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

plugins/nf-wave/src/main/io/seqera/wave/plugin/config/ReportOpts.groovy

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020-2022, Seqera Labs
+ * Copyright 2013-2023, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

plugins/nf-wave/src/main/io/seqera/wave/plugin/config/RetryOpts.groovy

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020-2022, Seqera Labs
+ * Copyright 2013-2023, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

plugins/nf-wave/src/main/io/seqera/wave/plugin/config/SpackOpts.groovy

@@ -19,6 +19,8 @@ package io.seqera.wave.plugin.config
 
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
+import io.seqera.wave.config.CondaOpts
+import io.seqera.wave.config.SpackOpts
 import nextflow.util.Duration
 /**
  * Model Wave client configuration