add support for injecting checksums for cargo crates

easybuild/framework/easyblock.py

@@ -148,6 +148,14 @@ def extra_options(extra=None):
 
         return extra
 
+    @staticmethod
+    def src_parameter_names():
+        """
+        Return list of EasyConfig parameter that contribute to the sources in the `src` member
+        (or equivalently to the `sources` parameter of a parsed EasyConfig)
+        """
+        return ['sources']
+
     #
     # INIT
     #
@@ -5082,8 +5090,8 @@ def make_checksum_lines(checksums, indent_level):
         if app.src:
             placeholder = '# PLACEHOLDER FOR SOURCES/PATCHES WITH CHECKSUMS'
 
-            # grab raw lines for source_urls, sources, patches
-            keys = ['patches', 'source_urls', 'sources']
+            # grab raw lines for the following params
+            keys = ['source_urls'] + app.src_parameter_names() + ['patches']
             raw = {}
             for key in keys:
                 regex = re.compile(r'^(%s(?:.|\n)*?\])\s*$' % key, re.M)
@@ -5094,13 +5102,11 @@ def make_checksum_lines(checksums, indent_level):
 
             _log.debug("Raw lines for %s easyconfig parameters: %s", '/'.join(keys), raw)
 
-            # inject combination of source_urls/sources/patches/checksums into easyconfig
-            # by replacing first occurence of placeholder that was put in place
-            sources_raw = raw.get('sources', '')
-            source_urls_raw = raw.get('source_urls', '')
-            patches_raw = raw.get('patches', '')
+            # inject combination of the grabbed lines and the checksums into the easyconfig
+            # by replacing first the occurence of the placeholder that was put in place
+            raw_text = ''.join(raw.get(key, '') for key in keys)
             regex = re.compile(placeholder + '\n', re.M)
-            ectxt = regex.sub(source_urls_raw + sources_raw + patches_raw + checksums_txt + '\n', ectxt, count=1)
+            ectxt = regex.sub(raw_text + checksums_txt + '\n', ectxt, count=1)
 
             # get rid of potential remaining placeholders
             ectxt = regex.sub('', ectxt)

test/framework/docs.py

@@ -44,6 +44,7 @@
 LIST_EASYBLOCKS_SIMPLE_TXT = """EasyBlock
 |-- bar
 |-- Bundle
+|-- Cargo
 |-- CMakeMake
 |-- CmdCp
 |-- ConfigureMake
@@ -106,6 +107,7 @@
 LIST_EASYBLOCKS_DETAILED_TXT = """EasyBlock (easybuild.framework.easyblock)
 |-- bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py)
 |-- Bundle (easybuild.easyblocks.generic.bundle @ %(topdir)s/generic/bundle.py)
+|-- Cargo (easybuild.easyblocks.generic.cargo @ %(topdir)s/generic/cargo.py)
 |-- CMakeMake (easybuild.easyblocks.generic.cmakemake @ %(topdir)s/generic/cmakemake.py)
 |-- CmdCp (easybuild.easyblocks.generic.cmdcp @ %(topdir)s/generic/cmdcp.py)
 |-- ConfigureMake (easybuild.easyblocks.generic.configuremake @ %(topdir)s/generic/configuremake.py)
@@ -169,6 +171,7 @@
 
   * bar
   * Bundle
+  * Cargo
   * CMakeMake
   * CmdCp
   * ConfigureMake
@@ -259,6 +262,7 @@
 
   * bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py)
   * Bundle (easybuild.easyblocks.generic.bundle @ %(topdir)s/generic/bundle.py)
+  * Cargo (easybuild.easyblocks.generic.cargo @ %(topdir)s/generic/cargo.py)
   * CMakeMake (easybuild.easyblocks.generic.cmakemake @ %(topdir)s/generic/cmakemake.py)
   * CmdCp (easybuild.easyblocks.generic.cmdcp @ %(topdir)s/generic/cmdcp.py)
   * ConfigureMake (easybuild.easyblocks.generic.configuremake @ %(topdir)s/generic/configuremake.py)
@@ -348,6 +352,7 @@
 LIST_EASYBLOCKS_SIMPLE_MD = """- **EasyBlock**
   - bar
   - Bundle
+  - Cargo
   - CMakeMake
   - CmdCp
   - ConfigureMake
@@ -410,6 +415,7 @@
 LIST_EASYBLOCKS_DETAILED_MD = """- **EasyBlock** (easybuild.framework.easyblock)
   - bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py)
   - Bundle (easybuild.easyblocks.generic.bundle @ %(topdir)s/generic/bundle.py)
+  - Cargo (easybuild.easyblocks.generic.cargo @ %(topdir)s/generic/cargo.py)
   - CMakeMake (easybuild.easyblocks.generic.cmakemake @ %(topdir)s/generic/cmakemake.py)
   - CmdCp (easybuild.easyblocks.generic.cmdcp @ %(topdir)s/generic/cmdcp.py)
   - ConfigureMake (easybuild.easyblocks.generic.configuremake @ %(topdir)s/generic/configuremake.py)
@@ -719,6 +725,7 @@ def test_get_easyblock_classes(self):
         expected = [
             'Bundle',
             'CMakeMake',
+            'Cargo',
             'ChildCustomDummyExtension',
             'ChildDeprecatedDummyExtension',
             'CmdCp',
@@ -934,6 +941,7 @@ def test_list_software(self):
             'homepage: https://easybuilders.github.io/easybuild',
             '',
             "  * toy v0.0: gompi/2018a, system",
+            "  * toy v0.0 (versionsuffix: '-cargo'): system",
             "  * toy v0.0 (versionsuffix: '-deps'): system",
             "  * toy v0.0 (versionsuffix: '-iter'): system",
             "  * toy v0.0 (versionsuffix: '-multiple'): system",
@@ -956,6 +964,7 @@ def test_list_software(self):
             'version    versionsuffix    toolchain',
             '=======    =============    ===========================',
             '``0.0``                     ``gompi/2018a``, ``system``',
+            '``0.0``    ``-cargo``       ``system``',
             '``0.0``    ``-deps``        ``system``',
             '``0.0``    ``-iter``        ``system``',
             '``0.0``    ``-multiple``    ``system``',

test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-cargo.eb

@@ -0,0 +1,17 @@
+name = 'toy'
+version = '0.0'
+versionsuffix = '-cargo'
+
+easyblock = 'Cargo'
+
+homepage = 'https://easybuilders.github.io/easybuild'
+description = "Toy C program, 100% toy."
+
+toolchain = SYSTEM
+
+crates = [
+    ('toy', 'extra.txt'),
+    ('toy', '0.0_gzip.patch.gz'),
+]
+
+moduleclass = 'tools'

test/framework/filetools.py

@@ -2609,11 +2609,13 @@ def test_index_functions(self):
         # load_index just returns None if there is no index in specified directory
         self.assertEqual(ft.load_index(self.test_prefix), None)
 
+        num_files = sum(len(files) for _, _, files in os.walk(test_ecs))
+
         # create index for test easyconfigs;
         # test with specified path with and without trailing '/'s
         for path in [test_ecs, test_ecs + '/', test_ecs + '//']:
             index = ft.create_index(path)
-            self.assertEqual(len(index), 94)
+            self.assertEqual(len(index), num_files)
 
             expected = [
                 os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'),

test/framework/options.py

@@ -1027,6 +1027,7 @@ def test_000_list_easyblocks(self):
             expected = '\n'.join([
                 "EasyBlock",
                 "|-- bar",
+                "|-- Cargo",
                 "|-- ConfigureMake",
                 "|   |-- MakeCp",
                 "|-- EB_EasyBuildMeta",
@@ -6513,6 +6514,55 @@ def test_inject_checksums(self):
         ]
         self.assertEqual(ext_opts['checksums'], expected_checksums)
 
+        # Also works for cargo crates
+        cargo_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-cargo.eb')
+        copy_file(cargo_ec, test_ec)
+        stdout, stderr = self._run_mock_eb([test_ec, '--inject-checksums'], raise_error=True, strip=True)
+        self.assertIn("injecting sha256 checksums in", stdout)
+        self.assertEqual(stderr, '')
+        expected_checksums = [
+            {'toy-extra.txt': '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458'},
+            {'toy-0.0_gzip.patch.gz': 'c5c51dd4b00fd490f8f8226f5fa609c30b66bda7ef6d3391ab2631508f3d5e41'},
+        ]
+        patterns = [r"^== injecting sha256 checksums for sources & patches in test\.eb\.\.\.$"]
+        patterns.extend(r"^== \* %s: %s$" % next(iter(entry.items())) for entry in expected_checksums)
+        for pattern in patterns:
+            regex = re.compile(pattern, re.M)
+            self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
+
+        ec = EasyConfigParser(test_ec).get_config_dict()
+        self.assertEqual(ec['checksums'], expected_checksums)
+
+        # crates also work with sources and patches (unusual use case)
+        copy_file(cargo_ec, test_ec)
+        write_file(test_ec, textwrap.dedent("""
+            sources = [SOURCE_TAR_GZ]
+            patches = [
+                'toy-0.0_fix-silly-typo-in-printf-statement.patch',
+            ]
+        """), append=True)
+        stdout, stderr = self._run_mock_eb([test_ec, '--inject-checksums'], raise_error=True, strip=True)
+        self.assertIn("injecting sha256 checksums in", stdout)
+        self.assertEqual(stderr, '')
+        expected_checksums = [
+            # Main source
+            {'toy-0.0.tar.gz': '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc'},
+            # Specified as "crates"
+            {'toy-extra.txt': '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458'},
+            {'toy-0.0_gzip.patch.gz': 'c5c51dd4b00fd490f8f8226f5fa609c30b66bda7ef6d3391ab2631508f3d5e41'},
+            # Patch
+            {'toy-0.0_fix-silly-typo-in-printf-statement.patch':
+             '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487'},
+        ]
+        patterns = [r"^== injecting sha256 checksums for sources & patches in test\.eb\.\.\.$"]
+        patterns.extend(r"^== \* %s: %s$" % next(iter(entry.items())) for entry in expected_checksums)
+        for pattern in patterns:
+            regex = re.compile(pattern, re.M)
+            self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
+
+        ec = EasyConfigParser(test_ec).get_config_dict()
+        self.assertEqual(ec['checksums'], expected_checksums)
+
         # passing easyconfig filename as argument to --inject-checksums results in error being reported,
         # because it's not a valid type of checksum
         args = ['--inject-checksums', test_ec]

test/framework/sandbox/easybuild/easyblocks/generic/cargo.py

@@ -0,0 +1,55 @@
+##
+# Copyright 2009-2024 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+from easybuild.framework.easyblock import EasyBlock
+from easybuild.framework.easyconfig import CUSTOM
+
+
+class Cargo(EasyBlock):
+    """Generic support for building/installing cargo crates."""
+
+    @staticmethod
+    def extra_options():
+        """Custom easyconfig parameters for bar."""
+        extra_vars = {
+            'crates': [[], "List of (crate, version, [repo, rev]) tuples to use", CUSTOM],
+        }
+        return EasyBlock.extra_options(extra_vars)
+
+    @staticmethod
+    def src_parameter_names():
+        return super().src_parameter_names() + ['crates']
+
+    def __init__(self, *args, **kwargs):
+        """Constructor for Cargo easyblock."""
+        super().__init__(*args, **kwargs)
+
+        # Populate sources from "crates" list of tuples
+        # For simplicity just assume (name,version.ext) tuples
+        sources = ['%s-%s' % crate_info for crate_info in self.cfg['crates']]
+
+        # copy EasyConfig instance before we make changes to it
+        self.cfg = self.cfg.copy()
+
+        self.cfg.update('sources', sources)