Include optional packages in pyproject.toml and conda environments

pyproject.toml

@@ -85,8 +85,14 @@ urls = {Homepage = "https://www.sagemath.org"}
 requires-python = ">=3.11, <3.14"
 
 [project.optional-dependencies]
-R = [
-    'rpy2 >=3.3',
+R = ['rpy2 >=3.3']
+extra = [
+  'p_group_cohomology >=3.3', # Only used in tests
+  'pycosat >=0.6.3',
+  'pynormaliz >=2.18; platform_machine != "aarch64" and platform_machine != "arm64"', # Not yet available for Linux aarch64
+  'igraph',
+  'sage_numerical_backends_coin',
+  'symengine >= 0.6.1', # Only used in tests
 ]
 giac = [
     'sagemath_giac',
@@ -118,7 +124,7 @@ consider_namespace_packages = true
 build-requires = [
   "virtual:compiler/c",
   "virtual:compiler/cpp",
-  "pkg:generic/pkg-config"
+  "pkg:generic/pkg-config",
 ]
 
 host-requires = [
@@ -176,6 +182,31 @@ dependencies = [
   "pkg:generic/tachyon",
 ]
 
+[external.optional-host-requires]
+extra = [
+  "pkg:generic/bliss",
+  "pkg:generic/coxeter3",
+  "pkg:generic/mcqd",
+  "pkg:generic/meataxe",
+  "pkg:generic/sirocco",
+  "pkg:generic/tdlib",
+]
+
+[external.optional-dependencies]
+extra = [
+  "pkg:generic/4ti2",
+  "pkg:generic/benzene",
+  "pkg:generic/buckygen",
+  "pkg:generic/csdp",
+  "pkg:generic/frobby",
+  "pkg:generic/kenzo",
+  "pkg:generic/latte-integrale",
+  "pkg:generic/lrslib",
+  "pkg:generic/plantri",
+  "pkg:generic/qepcad",
+  "pkg:generic/tides",
+]
+
 [dependency-groups]
 test = [
   "pytest",

tools/update-conda.py

@@ -59,10 +59,21 @@ def filter_requirements(dependencies: set[str], python: str, platform: str) -> s
         "osx-arm64": "darwin",
         "win-64": "win32",
     }[platform]
+    platform_machine = {
+        "linux-64": "x86_64",
+        "linux-aarch64": "aarch64",
+        "osx-64": "x86_64",
+        "osx-arm64": "arm64",
+        "win-64": "x86_64",
+    }[platform]
+    env = {
+        "python_version": python,
+        "sys_platform": sys_platform,
+        "platform_machine": platform_machine,
+    }
 
     def filter_dep(dep: str):
         req = Requirement(dep)
-        env = {"python_version": python, "sys_platform": sys_platform}
         if not req.marker or req.marker.evaluate(env):
             # Serialize the requirement without the marker
             req.marker = None
@@ -83,18 +94,19 @@ def update_conda(source_dir: Path, systems: list[str] | None) -> None:
             continue
 
         for python in pythons:
+            if python == "3.13" and platform_key != "win-64":
+                print(
+                    f"Skipping Python {python} for platform {platform_key} as it is not supported yet."
+                )
+                continue
+
             dependencies = get_dependencies(pyproject_toml, python, platform_key)
             for tag in tags:
                 # Pin Python version
                 pinned_dependencies = {
                     f"python={python}" if dep == "python" else dep
                     for dep in dependencies
                 }
-
-                dev_dependencies = get_dev_dependencies(pyproject_toml)
-                print(f"Adding dev dependencies: {dev_dependencies}")
-                pinned_dependencies = pinned_dependencies.union(dev_dependencies)
-
                 pinned_dependencies = sorted(pinned_dependencies)
 
                 env_file = source_dir / f"environment{tag}-{python}.yml"
@@ -135,29 +147,124 @@ def update_conda(source_dir: Path, systems: list[str] | None) -> None:
 
 def get_dependencies(pyproject_toml: Path, python: str, platform: str) -> set[str]:
     grayskull_config = Configuration("sagemath")
+    pyproject = tomllib.load(pyproject_toml)
     pyproject_metadata = merge_setup_toml_metadata(
         {}, get_all_toml_info(pyproject_toml)
     )
     requirements = extract_requirements(pyproject_metadata, grayskull_config, {})
-    all_requirements: set[str] = (
-        set(requirements.get("build", {}))
-        | set(requirements.get("host", {}))
-        | set(requirements.get("run", {}))
+    all_requirements: set[str] = set(
+        requirements.get("build", [])
+        + requirements.get("host", [])
+        + requirements.get("run", [])
+        + pyproject_metadata.get("install_requires", [])
+        + get_dev_dependencies(pyproject)
+        + get_optional_dependencies(pyproject)
     )
 
-    # Specify concrete package for some virtual packages
+    # Fix requirements that are not available on conda
+    all_requirements = {
+        # Following can be removed once https://github.com/regro/cf-scripts/pull/2176 is used in grayskull
+        req.replace("lrcalc", "python-lrcalc")
+        .replace("symengine", "python-symengine")
+        .replace("memory_allocator", "memory-allocator")
+        for req in all_requirements
+    }
+    # Exclude requirements not available on conda (for a given platform)
+    exclude_packages: set[str] = {
+        "p_group_cohomology",
+        "sage_numerical_backends_coin",
+        "sagemath_giac",
+        "pynormaliz",  # due to https://github.com/sagemath/sage/issues/40214
+        "rpy2",  # due to https://github.com/sagemath/sage/issues/40215
+        "latte-integrale",  # due to https://github.com/sagemath/sage/issues/40216
+    }
+    if platform in ("linux-aarch64", "osx-arm64"):
+        exclude_packages |= {
+            "4ti2",
+            "latte-integrale",
+            "lrslib",
+        }
+    elif platform == "win-64":
+        exclude_packages |= {
+            "4ti2",
+            "bc",
+            "brial",
+            "bliss",
+            "cddlib",
+            "cliquer",
+            "ecl",
+            "eclib",
+            "ecm",
+            "fflas-ffpack",
+            "fplll",
+            "gap-defaults",
+            "gengetopt",
+            "gfan",
+            "giac",
+            "givaro",
+            "iml",
+            "latte-integrale",
+            "lcalc",
+            "libatomic_ops",
+            "libbraiding",
+            "libhomfly",
+            "linbox",
+            "lrcalc",
+            "lrslib",
+            "m4",
+            "m4rie",
+            "maxima",
+            "mpfi",
+            "ncurses",
+            "ntl",
+            "palp",
+            "patch",
+            "ppl",
+            "primecount",
+            "pynormaliz",
+            "python-lrcalc",
+            "readline",
+            "rpy2",
+            "rw",
+            "singular",
+            "sirocco",
+            "sympow",
+            "tachyon",
+            "tar",
+            "texinfo",
+        }
+    print(all_requirements)
+    all_requirements = {
+        req
+        for req in all_requirements
+        if not any(
+            req == package or req.startswith(package + " ")
+            for package in exclude_packages
+        )
+    }
+
+    # Remove virtual packages to not confuse 'filter_requirements'
     all_requirements.remove("{{ blas }}")
-    all_requirements.add("blas=2.*=openblas")
     all_requirements.remove("{{ compiler('c') }}")
     all_requirements.remove("{{ compiler('cxx') }}")
+    all_requirements.discard("<{ pin_compatible('numpy') }}")
     # For some reason, grayskull mishandles the fortran compiler sometimes
     # so handle both cases
     for item in ["{{ compiler('fortran') }}", "{{ compiler'fortran' }}"]:
         try:
             all_requirements.remove(item)
-        except ValueError:
+        except (ValueError, KeyError):
             pass
-    all_requirements.append("fortran-compiler")
+    for with_comment in {req for req in all_requirements if "#" in req}:
+        all_requirements.discard(with_comment)
+
+    all_requirements = filter_requirements(all_requirements, python, platform)
+    all_requirements = set(
+        normalize_requirements_list(list(all_requirements), grayskull_config)
+    )
+    # Specify concrete package for some virtual packages
+    all_requirements.add("blas=2.*=openblas")
+    all_requirements.add("fortran-compiler")
     if platform == "win-64":
         all_requirements.add("vs2022_win-64")
         # For mingw:
@@ -167,82 +274,12 @@ def get_dependencies(pyproject_toml: Path, python: str, platform: str) -> set[st
         all_requirements.add("c-compiler")
         all_requirements.add("cxx-compiler")
 
-    # Filter out packages that are not available on Windows
-    if platform == "win-64":
-        # Remove packages that are not available on Windows
-        all_requirements.difference_update((
-                "bc",
-                "brial",
-                "cddlib",
-                "cliquer",
-                "ecl",
-                "eclib",
-                "ecm",
-                "fflas-ffpack",
-                "fplll",
-                "gap-defaults",
-                "gengetopt",
-                "gfan",
-                "giac",
-                "givaro",
-                "gmp",
-                "gmpy2",
-                "iml",
-                "lcalc",
-                "libatomic_ops",
-                "libbraiding",
-                "libhomfly",
-                "linbox",
-                "lrcalc",
-                "m4",
-                "m4rie",
-                "maxima",
-                "mpfi",
-                "ncurses",
-                "ntl",
-                "palp",
-                "patch",
-                "ppl",
-                "primecount",
-                "readline",
-                "rw",
-                "singular",
-                "sympow",
-                "tachyon",
-                "tar",
-                "texinfo",
-        ))
-
-    # Correct pypi name for some packages
-    python_requirements = set(pyproject_metadata.get("install_requires", []))
-    # Specify concrete packages for some packages not yet in grayskull
-    python_requirements.remove("pkg:generic/tachyon")
-    if platform != "win-64":
-        python_requirements.add("tachyon")
-    python_requirements.remove("pkg:generic/sagemath-elliptic-curves")
-    python_requirements.add("sagemath-db-elliptic-curves")
-    python_requirements.remove("pkg:generic/sagemath-polytopes-db")
-    python_requirements.add("sagemath-db-polytopes")
-    python_requirements.discard("pkg:generic/sagemath-graphs")
-    python_requirements.add("sagemath-db-graphs")
-    python_requirements.remove("memory_allocator")
-    python_requirements.add("memory-allocator")
-    # Following can be removed once https://github.com/regro/cf-scripts/pull/2176 is used in grayskull
-    python_requirements = {
-        req.replace("lrcalc", "python-lrcalc") for req in python_requirements
-    }
-    python_requirements = filter_requirements(python_requirements, python, platform)
-    all_requirements.update(
-        normalize_requirements_list(list(python_requirements), grayskull_config)
-    )
-    all_requirements.remove("<{ pin_compatible('numpy') }}")
-    all_requirements.remove("memory_allocator")
+    # Add additional dependencies based on platform
     if platform == "win-64":
         # Flint needs pthread.h
         all_requirements.add("winpthreads-devel")
         # Workaround for https://github.com/conda-forge/libpng-feedstock/issues/47
         all_requirements.add("zlib")
-
     if platform != "win-64":
         # Needed to run configure/bootstrap, can be deleted once we fully migrated to meson
         all_requirements.add("autoconf")
@@ -253,8 +290,7 @@ def get_dependencies(pyproject_toml: Path, python: str, platform: str) -> set[st
     return all_requirements
 
 
-def get_dev_dependencies(pyproject_toml: Path) -> list[str]:
-    pyproject = tomllib.load(pyproject_toml)
+def get_dev_dependencies(pyproject: dict) -> list[str]:
     dependency_groups = pyproject.get("dependency-groups", {})
     dev_dependencies = (
         dependency_groups.get("test", [])
@@ -267,4 +303,13 @@ def get_dev_dependencies(pyproject_toml: Path) -> list[str]:
     return dev_dependencies
 
 
+def get_optional_dependencies(pyproject: dict) -> list[str]:
+    optional_dependencies = []
+    optional_groups = pyproject.get("project", {}).get("optional-dependencies", {})
+    for _, dependencies in optional_groups.items():
+        optional_dependencies.extend(dependencies)
+    # print(f"Optional dependencies: {optional_dependencies}")  # Uncommented for debugging
+    return optional_dependencies
+
+
 update_conda(options.sourcedir, options.systems)