refactor: make python extension generate platform toolchains (#2875)

This makes the python bzlmod extension handle generating the
platform-specific toolchain
entries ("python_3_10_{platform}"). This is to eventually allow adding
additional
toolchains that aren't part of the PLATFORMS mapping in versions.bzl and
have
their own custom constraints.

The main things this refactor does are:
1. The bzlmod phase passes the full list of implementation toolchains
to create (previously, it relied on `hub_repo` to generate the
implementation
   names).
2. The name of a toolchain (the toolchain.name arg) and the repo that
implements
it (the py_toolchain_suite.user_repository_repo arg) are separate. This
allows
   future work to mixin toolchains that point to arbitrary repos.
3. The platform meta data uses a list of target settings instead of dict
of
flag values. This allows more arbitrary target settings. For now, flag
values
on the platform metadata is still looked for because it's known that
users
   patch python/versions.bzl.

Along the way:
* Factor out a platform_info helper in versions.bzl
* Factor out a NOT_ACTUALLY_PUBLIC constants to better denote things
that
  are public visibility, but actually internal.
* Add some docs to some internals so we don't have to chase down their
definitions.

Work towards #2081

Author: rickeylev

internal_dev_setup.bzl

@@ -34,11 +34,15 @@ def rules_python_internal_setup():
         name = "pythons_hub",
         minor_mapping = MINOR_MAPPING,
         default_python_version = "",
-        toolchain_prefixes = [],
-        toolchain_python_versions = [],
-        toolchain_set_python_version_constraints = [],
-        toolchain_user_repository_names = [],
         python_versions = sorted(TOOL_VERSIONS.keys()),
+        toolchain_names = [],
+        toolchain_repo_names = {},
+        toolchain_target_compatible_with_map = {},
+        toolchain_target_settings_map = {},
+        toolchain_platform_keys = {},
+        toolchain_python_versions = {},
+        toolchain_set_python_version_constraints = {},
+        base_toolchain_repo_names = [],
     )
 
     runtime_env_repo(name = "rules_python_runtime_env_tc_info")

python/private/config_settings.bzl

@@ -31,6 +31,10 @@ If the value is missing, then the default value is being used, see documentation
 {docs_url}/python/config_settings
 """
 
+# Indicates something needs public visibility so that other generated code can
+# access it, but it's not intended for general public usage.
+_NOT_ACTUALLY_PUBLIC = ["//visibility:public"]
+
 def construct_config_settings(*, name, default_version, versions, minor_mapping, documented_flags):  # buildifier: disable=function-docstring
     """Create a 'python_version' config flag and construct all config settings used in rules_python.
 
@@ -128,7 +132,30 @@ def construct_config_settings(*, name, default_version, versions, minor_mapping,
         # `whl_library` in the hub repo created by `pip.parse`.
         flag_values = {"current_config": "will-never-match"},
         # Only public so that PyPI hub repo can access it
-        visibility = ["//visibility:public"],
+        visibility = _NOT_ACTUALLY_PUBLIC,
+    )
+
+    libc = Label("//python/config_settings:py_linux_libc")
+    native.config_setting(
+        name = "_is_py_linux_libc_glibc",
+        flag_values = {libc: "glibc"},
+        visibility = _NOT_ACTUALLY_PUBLIC,
+    )
+    native.config_setting(
+        name = "_is_py_linux_libc_musl",
+        flag_values = {libc: "glibc"},
+        visibility = _NOT_ACTUALLY_PUBLIC,
+    )
+    freethreaded = Label("//python/config_settings:py_freethreaded")
+    native.config_setting(
+        name = "_is_py_freethreaded_yes",
+        flag_values = {freethreaded: "yes"},
+        visibility = _NOT_ACTUALLY_PUBLIC,
+    )
+    native.config_setting(
+        name = "_is_py_freethreaded_no",
+        flag_values = {freethreaded: "no"},
+        visibility = _NOT_ACTUALLY_PUBLIC,
     )
 
 def _python_version_flag_impl(ctx):

python/private/py_repositories.bzl

@@ -39,11 +39,15 @@ def py_repositories():
         name = "pythons_hub",
         minor_mapping = MINOR_MAPPING,
         default_python_version = "",
-        toolchain_prefixes = [],
-        toolchain_python_versions = [],
-        toolchain_set_python_version_constraints = [],
-        toolchain_user_repository_names = [],
         python_versions = sorted(TOOL_VERSIONS.keys()),
+        toolchain_names = [],
+        toolchain_repo_names = {},
+        toolchain_target_compatible_with_map = {},
+        toolchain_target_settings_map = {},
+        toolchain_platform_keys = {},
+        toolchain_python_versions = {},
+        toolchain_set_python_version_constraints = {},
+        base_toolchain_repo_names = [],
     )
     http_archive(
         name = "bazel_skylib",

python/private/py_toolchain_suite.bzl

@@ -34,15 +34,20 @@ def py_toolchain_suite(
         python_version,
         set_python_version_constraint,
         flag_values,
+        target_settings = [],
         target_compatible_with = []):
     """For internal use only.
 
     Args:
         prefix: Prefix for toolchain target names.
-        user_repository_name: The name of the user repository.
+        user_repository_name: The name of the repository with the toolchain
+            implementation (it's assumed to have particular target names within
+            it). Does not include the leading "@".
         python_version: The full (X.Y.Z) version of the interpreter.
         set_python_version_constraint: True or False as a string.
-        flag_values: Extra flag values to match for this toolchain.
+        flag_values: Extra flag values to match for this toolchain. These
+            are prepended to target_settings.
+        target_settings: Extra target_settings to match for this toolchain.
         target_compatible_with: list constraints the toolchains are compatible with.
     """
 
@@ -82,7 +87,7 @@ def py_toolchain_suite(
             match_any = match_any,
             visibility = ["//visibility:private"],
         )
-        target_settings = [name]
+        target_settings = [name] + target_settings
     else:
         fail(("Invalid set_python_version_constraint value: got {} {}, wanted " +
               "either the string 'True' or the string 'False'; " +

python/private/python.bzl

@@ -22,15 +22,9 @@ load(":python_register_toolchains.bzl", "python_register_toolchains")
 load(":pythons_hub.bzl", "hub_repo")
 load(":repo_utils.bzl", "repo_utils")
 load(":semver.bzl", "semver")
-load(":text_util.bzl", "render")
 load(":toolchains_repo.bzl", "multi_toolchain_aliases")
 load(":util.bzl", "IS_BAZEL_6_4_OR_HIGHER")
 
-# This limit can be increased essentially arbitrarily, but doing so will cause a rebuild of all
-# targets using any of these toolchains due to the changed repository name.
-_MAX_NUM_TOOLCHAINS = 9999
-_TOOLCHAIN_INDEX_PAD_LENGTH = len(str(_MAX_NUM_TOOLCHAINS))
-
 def parse_modules(*, module_ctx, _fail = fail):
     """Parse the modules and return a struct for registrations.
 
@@ -240,9 +234,6 @@ def parse_modules(*, module_ctx, _fail = fail):
     # toolchain. We need the default last.
     toolchains.append(default_toolchain)
 
-    if len(toolchains) > _MAX_NUM_TOOLCHAINS:
-        fail("more than {} python versions are not supported".format(_MAX_NUM_TOOLCHAINS))
-
     # sort the toolchains so that the toolchain versions that are in the
     # `minor_mapping` are coming first. This ensures that `python_version =
     # "3.X"` transitions work as expected.
@@ -275,6 +266,9 @@ def parse_modules(*, module_ctx, _fail = fail):
 def _python_impl(module_ctx):
     py = parse_modules(module_ctx = module_ctx)
 
+    # dict[str version, list[str] platforms]; where version is full
+    # python version string ("3.4.5"), and platforms are keys from
+    # the PLATFORMS global.
     loaded_platforms = {}
     for toolchain_info in py.toolchains:
         # Ensure that we pass the full version here.
@@ -297,30 +291,82 @@ def _python_impl(module_ctx):
             **kwargs
         )
 
-    # Create the pythons_hub repo for the interpreter meta data and the
-    # the various toolchains.
+    # List of the base names ("python_3_10") for the toolchain repos
+    base_toolchain_repo_names = []
+
+    # list[str] The infix to use for the resulting toolchain() `name` arg.
+    toolchain_names = []
+
+    # dict[str i, str repo]; where repo is the full repo name
+    # ("python_3_10_unknown-linux-x86_64") for the toolchain
+    # i corresponds to index `i` in toolchain_names
+    toolchain_repo_names = {}
+
+    # dict[str i, list[str] constraints]; where constraints is a list
+    # of labels for target_compatible_with
+    # i corresponds to index `i` in toolchain_names
+    toolchain_tcw_map = {}
+
+    # dict[str i, list[str] settings]; where settings is a list
+    # of labels for target_settings
+    # i corresponds to index `i` in toolchain_names
+    toolchain_ts_map = {}
+
+    # dict[str i, str set_constraint]; where set_constraint is the string
+    # "True" or "False".
+    # i corresponds to index `i` in toolchain_names
+    toolchain_set_python_version_constraints = {}
+
+    # dict[str i, str python_version]; where python_version is the full
+    # python version ("3.4.5").
+    toolchain_python_versions = {}
+
+    # dict[str i, str platform_key]; where platform_key is the key within
+    # the PLATFORMS global for this toolchain
+    toolchain_platform_keys = {}
+
+    # Split the toolchain info into separate objects so they can be passed onto
+    # the repository rule.
+    for i, t in enumerate(py.toolchains):
+        is_last = (i + 1) == len(py.toolchains)
+        base_name = t.name
+        base_toolchain_repo_names.append(base_name)
+        fv = full_version(version = t.python_version, minor_mapping = py.config.minor_mapping)
+        for platform in loaded_platforms[fv]:
+            if platform not in PLATFORMS:
+                continue
+            key = str(len(toolchain_names))
+
+            full_name = "{}_{}".format(base_name, platform)
+            toolchain_names.append(full_name)
+            toolchain_repo_names[key] = full_name
+            toolchain_tcw_map[key] = PLATFORMS[platform].compatible_with
+
+            # The target_settings attribute may not be present for users
+            # patching python/versions.bzl.
+            toolchain_ts_map[key] = getattr(PLATFORMS[platform], "target_settings", [])
+            toolchain_platform_keys[key] = platform
+            toolchain_python_versions[key] = fv
+
+            # The last toolchain is the default; it can't have version constraints
+            # Despite the implication of the arg name, the values are strs, not bools
+            toolchain_set_python_version_constraints[key] = (
+                "True" if not is_last else "False"
+            )
+
     hub_repo(
         name = "pythons_hub",
-        # Last toolchain is default
+        toolchain_names = toolchain_names,
+        toolchain_repo_names = toolchain_repo_names,
+        toolchain_target_compatible_with_map = toolchain_tcw_map,
+        toolchain_target_settings_map = toolchain_ts_map,
+        toolchain_platform_keys = toolchain_platform_keys,
+        toolchain_python_versions = toolchain_python_versions,
+        toolchain_set_python_version_constraints = toolchain_set_python_version_constraints,
+        base_toolchain_repo_names = [t.name for t in py.toolchains],
         default_python_version = py.default_python_version,
         minor_mapping = py.config.minor_mapping,
         python_versions = list(py.config.default["tool_versions"].keys()),
-        toolchain_prefixes = [
-            render.toolchain_prefix(index, toolchain.name, _TOOLCHAIN_INDEX_PAD_LENGTH)
-            for index, toolchain in enumerate(py.toolchains)
-        ],
-        toolchain_python_versions = [
-            full_version(version = t.python_version, minor_mapping = py.config.minor_mapping)
-            for t in py.toolchains
-        ],
-        # The last toolchain is the default; it can't have version constraints
-        # Despite the implication of the arg name, the values are strs, not bools
-        toolchain_set_python_version_constraints = [
-            "True" if i != len(py.toolchains) - 1 else "False"
-            for i in range(len(py.toolchains))
-        ],
-        toolchain_user_repository_names = [t.name for t in py.toolchains],
-        loaded_platforms = loaded_platforms,
     )
 
     # This is require in order to support multiple version py_test