pythongh-133779: Revert Windows generation of pyconfig.h and go back to a static header. (pythonGH-133966)

Extension builders must specify Py_GIL_DISABLED if they want to link to the free-threaded builds.
This was usually the case already, but this change guarantees it in all circumstances.

Author: zooba

Doc/howto/free-threading-extensions.rst

@@ -23,6 +23,14 @@ You can use it to enable code that only runs under the free-threaded build::
     /* code that only runs in the free-threaded build */
     #endif
 
+.. note::
+
+   On Windows, this macro is not defined automatically, but must be specified
+   to the compiler when building. The :func:`sysconfig.get_config_var` function
+   can be used to determine whether the current running interpreter had the
+   macro defined.
+
+
 Module Initialization
 =====================
 

Doc/whatsnew/3.14.rst

@@ -829,6 +829,12 @@ Kumar Aditya, Edgar Margffoy, and many others.
 Some of these contributors are employed by Meta, which has continued to provide
 significant engineering resources to support this project.
 
+From 3.14, when compiling extension modules for the free-threaded build of
+CPython on Windows, the preprocessor variable ``Py_GIL_DISABLED`` now needs to
+be specified by the build backend, as it will no longer be determined
+automatically by the C compiler. For a running interpreter, the setting that
+was used at compile time can be found using :func:`sysconfig.get_config_var`.
+
 
 .. _whatsnew314-pyrepl-highlighting:
 

Include/Python.h

@@ -59,6 +59,14 @@
 #  include <intrin.h>             // __readgsqword()
 #endif
 
+// Suppress known warnings in Python header files.
+#if defined(_MSC_VER)
+// Warning that alignas behaviour has changed. Doesn't affect us, because we
+// never relied on the old behaviour.
+#pragma warning(push)
+#pragma warning(disable: 5274)
+#endif
+
 // Include Python header files
 #include "pyport.h"
 #include "pymacro.h"
@@ -138,4 +146,9 @@
 #include "cpython/pyfpe.h"
 #include "cpython/tracemalloc.h"
 
+// Restore warning filter
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
 #endif /* !Py_PYTHON_H */

Lib/sysconfig/__init__.py

@@ -457,7 +457,7 @@ def get_config_h_filename():
     """Return the path of pyconfig.h."""
     if _PYTHON_BUILD:
         if os.name == "nt":
-            inc_dir = os.path.dirname(sys._base_executable)
+            inc_dir = os.path.join(_PROJECT_BASE, 'PC')
         else:
             inc_dir = _PROJECT_BASE
     else:

Lib/test/test_sysconfig.py

@@ -531,13 +531,10 @@ def test_srcdir(self):
             Python_h = os.path.join(srcdir, 'Include', 'Python.h')
             self.assertTrue(os.path.exists(Python_h), Python_h)
             # <srcdir>/PC/pyconfig.h.in always exists even if unused
-            pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h.in')
-            self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h)
             pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in')
             self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in)
             if os.name == 'nt':
-                # <executable dir>/pyconfig.h exists on Windows in a build tree
-                pyconfig_h = os.path.join(sys.executable, '..', 'pyconfig.h')
+                pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h')
                 self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h)
         elif os.name == 'posix':
             makefile_dir = os.path.dirname(sysconfig.get_makefile_filename())

Misc/NEWS.d/next/Windows/2025-05-13-13-25-27.gh-issue-133779.-YcTBz.rst

@@ -0,0 +1,6 @@
+Reverts the change to generate different :file:`pyconfig.h` files based on
+compiler settings, as it was frequently causing extension builds to break.
+In particular, the ``Py_GIL_DISABLED`` preprocessor variable must now always
+be defined explicitly when compiling for the experimental free-threaded
+runtime. The :func:`sysconfig.get_config_var` function can be used to
+determine whether the current runtime was compiled with that flag or not.

PC/pyconfig.h.in renamed to PC/pyconfig.h

@@ -99,12 +99,17 @@ WIN32 is still required for the locale module.
 #  define Py_DEBUG 1
 #endif
 
-/* Define to 1 if you want to disable the GIL */
-/* Uncomment the definition for free-threaded builds, or define it manually
- * when compiling extension modules. Note that we test with #ifdef, so
- * defining as 0 will still disable the GIL. */
-#ifndef Py_GIL_DISABLED
-/* #define Py_GIL_DISABLED 1 */
+/* Define to 1 when compiling for experimental free-threaded builds */
+#ifdef Py_GIL_DISABLED
+/* We undefine if it was set to zero because all later checks are #ifdef.
+ * Note that non-Windows builds do not do this, and so every effort should
+ * be made to avoid defining the variable at all when not desired. However,
+ * sysconfig.get_config_var always returns a 1 or a 0, and so it seems likely
+ * that a build backend will define it with the value.
+ */
+#if Py_GIL_DISABLED == 0
+#undef Py_GIL_DISABLED
+#endif
 #endif
 
 /* Compiler specific defines */

PCbuild/_freeze_module.vcxproj

@@ -276,7 +276,7 @@
     <ClCompile Include="..\Python\uniqueid.c" />
   </ItemGroup>
   <ItemGroup>
-    <ClInclude Include="..\PC\pyconfig.h.in" />
+    <ClInclude Include="..\PC\pyconfig.h" />
   </ItemGroup>
   <ItemGroup>
     <!-- BEGIN frozen modules -->
@@ -436,31 +436,6 @@
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
 
-  <!-- Direct copy from pythoncore.vcxproj, but this one is only used for our
-       own build. All other extension modules will use the copy that pythoncore
-       generates. -->
-  <Target Name="_UpdatePyconfig" BeforeTargets="PrepareForBuild">
-    <MakeDir Directories="$(IntDir)" Condition="!Exists($(IntDir))" />
-    <ItemGroup>
-      <PyConfigH Remove="@(PyConfigH)" />
-      <PyConfigH Include="@(ClInclude)" Condition="'%(Filename)%(Extension)' == 'pyconfig.h.in'" />
-    </ItemGroup>
-    <Error Text="Did not find pyconfig.h" Condition="@(ClInclude) == ''" />
-    <PropertyGroup>
-      <PyConfigH>@(PyConfigH->'%(FullPath)', ';')</PyConfigH>
-      <PyConfigHText>$([System.IO.File]::ReadAllText($(PyConfigH)))</PyConfigHText>
-      <OldPyConfigH Condition="Exists('$(IntDir)pyconfig.h')">$([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h'))</OldPyConfigH>
-    </PropertyGroup>
-    <PropertyGroup Condition="$(DisableGil) == 'true'">
-      <PyConfigHText>$(PyConfigHText.Replace('/* #define Py_GIL_DISABLED 1 */', '#define Py_GIL_DISABLED 1'))</PyConfigHText>
-    </PropertyGroup>
-    <Message Text="Updating pyconfig.h" Condition="$(PyConfigHText.TrimEnd()) != $(OldPyConfigH.TrimEnd())" />
-    <WriteLinesToFile File="$(IntDir)pyconfig.h"
-                      Lines="$(PyConfigHText)"
-                      Overwrite="true"
-                      Condition="$(PyConfigHText.TrimEnd()) != $(OldPyConfigH.TrimEnd())" />
-  </Target>
-
   <Target Name="_RebuildGetPath" AfterTargets="_RebuildFrozen" Condition="$(Configuration) != 'PGUpdate'">
     <Exec Command='"$(TargetPath)" "%(GetPath.ModName)" "%(GetPath.FullPath)" "%(GetPath.IntFile)"' />
 

PCbuild/pyproject.props

@@ -10,10 +10,9 @@
     <Py_IntDir Condition="'$(Py_IntDir)' == ''">$(MSBuildThisFileDirectory)obj\</Py_IntDir>
     <IntDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\</IntDir>
     <IntDir>$(IntDir.Replace(`\\`, `\`))</IntDir>
-    <!-- pyconfig.h is updated by pythoncore.vcxproj, so it's always in pythoncore's IntDir -->
-    <GeneratedPyConfigDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\pythoncore\</GeneratedPyConfigDir>
     <GeneratedFrozenModulesDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen\</GeneratedFrozenModulesDir>
     <GeneratedZlibNgDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\zlib-ng\</GeneratedZlibNgDir>
+    <GeneratedJitStencilsDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_$(Configuration)</GeneratedJitStencilsDir>
     <TargetName Condition="'$(TargetName)' == ''">$(ProjectName)</TargetName>
     <TargetName>$(TargetName)$(PyDebugExt)</TargetName>
     <GenerateManifest>false</GenerateManifest>
@@ -49,11 +48,12 @@
     <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64;</_PlatformPreprocessorDefinition>
     <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64' and $(PlatformToolset) != 'ClangCL'">_M_X64;$(_PlatformPreprocessorDefinition)</_PlatformPreprocessorDefinition>
     <_Py3NamePreprocessorDefinition>PY3_DLLNAME=L"$(Py3DllName)$(PyDebugExt)";</_Py3NamePreprocessorDefinition>
+    <_FreeThreadedPreprocessorDefinition Condition="$(DisableGil) == 'true'">Py_GIL_DISABLED=1;</_FreeThreadedPreprocessorDefinition>
   </PropertyGroup>
   <ItemDefinitionGroup>
     <ClCompile>
-      <AdditionalIncludeDirectories>$(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(GeneratedPyConfigDir);$(PySourcePath)PC;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PyStatsPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>$(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(PySourcePath)PC;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;$(_Py3NamePreprocessorDefinition)$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PyStatsPreprocessorDefinition)$(_PydPreprocessorDefinition)$(_FreeThreadedPreprocessorDefinition)%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="'$(SupportPGO)' and ($(Configuration) == 'PGInstrument' or $(Configuration) == 'PGUpdate')">_Py_USING_PGO=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
 
       <Optimization>MaxSpeed</Optimization>

PCbuild/pythoncore.vcxproj

@@ -102,6 +102,7 @@
       <AdditionalOptions>/Zm200  %(AdditionalOptions)</AdditionalOptions>
       <AdditionalIncludeDirectories>$(PySourcePath)Modules\_hacl;$(PySourcePath)Modules\_hacl\include;$(PySourcePath)Python;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <AdditionalIncludeDirectories Condition="$(IncludeExternals)">$(zlibNgDir);$(GeneratedZlibNgDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories Condition="'$(UseJIT)' == 'true'">$(GeneratedJitStencilsDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>_USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";ZLIB_COMPAT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="$(IncludeExternals)">_Py_HAVE_ZLIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="'$(UseJIT)' == 'true'">_Py_JIT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
@@ -409,7 +410,7 @@
     <ClInclude Include="..\Parser\string_parser.h" />
     <ClInclude Include="..\Parser\pegen.h" />
     <ClInclude Include="..\PC\errmap.h" />
-    <ClInclude Include="..\PC\pyconfig.h.in" />
+    <ClInclude Include="..\PC\pyconfig.h" />
     <ClInclude Include="..\Python\condvar.h" />
     <ClInclude Include="..\Python\stdlib_module_names.h" />
     <ClInclude Include="..\Python\thread_nt.h" />
@@ -688,34 +689,6 @@
   </ImportGroup>
   <Target Name="_TriggerRegen" BeforeTargets="PrepareForBuild" DependsOnTargets="Regen" />
 
-  <Target Name="_UpdatePyconfig" BeforeTargets="PrepareForBuild">
-    <MakeDir Directories="$(IntDir)" Condition="!Exists($(IntDir))" />
-    <ItemGroup>
-      <PyConfigH Remove="@(PyConfigH)" />
-      <PyConfigH Include="@(ClInclude)" Condition="'%(Filename)%(Extension)' == 'pyconfig.h.in'" />
-    </ItemGroup>
-    <Error Text="Did not find pyconfig.h" Condition="@(ClInclude) == ''" />
-    <PropertyGroup>
-      <PyConfigH>@(PyConfigH->'%(FullPath)', ';')</PyConfigH>
-      <PyConfigHText>$([System.IO.File]::ReadAllText($(PyConfigH)))</PyConfigHText>
-      <OldPyConfigH Condition="Exists('$(IntDir)pyconfig.h')">$([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h'))</OldPyConfigH>
-    </PropertyGroup>
-    <PropertyGroup Condition="$(DisableGil) == 'true'">
-      <PyConfigHText>$(PyConfigHText.Replace('/* #define Py_GIL_DISABLED 1 */', '#define Py_GIL_DISABLED 1'))</PyConfigHText>
-    </PropertyGroup>
-    <Message Text="Updating pyconfig.h" Condition="$(PyConfigHText.TrimEnd()) != $(OldPyConfigH.TrimEnd())" />
-    <WriteLinesToFile File="$(IntDir)pyconfig.h"
-                      Lines="$(PyConfigHText)"
-                      Overwrite="true"
-                      Condition="$(PyConfigHText.TrimEnd()) != $(OldPyConfigH.TrimEnd())" />
-  </Target>
-  <Target Name="_CopyPyconfig" Inputs="$(IntDir)pyconfig.h" Outputs="$(OutDir)pyconfig.h" AfterTargets="Build" DependsOnTargets="_UpdatePyconfig">
-    <Copy SourceFiles="$(IntDir)pyconfig.h" DestinationFolder="$(OutDir)" />
-  </Target>
-  <Target Name="_CleanPyconfig" AfterTargets="Clean">
-    <Delete Files="$(IntDir)pyconfig.h;$(OutDir)pyconfig.h" />
-  </Target>
-
   <Target Name="_GetBuildInfo" BeforeTargets="PrepareForBuild">
     <PropertyGroup>
       <GIT Condition="$(GIT) == ''">git</GIT>

PCbuild/regen.targets

@@ -29,12 +29,12 @@
     <_KeywordSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" />
     <_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" />
     <!-- Taken from _Target._compute_digest in Tools\jit\_targets.py: -->
-    <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/>
+    <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(PySourcePath)PC\pyconfig.h;$(PySourcePath)Tools\jit\**"/>
     <!-- Need to explicitly enumerate these, since globbing doesn't work for missing outputs: -->
-    <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/>
-    <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-aarch64-pc-windows-msvc.h" Condition="$(Platform) == 'ARM64'"/>
-    <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-i686-pc-windows-msvc.h" Condition="$(Platform) == 'Win32'"/>
-    <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-x86_64-pc-windows-msvc.h" Condition="$(Platform) == 'x64'"/>
+    <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils.h"/>
+    <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-aarch64-pc-windows-msvc.h" Condition="$(Platform) == 'ARM64'"/>
+    <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-i686-pc-windows-msvc.h" Condition="$(Platform) == 'Win32'"/>
+    <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-x86_64-pc-windows-msvc.h" Condition="$(Platform) == 'x64'"/>
     <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/>
     <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/>
     <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" />
@@ -116,7 +116,7 @@
 
   <Target Name="_RegenJIT"
           Condition="'$(UseJIT)' == 'true'"
-          DependsOnTargets="_UpdatePyconfig;FindPythonForBuild"
+          DependsOnTargets="FindPythonForBuild"
           Inputs="@(_JITSources)"
           Outputs="@(_JITOutputs)">
     <PropertyGroup>
@@ -126,7 +126,7 @@
       <JITArgs Condition="$(Configuration) == 'Debug'">$(JITArgs) --debug</JITArgs>
     </PropertyGroup>
     <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\jit\build.py" $(JITArgs)'
-          WorkingDirectory="$(GeneratedPyConfigDir)"/>
+          WorkingDirectory="$(GeneratedJitStencilsDir)"/>
   </Target>
   <Target Name="_CleanJIT" AfterTargets="Clean">
     <Delete Files="@(_JITOutputs)"/>

Tools/msi/dev/dev_files.wxs

@@ -3,7 +3,7 @@
     <Fragment>
         <ComponentGroup Id="dev_pyconfig">
             <Component Id="include_pyconfig.h" Directory="include" Guid="*">
-                <File Id="include_pyconfig.h" Name="pyconfig.h" Source="pyconfig.h" KeyPath="yes" />
+                <File Id="include_pyconfig.h" Name="pyconfig.h" Source="!(bindpath.src)PC\pyconfig.h" KeyPath="yes" />
             </Component>
         </ComponentGroup>
     </Fragment>

Tools/peg_generator/pegen/build.py

@@ -108,6 +108,8 @@ def compile_c_extension(
     extra_compile_args.append("-DPy_BUILD_CORE_MODULE")
     # Define _Py_TEST_PEGEN to not call PyAST_Validate() in Parser/pegen.c
     extra_compile_args.append("-D_Py_TEST_PEGEN")
+    if sys.platform == "win32" and sysconfig.get_config_var("Py_GIL_DISABLED"):
+        extra_compile_args.append("-DPy_GIL_DISABLED")
     extra_link_args = get_extra_flags("LDFLAGS", "PY_LDFLAGS_NODIST")
     if keep_asserts:
         extra_compile_args.append("-UNDEBUG")