D7net
Home
Console
Upload
information
Create File
Create Folder
About
Tools
:
/
opt
/
saltstack
/
salt
/
lib
/
python3.10
/
site-packages
/
relenv
/
build
/
Filename :
windows.py
back
Copy
# Copyright 2022-2026 Broadcom. # SPDX-License-Identifier: Apache-2 # mypy: ignore-errors """ The windows build process. """ from __future__ import annotations import glob import json import logging import os import pathlib import shutil import subprocess import sys import tarfile import time from collections.abc import MutableMapping from typing import IO from ..common import ( MODULE_DIR, WIN32, arches, download_url, extract_archive, runcmd, ) from .common import ( Dirs, builds, create_archive, get_dependency_version, install_runtime, patch_file, update_ensurepip, update_sbom_checksums, ) log = logging.getLogger(__name__) ARCHES = arches[WIN32] EnvMapping = MutableMapping[str, str] if sys.platform == WIN32: import ctypes kernel32 = ctypes.windll.kernel32 kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7) def populate_env(env: EnvMapping, dirs: Dirs) -> None: """ Make sure we have the correct environment variables set. :param env: The environment dictionary :type env: dict :param dirs: The working directories :type dirs: ``relenv.build.common.Dirs`` """ env["MSBUILDDISABLENODEREUSE"] = "1" def find_vcvarsall(env: EnvMapping) -> pathlib.Path | None: """ Locate vcvarsall.bat using multiple strategies. """ # 1. Check MSBUILD env var and search upwards msbuild_path = env.get("MSBUILD") if msbuild_path: msbuild_path = pathlib.Path(msbuild_path) if msbuild_path.exists(): for parent in msbuild_path.parents: candidate = parent / "VC" / "Auxiliary" / "Build" / "vcvarsall.bat" if candidate.exists(): return candidate # 2. Use vswhere.exe if available vswhere = shutil.which("vswhere.exe") if not vswhere: # Check common location vswhere_path = ( pathlib.Path(os.environ.get("ProgramFiles(x86)", "C:\\Program Files (x86)")) / "Microsoft Visual Studio" / "Installer" / "vswhere.exe" ) if vswhere_path.exists(): vswhere = str(vswhere_path) if vswhere: try: # -latest: Use newest version # -products *: Search all products (Enterprise, Professional, Community, BuildTools) # -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64: Ensure C++ tools are present # -property installationPath: Return the path cmd = [ vswhere, "-latest", "-products", "*", "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "-property", "installationPath", ] result = subprocess.run(cmd, capture_output=True, text=True, check=True) vs_path = result.stdout.strip() if vs_path: candidate = pathlib.Path(vs_path) / "VC" / "Auxiliary" / "Build" / "vcvarsall.bat" if candidate.exists(): return candidate except subprocess.CalledProcessError: pass # 3. Check common installation paths as a last resort program_files = [ os.environ.get("ProgramFiles", "C:\\Program Files"), os.environ.get("ProgramFiles(x86)", "C:\\Program Files (x86)"), ] editions = ["Enterprise", "Professional", "Community", "BuildTools"] years = ["2022", "2019", "2017"] for pf in program_files: for year in years: for edition in editions: candidate = ( pathlib.Path(pf) / "Microsoft Visual Studio" / year / edition / "VC" / "Auxiliary" / "Build" / "vcvarsall.bat" ) if candidate.exists(): return candidate return None def update_props(source: pathlib.Path, old: str, new: str) -> None: """ Overwrite a dependency string for Windows PCBuild. :param source: Python's source directory :type source: str :param old: Regular expression to search for :type old: str :param new: Replacement text :type new: str """ log.info("Patching props in %s: %s -> %s", source, old, new) # patch_file uses re.sub, so we need to ensure backslashes are preserved. new_escaped = new.replace("\\", "\\\\") patch_file(source / "PCbuild" / "python.props", old, new_escaped) patch_file(source / "PCbuild" / "get_externals.bat", old, new_escaped) def flatten_externals(dirs: Dirs, name: str, version: str) -> None: """ Handle nested folders in extracted tarballs. """ # Look for the extracted directory # For cpython-bin-deps, it often extracts to <repo>-<tag>/... # We want it to be in externals/<name>-<version>/ externals_dir = dirs.source / "externals" # Identify what was actually extracted # extract_archive usually extracts into externals_dir # We search for any directory that isn't 'zips' extracted_dirs = [x for x in externals_dir.iterdir() if x.is_dir() and x.name != "zips"] target_dir = externals_dir / f"{name}-{version}" for d in extracted_dirs: if d == target_dir: # Check if it's nested (e.g. openssl-3.0.15/openssl-3.0.15/...) subdirs = [x for x in d.iterdir() if x.is_dir()] if len(subdirs) == 1 and subdirs[0].name.startswith(name): log.info("Flattening nested %s", d.name) temp_dir = externals_dir / f"{name}-{version}-tmp" shutil.move(str(subdirs[0]), str(temp_dir)) shutil.rmtree(str(d)) shutil.move(str(temp_dir), str(d)) continue if d.name.startswith(name) or "cpython-bin-deps" in d.name: log.info("Moving %s to %s", d.name, target_dir.name) if target_dir.exists(): shutil.rmtree(str(target_dir)) shutil.move(str(d), str(target_dir)) # Recurse once to handle nested folder inside the renamed folder flatten_externals(dirs, name, version) def get_externals_source(externals_dir: pathlib.Path, url: str) -> None: """ Download external source code dependency. """ zips_dir = externals_dir / "zips" zips_dir.mkdir(parents=True, exist_ok=True) local_file = download_url(url=url, dest=str(zips_dir)) extract_archive(archive=str(local_file), to_dir=str(externals_dir)) try: os.remove(local_file) except OSError: log.exception("Failed to remove temporary file") def update_sqlite(dirs: Dirs, env: EnvMapping) -> None: """ Update the SQLITE library. """ sqlite_info = get_dependency_version("sqlite", "win32") if not sqlite_info: return version = sqlite_info["version"] sqliteversion = sqlite_info.get("sqliteversion", "3500400") url = sqlite_info["url"].format(version=sqliteversion) sha256 = sqlite_info["sha256"] ref_loc = f"cpe:2.3:a:sqlite:sqlite:{version}:*:*:*:*:*:*:*" # noqa: E231 target_dir = dirs.source / "externals" / f"sqlite-{version}" update_props(dirs.source, r"sqlite-\d+(\.\d+)*", f"sqlite-{version}") if not target_dir.exists(): get_externals_source(externals_dir=dirs.source / "externals", url=url) # Fix the extracted directory name (sqlite-autoconf-...) for d in (dirs.source / "externals").iterdir(): if d.is_dir() and d.name.startswith("sqlite-autoconf"): shutil.move(str(d), str(target_dir)) # Update externals.spdx.json if env["RELENV_PY_MAJOR_VERSION"] in ["3.12", "3.13", "3.14"]: spdx_json = dirs.source / "Misc" / "externals.spdx.json" if spdx_json.exists(): with open(str(spdx_json)) as f: data = json.load(f) for pkg in data["packages"]: if pkg["name"] == "sqlite": pkg["versionInfo"] = version pkg["downloadLocation"] = url pkg["checksums"][0]["checksumValue"] = sha256 pkg["externalRefs"][0]["referenceLocator"] = ref_loc with open(str(spdx_json), "w") as f: json.dump(data, f, indent=2) def update_xz(dirs: Dirs, env: EnvMapping) -> None: """ Update the XZ library. """ xz_info = get_dependency_version("xz", "win32") if not xz_info: return version = xz_info["version"] url = xz_info["url"].format(version=version) sha256 = xz_info["sha256"] ref_loc = f"cpe:2.3:a:tukaani:xz:{version}:*:*:*:*:*:*:*" # noqa: E231 target_dir = dirs.source / "externals" / f"xz-{version}" update_props(dirs.source, r"xz-\d+(\.\d+)*", f"xz-{version}") if not target_dir.exists(): get_externals_source(externals_dir=dirs.source / "externals", url=url) flatten_externals(dirs, "xz", version) # Bring config.h for MSBuild compatibility config_file = target_dir / "src" / "common" / "config.h" config_file_source = dirs.root / "_resources" / "xz" / "config.h" if not config_file.exists(): config_file.parent.mkdir(parents=True, exist_ok=True) shutil.copy(str(config_file_source), str(config_file)) # Copy missing crc source files check_dir = target_dir / "src" / "liblzma" / "check" for filename in ["crc32_table.c", "crc64_table.c"]: target_file = check_dir / filename source_file = dirs.root / "_resources" / "xz" / filename if not target_file.exists(): target_file.parent.mkdir(parents=True, exist_ok=True) shutil.copy(str(source_file), str(target_file)) # Update externals.spdx.json if env["RELENV_PY_MAJOR_VERSION"] in ["3.12", "3.13", "3.14"]: spdx_json = dirs.source / "Misc" / "externals.spdx.json" if spdx_json.exists(): with open(str(spdx_json)) as f: data = json.load(f) for pkg in data["packages"]: if pkg["name"] == "xz": pkg["versionInfo"] = version pkg["downloadLocation"] = url pkg["checksums"][0]["checksumValue"] = sha256 pkg["externalRefs"][0]["referenceLocator"] = ref_loc with open(str(spdx_json), "w") as f: json.dump(data, f, indent=2) def update_expat(dirs: Dirs, env: EnvMapping) -> None: """ Update the EXPAT library. """ expat_info = get_dependency_version("expat", "win32") if not expat_info: return version = expat_info["version"] url = expat_info["url"].format(version=version) sha256 = expat_info["sha256"] bash_refresh = dirs.source / "Modules" / "expat" / "refresh.sh" if bash_refresh.exists(): patch_file( bash_refresh, old=r'expected_libexpat_tag="R_\d+(_\d+)*"', new=f'expected_libexpat_tag="R_{version.replace(".", "_")}"', ) patch_file( bash_refresh, old=r'expected_libexpat_version="\d+(\.\d+)*"', new=f'expected_libexpat_version="{version}"', ) patch_file( bash_refresh, old='expected_libexpat_sha256=".*"', new=f'expected_libexpat_sha256="{sha256}"', ) get_externals_source(externals_dir=dirs.source / "Modules" / "expat", url=url) expat_lib_dir = dirs.source / "Modules" / "expat" / f"expat-{version}" / "lib" expat_dir = dirs.source / "Modules" / "expat" updated_files = [] for ext in ["*.h", "*.c"]: for file in glob.glob(str(expat_lib_dir / ext)): target = expat_dir / os.path.basename(file) if target.exists(): target.unlink() shutil.move(file, str(expat_dir)) updated_files.append(target) now = time.time() for target_file in updated_files: os.utime(target_file, (now, now)) # For expat >= 2.8.0, new entropy source files are required but not compiled # by Python's build system. Include them directly in xmlparse.c. xmlparse_c = expat_dir / "xmlparse.c" if xmlparse_c.exists(): with open(str(xmlparse_c), "a") as f: f.write("\n/* Relenv: include new entropy sources for expat >= 2.8.0 */\n") f.write('#if defined(_WIN32)\n#include "random_rand_s.c"\n#endif\n') f.write('#if defined(HAVE_GETENTROPY)\n#include "random_getentropy.c"\n#endif\n') f.write("#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM)\n") f.write('#include "random_getrandom.c"\n#endif\n') f.write('#if defined(HAVE_ARC4RANDOM_BUF)\n#include "random_arc4random_buf.c"\n#endif\n') f.write('#if defined(HAVE_ARC4RANDOM)\n#include "random_arc4random.c"\n#endif\n') f.write('#if !defined(_WIN32) && defined(XML_DEV_URANDOM)\n#include "random_dev_urandom.c"\n#endif\n') # Update SBOM with correct checksums for updated expat files files_to_update = {f"Modules/expat/{f.name}": f for f in updated_files} if bash_refresh.exists(): files_to_update["Modules/expat/refresh.sh"] = bash_refresh update_sbom_checksums(dirs.source, files_to_update) def update_openssl(dirs: Dirs, env: EnvMapping) -> None: """ Update the OPENSSL library. """ openssl_info = get_dependency_version("openssl", "win32") if not openssl_info: return version = openssl_info["version"] url = openssl_info["url"].format(version=version) sha256 = openssl_info["sha256"] ref_loc = f"cpe:2.3:a:openssl:openssl:{version}:*:*:*:*:*:*:*" # noqa: E231 is_binary = "cpython-bin-deps" in url target_dir = dirs.source / "externals" / f"openssl-{version}-{env['RELENV_HOST_ARCH']}" update_props( dirs.source, r"openssl-\d+(\.\d+)*[a-z]*", f"openssl-{version}-{env['RELENV_HOST_ARCH']}", ) # Binary deps tarball from cpython-bin-deps includes both source and binaries # We need to ensure openssl-bin-<version> is also pointed to the same place if needed update_props( dirs.source, r"openssl-bin-\d+(\.\d+)*[a-z]*", f"openssl-{version}-{env['RELENV_HOST_ARCH']}", ) if not target_dir.exists(): get_externals_source(externals_dir=dirs.source / "externals", url=url) # flatten_externals(dirs, "openssl", version) would move it to openssl-<version> # but we want openssl-<version>-<arch>. # We'll find it and move it ourselves. for d in (dirs.source / "externals").iterdir(): if d.is_dir() and (d.name == f"openssl-{version}" or d.name.startswith(f"openssl-{version}")): if d != target_dir: if target_dir.exists(): shutil.rmtree(str(target_dir)) shutil.move(str(d), str(target_dir)) break # Now flatten if it's nested subdirs = [x for x in target_dir.iterdir() if x.is_dir()] if len(subdirs) == 1 and subdirs[0].name.startswith("openssl"): log.info("Flattening nested OpenSSL") temp_dir = target_dir.parent / f"openssl-{version}-tmp" shutil.move(str(subdirs[0]), str(temp_dir)) shutil.rmtree(str(target_dir)) shutil.move(str(temp_dir), str(target_dir)) if not is_binary: # Build from source log.info("Building OpenSSL %s (%s) from source", version, env["RELENV_HOST_ARCH"]) perl_dir = update_perl(dirs, env) perl_bin = perl_dir / "perl" / "bin" / "perl.exe" nasm_info = get_dependency_version("nasm", "win32") nasm_version = nasm_info["version"] nasm_dir = dirs.source / "externals" / f"nasm-{nasm_version}" # Find nasm.exe nasm_exe = list(nasm_dir.glob("**/nasm.exe")) if not nasm_exe: log.error("Could not find nasm.exe in %s", nasm_dir) return arch_map = { "amd64": "VC-WIN64A", "x86": "VC-WIN32", "arm64": "VC-WIN64-ARM", } target = arch_map.get(env["RELENV_HOST_ARCH"], "VC-WIN64A") vcvars = find_vcvarsall(env) if not vcvars: log.warning("Could not find vcvarsall.bat, build may fail") vcvars_cmd = "echo" else: vcvars_arch = "x64" if env["RELENV_HOST_ARCH"] == "amd64" else env["RELENV_HOST_ARCH"] vcvars_cmd = f'call "{vcvars}" {vcvars_arch}' env_path = os.environ.get("PATH", "") build_env = env.copy() build_env["PATH"] = f"{perl_bin.parent};{nasm_exe[0].parent};{env_path}" # noqa: E231,E702 prefix = target_dir / "build" openssldir = prefix / "ssl" # Create a temporary batch file to run the build # This is more robust than passing a long string to cmd /c build_bat = target_dir / "relenv_build_openssl.bat" with open(str(build_bat), "w") as f: f.write("@echo off\n") f.write(f"{vcvars_cmd}\n") f.write("if %errorlevel% neq 0 exit /b %errorlevel%\n") f.write(f'cd /d "{target_dir}"\n') f.write( f'"{perl_bin}" Configure {target} --prefix="{prefix}" ' f'--openssldir="{openssldir}" no-unit-test no-tests\n' ) f.write("if %errorlevel% neq 0 exit /b %errorlevel%\n") f.write("nmake\n") f.write("if %errorlevel% neq 0 exit /b %errorlevel%\n") f.write("nmake install_sw\n") f.write("if %errorlevel% neq 0 exit /b %errorlevel%\n") log.info("Running OpenSSL build batch file") runcmd([str(build_bat)], env=build_env) # CPython expects binaries in a specific structure # opensslOutDir = $(ExternalsDir)openssl-bin-<version)\$(ArchName)\ # We'll move them to match. arch_to_archname = { "amd64": "amd64", "x86": "win32", "arm64": "arm64", "arm": "arm32", } arch_name = arch_to_archname.get(env["RELENV_HOST_ARCH"], env["RELENV_HOST_ARCH"]) out_dir = target_dir / arch_name out_dir.mkdir(parents=True, exist_ok=True) prefix = target_dir / "build" bin_dir = prefix / "bin" lib_dir = prefix / "lib" inc_dir = prefix / "include" if prefix.exists(): for f in bin_dir.glob("*.dll"): shutil.copy(str(f), str(out_dir)) for f in bin_dir.glob("*.pdb"): shutil.copy(str(f), str(out_dir)) for f in lib_dir.glob("*.lib"): shutil.copy(str(f), str(out_dir)) # CPython expects headers in $(opensslOutDir)include (out_dir / "include").mkdir(parents=True, exist_ok=True) shutil.copytree(str(inc_dir), str(out_dir / "include"), dirs_exist_ok=True) # CPython specifically looks for applink.c in the include directory if (target_dir / "ms" / "applink.c").exists(): shutil.copy( target_dir / "ms" / "applink.c", out_dir / "include" / "applink.c", ) # Copy LICENSE file to out_dir to satisfy CPython build for license_file in ["LICENSE", "LICENSE.txt", "COPYING"]: if (target_dir / license_file).exists(): shutil.copy(str(target_dir / license_file), str(out_dir / "LICENSE")) break if is_binary: # Ensure include/openssl exists inc_openssl_dir = target_dir / "include" / "openssl" if not inc_openssl_dir.exists(): # Try to find headers and move them for h in target_dir.glob("**/opensslv.h"): if h.parent.name == "openssl": # Found it, move its parent to include/ shutil.copytree(str(h.parent), str(inc_openssl_dir), dirs_exist_ok=True) break # Ensure applink.c is in include/ if not (target_dir / "include" / "applink.c").exists(): for a in target_dir.glob("**/applink.c"): shutil.copy(str(a), str(target_dir / "include" / "applink.c")) break if not is_binary: # Update props to point to our custom build # opensslDir is the base directory update_props( dirs.source, r"<opensslDir.*>.*</opensslDir>", ( f"<opensslDir Condition=\"$(opensslDir) == ''\">" f"$(ExternalsDir)openssl-{version}-{env['RELENV_HOST_ARCH']}\\" f"</opensslDir>" ), ) # opensslOutDir is where the binaries and include folder are update_props( dirs.source, r"<opensslOutDir.*>.*</opensslOutDir>", (f"<opensslOutDir Condition=\"$(opensslOutDir) == ''\">$(opensslDir){arch_name}\\</opensslOutDir>"), ) # Patch openssl.props to use correct DLL suffix for OpenSSL 3.x if version.startswith("3."): suffix = "-3" if not is_binary and env["RELENV_HOST_ARCH"] == "amd64": suffix = "-3-x64" log.info("Patching openssl.props DLL suffix to %s", suffix) patch_file( dirs.source / "PCbuild" / "openssl.props", r"<_DLLSuffix>.*</_DLLSuffix>", f"<_DLLSuffix>{suffix}</_DLLSuffix>", ) # Update externals.spdx.json if env["RELENV_PY_MAJOR_VERSION"] in ["3.12", "3.13", "3.14"]: spdx_json = dirs.source / "Misc" / "externals.spdx.json" if spdx_json.exists(): with open(str(spdx_json)) as f: data = json.load(f) for pkg in data["packages"]: if pkg["name"] == "openssl": pkg["versionInfo"] = version pkg["downloadLocation"] = url pkg["checksums"][0]["checksumValue"] = sha256 pkg["externalRefs"][0]["referenceLocator"] = ref_loc with open(str(spdx_json), "w") as f: json.dump(data, f, indent=2) def update_bzip2(dirs: Dirs, env: EnvMapping) -> None: """ Update the BZIP2 library. """ bzip2_info = get_dependency_version("bzip2", "win32") if not bzip2_info: return version = bzip2_info["version"] url = bzip2_info["url"].format(version=version) sha256 = bzip2_info["sha256"] ref_loc = f"cpe:2.3:a:bzip:bzip2:{version}:*:*:*:*:*:*:*" # noqa: E231 target_dir = dirs.source / "externals" / f"bzip2-{version}" update_props(dirs.source, r"bzip2-\d+(\.\d+)*", f"bzip2-{version}") if not target_dir.exists(): get_externals_source(externals_dir=dirs.source / "externals", url=url) flatten_externals(dirs, "bzip2", version) # Update externals.spdx.json if env["RELENV_PY_MAJOR_VERSION"] in ["3.12", "3.13", "3.14"]: spdx_json = dirs.source / "Misc" / "externals.spdx.json" if spdx_json.exists(): with open(str(spdx_json)) as f: data = json.load(f) for pkg in data["packages"]: if pkg["name"] == "bzip2": pkg["versionInfo"] = version pkg["downloadLocation"] = url pkg["checksums"][0]["checksumValue"] = sha256 pkg["externalRefs"][0]["referenceLocator"] = ref_loc with open(str(spdx_json), "w") as f: json.dump(data, f, indent=2) def update_libffi(dirs: Dirs, env: EnvMapping) -> None: """ Update the LIBFFI library. """ libffi_info = get_dependency_version("libffi", "win32") if not libffi_info: return version = libffi_info["version"] url = libffi_info["url"].format(version=version) sha256 = libffi_info["sha256"] ref_loc = f"cpe:2.3:a:libffi_project:libffi:{version}:*:*:*:*:*:*:*" # noqa: E231 target_dir = dirs.source / "externals" / f"libffi-{version}" update_props(dirs.source, r"libffi-\d+(\.\d+)*", f"libffi-{version}") if not target_dir.exists(): get_externals_source(externals_dir=dirs.source / "externals", url=url) flatten_externals(dirs, "libffi", version) # Patch libffi library name if needed. # Newer libffi (3.4.4+) uses libffi-8.lib, older uses libffi-7.lib. # We'll search for the lib file after extraction. # Find the .lib file to determine the name lib_files = list(target_dir.glob("**/*.lib")) if lib_files: lib_name = lib_files[0].name if lib_name != "libffi-7.lib": log.info("Patching libffi library name to %s", lib_name) patch_file(dirs.source / "PCbuild" / "libffi.props", r"libffi-7\.lib", lib_name) patch_file( dirs.source / "PCbuild" / "libffi.props", r"libffi-7\.dll", lib_name.replace(".lib", ".dll"), ) # Update externals.spdx.json if env["RELENV_PY_MAJOR_VERSION"] in ["3.12", "3.13", "3.14"]: spdx_json = dirs.source / "Misc" / "externals.spdx.json" if spdx_json.exists(): with open(str(spdx_json)) as f: data = json.load(f) for pkg in data["packages"]: if pkg["name"] == "libffi": pkg["versionInfo"] = version pkg["downloadLocation"] = url pkg["checksums"][0]["checksumValue"] = sha256 pkg["externalRefs"][0]["referenceLocator"] = ref_loc with open(str(spdx_json), "w") as f: json.dump(data, f, indent=2) def update_zlib(dirs: Dirs, env: EnvMapping) -> None: """ Update the ZLIB library. """ zlib_info = get_dependency_version("zlib", "win32") if not zlib_info: return version = zlib_info["version"] url = zlib_info["url"].format(version=version) sha256 = zlib_info["sha256"] ref_loc = f"cpe:2.3:a:gnu:zlib:{version}:*:*:*:*:*:*:*" # noqa: E231 target_dir = dirs.source / "externals" / f"zlib-{version}" update_props(dirs.source, r"zlib-\d+(\.\d+)*", f"zlib-{version}") if not target_dir.exists(): get_externals_source(externals_dir=dirs.source / "externals", url=url) flatten_externals(dirs, "zlib", version) # Update externals.spdx.json if env["RELENV_PY_MAJOR_VERSION"] in ["3.12", "3.13", "3.14"]: spdx_json = dirs.source / "Misc" / "externals.spdx.json" if spdx_json.exists(): with open(str(spdx_json)) as f: data = json.load(f) for pkg in data["packages"]: if pkg["name"] == "zlib": pkg["versionInfo"] = version pkg["downloadLocation"] = url pkg["checksums"][0]["checksumValue"] = sha256 pkg["externalRefs"][0]["referenceLocator"] = ref_loc with open(str(spdx_json), "w") as f: json.dump(data, f, indent=2) def update_zlib_ng(dirs: Dirs, env: EnvMapping) -> None: """ Update the zlib-ng library. Python 3.14 replaced zlib with zlib-ng for the bundled zlib module on Windows. The PCbuild project expects sources in ``externals/zlib-ng-<version>/`` (notably ``zlib.h.in`` and ``zlib-ng.h.in`` at the top level). """ zlib_ng_info = get_dependency_version("zlib-ng", "win32") if not zlib_ng_info: return version = zlib_ng_info["version"] url = zlib_ng_info["url"].format(version=version) sha256 = zlib_ng_info["sha256"] ref_loc = f"cpe:2.3:a:zlib-ng:zlib-ng:{version}:*:*:*:*:*:*:*" # noqa: E231 target_dir = dirs.source / "externals" / f"zlib-ng-{version}" update_props(dirs.source, r"zlib-ng-\d+(\.\d+)*", f"zlib-ng-{version}") if not target_dir.exists(): get_externals_source(externals_dir=dirs.source / "externals", url=url) flatten_externals(dirs, "zlib-ng", version) if env["RELENV_PY_MAJOR_VERSION"] in ["3.14"]: spdx_json = dirs.source / "Misc" / "externals.spdx.json" if spdx_json.exists(): with open(str(spdx_json)) as f: data = json.load(f) for pkg in data["packages"]: if pkg["name"] == "zlib-ng": pkg["versionInfo"] = version pkg["downloadLocation"] = url pkg["checksums"][0]["checksumValue"] = sha256 pkg["externalRefs"][0]["referenceLocator"] = ref_loc with open(str(spdx_json), "w") as f: json.dump(data, f, indent=2) def update_zstd(dirs: Dirs, env: EnvMapping) -> None: """ Update the zstd library. Python 3.14 added a bundled ``_zstd`` extension whose Windows project compiles zstd from source in ``externals/zstd-<version>/``. """ zstd_info = get_dependency_version("zstd", "win32") if not zstd_info: return version = zstd_info["version"] url = zstd_info["url"].format(version=version) sha256 = zstd_info["sha256"] ref_loc = f"cpe:2.3:a:facebook:zstandard:{version}:*:*:*:*:*:*:*" # noqa: E231 target_dir = dirs.source / "externals" / f"zstd-{version}" update_props(dirs.source, r"zstd-\d+(\.\d+)*", f"zstd-{version}") if not target_dir.exists(): get_externals_source(externals_dir=dirs.source / "externals", url=url) flatten_externals(dirs, "zstd", version) if env["RELENV_PY_MAJOR_VERSION"] in ["3.14"]: spdx_json = dirs.source / "Misc" / "externals.spdx.json" if spdx_json.exists(): with open(str(spdx_json)) as f: data = json.load(f) for pkg in data["packages"]: if pkg["name"] == "zstd": pkg["versionInfo"] = version pkg["downloadLocation"] = url pkg["checksums"][0]["checksumValue"] = sha256 pkg["externalRefs"][0]["referenceLocator"] = ref_loc with open(str(spdx_json), "w") as f: json.dump(data, f, indent=2) def update_mpdecimal(dirs: Dirs, env: EnvMapping) -> None: """ Update the MPDECIMAL library. """ mpdecimal_info = get_dependency_version("mpdecimal", "win32") if not mpdecimal_info: return version = mpdecimal_info["version"] url = mpdecimal_info["url"].format(version=version) target_dir = dirs.source / "externals" / f"mpdecimal-{version}" update_props(dirs.source, r"mpdecimal-\d+(\.\d+)*", f"mpdecimal-{version}") if not target_dir.exists(): get_externals_source(externals_dir=dirs.source / "externals", url=url) flatten_externals(dirs, "mpdecimal", version) def update_nasm(dirs: Dirs, env: EnvMapping) -> None: """ Update the NASM library. """ nasm_info = get_dependency_version("nasm", "win32") if not nasm_info: return version = nasm_info["version"] url = nasm_info["url"].format(version=version) target_dir = dirs.source / "externals" / f"nasm-{version}" update_props(dirs.source, r"nasm-\d+(\.\d+)*", f"nasm-{version}") if not target_dir.exists(): get_externals_source(externals_dir=dirs.source / "externals", url=url) flatten_externals(dirs, "nasm", version) def update_perl(dirs: Dirs, env: EnvMapping) -> pathlib.Path: """ Update the Perl library. """ perl_info = get_dependency_version("perl", "win32") if not perl_info: return None version = perl_info["version"] url = perl_info["url"].format(version=version) target_dir = dirs.source / "externals" / f"perl-{version}" if not target_dir.exists(): target_dir.mkdir(parents=True, exist_ok=True) get_externals_source(externals_dir=target_dir, url=url) return target_dir def copy_pyconfig_h(source: pathlib.Path, build_dir: pathlib.Path, dest_dir: pathlib.Path) -> pathlib.Path: """ Copy ``pyconfig.h`` into the onedir's ``Include`` directory. Python <= 3.12 ships a checked-in ``PC/pyconfig.h``. Python 3.13+ replaced that with ``PC/pyconfig.h.in`` and MSBuild generates the real header into the build output directory. This mirrors the logic in CPython's ``PC/layout/main.py`` so the onedir's ``Include/`` always ends up with a ``pyconfig.h`` next to ``Python.h`` -- without it C extensions cannot locate the configuration macros and fail with ``fatal error C1083: Cannot open include file: 'pyconfig.h'``. """ pc_dir = source / "PC" if (pc_dir / "pyconfig.h.in").is_file(): pyconfig_src = build_dir / "pyconfig.h" else: pyconfig_src = pc_dir / "pyconfig.h" if not pyconfig_src.is_file(): raise RuntimeError( f"Expected pyconfig.h at {pyconfig_src} but CPython build did " "not produce it. Check that the MSBuild step ran successfully." ) dest = dest_dir / "pyconfig.h" shutil.copy(src=str(pyconfig_src), dst=str(dest)) return dest def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: """ Run the commands to build Python. """ externals_dir = dirs.source / "externals" externals_dir.mkdir(parents=True, exist_ok=True) update_sqlite(dirs=dirs, env=env) update_xz(dirs=dirs, env=env) update_expat(dirs=dirs, env=env) update_bzip2(dirs=dirs, env=env) update_libffi(dirs=dirs, env=env) update_zlib(dirs=dirs, env=env) update_zlib_ng(dirs=dirs, env=env) update_zstd(dirs=dirs, env=env) update_mpdecimal(dirs=dirs, env=env) update_nasm(dirs=dirs, env=env) update_perl(dirs=dirs, env=env) update_openssl(dirs=dirs, env=env) # Disable SBOM validation in Python 3.12+ regen_targets = dirs.source / "PCbuild" / "regen.targets" if regen_targets.exists(): log.info("Patching regen.targets to skip SBOM generation") patch_file( regen_targets, r'Command="py -3\.\d+ .*generate_sbom\.py.*"', 'Command="echo skipping sbom"', ) # Secondary defense: overwrite the script itself if it exists sbom_script = dirs.source / "Tools" / "build" / "generate_sbom.py" if sbom_script.exists(): with open(str(sbom_script), "w") as f: f.write("import sys\nif __name__ == '__main__':\n sys.exit(0)\n") # Disable get_externals.bat to avoid network fetches during MSBuild batch_file = dirs.source / "PCbuild" / "get_externals.bat" if batch_file.exists(): with open(str(batch_file), "w") as f: f.write("@echo off\necho skipping fetch\n") arch_to_plat = {"amd64": "x64", "x86": "win32", "arm64": "arm64"} arch = env["RELENV_HOST_ARCH"] plat = arch_to_plat[arch] # -e skips fetching externals if they already exist. cmd = [ str(dirs.source / "PCbuild" / "build.bat"), "-e", "-p", plat, "--no-tkinter", ] log.info("Start PCbuild") runcmd(cmd, env=env, stderr=logfp, stdout=logfp) log.info("PCbuild finished") if arch == "amd64": build_dir = dirs.source / "PCbuild" / arch else: build_dir = dirs.source / "PCbuild" / plat bin_dir = dirs.prefix / "Scripts" bin_dir.mkdir(parents=True, exist_ok=True) binaries = [ "py.exe", "pyw.exe", "python.exe", "pythonw.exe", "python3.dll", f"python{env['RELENV_PY_MAJOR_VERSION'].replace('.', '')}.dll", "vcruntime140.dll", "venvlauncher.exe", "venvwlauncher.exe", ] for binary in binaries: shutil.move(src=str(build_dir / binary), dst=str(bin_dir / binary)) (dirs.prefix / "DLLs").mkdir(parents=True, exist_ok=True) for file in glob.glob(str(build_dir / "*.pyd")): shutil.move(src=file, dst=str(dirs.prefix / "DLLs")) for file in glob.glob(str(build_dir / "*.dll")): shutil.move(src=file, dst=str(dirs.prefix / "DLLs")) shutil.copytree( src=str(dirs.source / "Include"), dst=str(dirs.prefix / "Include"), dirs_exist_ok=True, ) copy_pyconfig_h(dirs.source, build_dir, dirs.prefix / "Include") shutil.copytree(src=str(dirs.source / "Lib"), dst=str(dirs.prefix / "Lib"), dirs_exist_ok=True) os.makedirs(str(dirs.prefix / "Lib" / "site-packages"), exist_ok=True) (dirs.prefix / "libs").mkdir(parents=True, exist_ok=True) shutil.copy( src=str(build_dir / "python3.lib"), dst=str(dirs.prefix / "libs" / "python3.lib"), ) pylib = f"python{env['RELENV_PY_MAJOR_VERSION'].replace('.', '')}.lib" shutil.copy(src=str(build_dir / pylib), dst=str(dirs.prefix / "libs" / pylib)) build = builds.add("win32", populate_env=populate_env) build.add( "python", build_func=build_python, download={ "url": "https://www.python.org/ftp/python/{version}/Python-{version}.tar.xz", "version": build.version, "checksum": None, }, ) def finalize(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: """ Finalize sitecustomize, relenv runtime, and pip for Windows. """ sitepackages = dirs.prefix / "Lib" / "site-packages" install_runtime(sitepackages) update_ensurepip(dirs.prefix / "Lib") python = dirs.prefix / "Scripts" / "python.exe" runcmd([str(python), "-m", "ensurepip"], env=env, stderr=logfp, stdout=logfp) def runpip(pkg: str | os.PathLike[str]) -> None: env = os.environ.copy() cmd = [str(python), "-m", "pip", "install", str(pkg)] runcmd(cmd, env=env, stderr=logfp, stdout=logfp) runpip("wheel") if (MODULE_DIR.parent / ".git").exists(): runpip(MODULE_DIR.parent) else: runpip("relenv") for root, _, files in os.walk(dirs.prefix): for file in files: if file.endswith(".pyc"): os.remove(pathlib.Path(root) / file) globs = [ "*.exe", "*.py", "*.pyd", "*.dll", "*.lib", "*.whl", "/Include/*", "/Lib/site-packages/*", ] archive = f"{dirs.prefix}.tar.xz" with tarfile.open(archive, mode="w:xz") as fp: create_archive(fp, dirs.prefix, globs, logfp) build.add("relenv-finalize", build_func=finalize, wait_on=["python"])