diff --git a/CHANGES.md b/CHANGES.md index e204aef..045b3ab 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,14 @@ # Release Notes +## 0.16.1 + +This release fixes interpreter detection for `pyenv` shims on unix. Previously the cache of +interpreter information for the shimmed Python was based on the shim and not the Python executable +it resolved to. + +In addition, interpreter discovery on Windows is fixed to only discover `python.exe` / `pypy.exe`. +The windowed versions are resolved based off the console versions just in time when needed. + ## 0.16.0 This release fixes interpreter discovery for Windows. The [PEP-514 spec]( diff --git a/Cargo.lock b/Cargo.lock index d95680d..4c745f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1458,7 +1458,6 @@ version = "0.0.0" dependencies = [ "anyhow", "cache", - "elf", "fs-err", "indexmap 2.14.0", "log", @@ -2097,7 +2096,7 @@ dependencies = [ [[package]] name = "pexrc" -version = "0.16.0" +version = "0.16.1" dependencies = [ "anstream", "anyhow", @@ -2122,7 +2121,6 @@ dependencies = [ "logging", "owo-colors", "pex", - "pexrs", "platform", "python-platform", "python-proxy", diff --git a/Cargo.toml b/Cargo.toml index 95766f2..4221965 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ cargo-features = ["profile-rustflags"] [package] name = "pexrc" -version = "0.16.0" +version = "0.16.1" edition = { workspace = true } publish = false @@ -247,7 +247,6 @@ log = { workspace = true } logging = { path = "crates/logging" } owo-colors = { workspace = true } pex = { path = "crates/pex" } -pexrs = { path = "crates/pexrs" } platform = { path = "crates/platform" } python-platform = { path = "crates/python-platform" } python-proxy = { path = "crates/python-proxy" } diff --git a/crates/cache/src/lib.rs b/crates/cache/src/lib.rs index 533e1ab..a2dad13 100644 --- a/crates/cache/src/lib.rs +++ b/crates/cache/src/lib.rs @@ -168,9 +168,9 @@ impl CacheDir { fn version(&self) -> &'static str { match self { - CacheDir::Interpreter => "1", + CacheDir::Interpreter => "2", CacheDir::PythonProxy => "0", - CacheDir::Venv => "0", + CacheDir::Venv => "1", } } diff --git a/crates/interpreter/Cargo.toml b/crates/interpreter/Cargo.toml index a10a5e1..ecf3928 100644 --- a/crates/interpreter/Cargo.toml +++ b/crates/interpreter/Cargo.toml @@ -12,7 +12,6 @@ cache = { path = "../cache" } indexmap = { workspace = true } pep440_rs = { workspace = true } pep508_rs = { workspace = true } -elf = { workspace = true } fs-err = { workspace = true } log = { workspace = true } logging_timer = { workspace = true } diff --git a/crates/interpreter/src/constraints/windows.rs b/crates/interpreter/src/constraints/windows.rs index c5a69f7..2368d5e 100644 --- a/crates/interpreter/src/constraints/windows.rs +++ b/crates/interpreter/src/constraints/windows.rs @@ -228,7 +228,12 @@ fn iter_python_exes( && let Some(file_stem) = file_stem.to_str() { for name in &names { - if file_stem.starts_with(name) { + if file_stem.starts_with(name) + // N.B.: Since this is a {PEX_PYTHON_,}PATH search, + // we may pick up pythonw.exe, pypyw.exe, etc. but + // we only want to report the console pythons. + && !file_stem.ends_with("w") + { yield file; break; } diff --git a/crates/interpreter/src/interpreter.rs b/crates/interpreter/src/interpreter.rs index 70cfa17..cd07019 100644 --- a/crates/interpreter/src/interpreter.rs +++ b/crates/interpreter/src/interpreter.rs @@ -331,8 +331,16 @@ impl Interpreter { python_exe: &Path, identification_script: &IdentifyInterpreter, ) -> anyhow::Result { - let interpreter_info = Self::interpreter_info(python_exe)?; - Self::load_internal(&interpreter_info, python_exe, identification_script) + #[cfg(unix)] + let python_exe = crate::pyenv::Pyenv::locate() + .map(|pyenv| pyenv.resolve_if_shim(python_exe)) + .unwrap_or(Ok(Cow::Borrowed(python_exe)))?; + let interpreter_info = Self::interpreter_info(python_exe.as_path())?; + Self::load_internal( + &interpreter_info, + python_exe.as_path(), + identification_script, + ) } fn load_internal( diff --git a/crates/interpreter/src/lib.rs b/crates/interpreter/src/lib.rs index d5dc66d..4535840 100644 --- a/crates/interpreter/src/lib.rs +++ b/crates/interpreter/src/lib.rs @@ -6,11 +6,13 @@ #![cfg_attr(windows, feature(coroutine_trait))] #![cfg_attr(windows, feature(iter_from_coroutine))] #![feature(stmt_expr_attributes)] +#![feature(str_as_str)] mod constraints; mod interpreter; mod platform; +mod pyenv; mod search_path; mod tag; mod version; diff --git a/crates/interpreter/src/pyenv.rs b/crates/interpreter/src/pyenv.rs new file mode 100644 index 0000000..3c8c336 --- /dev/null +++ b/crates/interpreter/src/pyenv.rs @@ -0,0 +1,183 @@ +// Copyright 2026 Pex project contributors. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(unix)] + +use std::borrow::Cow; +use std::env; +use std::env::VarError; +use std::ffi::OsStr; +use std::fmt::Display; +use std::io::{BufRead, BufReader, Read}; +use std::os::unix::ffi::OsStrExt; +use std::path::{Component, Path, PathBuf}; + +use anyhow::{Context, bail}; +use fs_err::File; +use log::warn; +use logging_timer::time; +use python_platform::Implementation; + +pub(crate) struct Pyenv(PathBuf); + +impl Pyenv { + pub(crate) fn locate() -> Option { + env::var_os("PYENV_ROOT").map(PathBuf::from).map(Self) + } + + #[time("debug", "Pyenv.{}")] + pub(crate) fn resolve_if_shim<'a>(&self, path: &'a Path) -> anyhow::Result> { + if let Ok(rel_path) = path.strip_prefix(&self.0) + && let Some(component) = rel_path.components().next() + && matches!(component, Component::Normal(name) if name == OsStr::from_bytes(b"shims")) + && let Some(shim) = PythonShim::parse(path)? + { + Ok(Cow::Owned(shim.select_version(&self.0)?)) + } else { + Ok(Cow::Borrowed(path)) + } + } +} + +struct PythonShim<'a> { + path: &'a Path, + implementation: Implementation, + version: &'a str, +} + +impl<'a> PythonShim<'a> { + fn parse(path: &'a Path) -> anyhow::Result> { + // Sanity-check this is a shell script, which all shims should be. + let mut file = File::open(path)?; + let mut buf: [u8; 2] = [0; 2]; + file.read_exact(&mut buf).with_context(|| { + format!( + "Failed to confirm {path} as a pyenv shim script", + path = path.display() + ) + })?; + if &buf != b"#!" { + bail!( + "Although {path} is under the pyenv shims dir, it is not a shim script.\n\ + First two bytes are {buf:?}", + path = path.display() + ); + } + + if let Some(file_name) = path.file_name() + && let Some(file_name) = file_name.to_str() + && let Some((implementation, version)) = file_name + .strip_prefix("python") + .map(|version| (Implementation::CPython, version)) + .or_else(|| { + file_name + .strip_prefix("pypy") + .map(|version| (Implementation::PyPy, version)) + }) + { + return Ok(Some(Self { + path, + implementation, + version, + })); + } + Ok(None) + } + + fn select_version(&self, pyenv_root: &Path) -> anyhow::Result { + if let Some((source, active_versions)) = active_versions(pyenv_root)? { + for version in &active_versions { + if let Some(search_version) = match self.implementation { + Implementation::CPython => { + Some(version.strip_prefix("pypy").unwrap_or(version)) + } + Implementation::PyPy => version.strip_prefix("pypy"), + } && search_version.starts_with(self.version) + { + let mut versions_dir = pyenv_root.join("versions"); + versions_dir.push(version); + versions_dir.push("bin"); + versions_dir.push("python"); + if let Ok(python_exe) = versions_dir.canonicalize() { + return Ok(python_exe); + } + } + } + + struct ActivatedVersions(Vec); + impl Display for ActivatedVersions { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + for version in &self.0 { + writeln!(f, "+ {version}")?; + } + Ok(()) + } + } + let activated_versions = ActivatedVersions(active_versions); + bail!( + "The pyenv shim at {path} has no corresponding versions activated.\n\ + Versions activated via {source} are:\n\ + {activated_versions}", + path = self.path.display() + ) + } else { + bail!( + "There are no versions activated for pyenv shim at {path}", + path = self.path.display() + ) + } + } +} + +fn active_versions(pyenv_root: &Path) -> anyhow::Result)>> { + // See: https://github.com/pyenv/pyenv/blob/c425c9ec3a409a8d432bc9f4851da8e3f040bddc/README.md#understanding-python-version-selection + if let Ok(shell_versions) = env::var("PYENV_VERSION").inspect_err(|err| { + if let VarError::NotUnicode(value) = err { + warn!( + "Skipping non-utf8 env setting PYENV_VERSION={value}", + value = value.display() + ) + } + }) { + return Ok(Some(( + format!("PYENV_VERSION={shell_versions}"), + shell_versions.split(":").map(str::to_string).collect(), + ))); + } else { + let start = env::var_os("PYENV_DIR") + .map(PathBuf::from) + .map(Ok) + .unwrap_or_else(env::current_dir) + .context("Failed to determine starting directory for pyenv version file search.")?; + for dir in start.ancestors() { + if let Ok(local_version_file) = File::open(dir.join(".python-version")) { + return Ok(Some(( + local_version_file.path().display().to_string(), + read_versions(local_version_file), + ))); + } + } + if let Ok(global_version_file) = File::open(pyenv_root.join("version")) { + return Ok(Some(( + global_version_file.path().display().to_string(), + read_versions(global_version_file), + ))); + } + } + Ok(None) +} + +fn read_versions(version_file: impl Read) -> Vec { + BufReader::new(version_file) + .lines() + .map_while(Result::ok) + .filter_map(|line| { + let contents = line.trim(); + if contents.starts_with("#") { + None + } else { + Some(contents.to_string()) + } + }) + .collect() +} diff --git a/crates/python-platform/src/implementation.rs b/crates/python-platform/src/implementation.rs index 55bcafd..b1e49cc 100644 --- a/crates/python-platform/src/implementation.rs +++ b/crates/python-platform/src/implementation.rs @@ -6,7 +6,7 @@ use std::str::FromStr; use anyhow::bail; #[derive(Copy, Clone)] -pub(crate) enum Implementation { +pub enum Implementation { CPython, PyPy, } diff --git a/crates/python-platform/src/lib.rs b/crates/python-platform/src/lib.rs index 5268da8..85a14b7 100644 --- a/crates/python-platform/src/lib.rs +++ b/crates/python-platform/src/lib.rs @@ -28,7 +28,7 @@ use pep508_rs::pep440_rs::Version; use serde::{Deserialize, Serialize}; pub use crate::arch::Arch; -use crate::implementation::Implementation; +pub use crate::implementation::Implementation; pub use crate::linux::LinuxInfo; use crate::mac::Release; pub use crate::markers::{PlatformRelease, PlatformVersion}; diff --git a/crates/scripts/src/interpreter.py b/crates/scripts/src/interpreter.py index bcc63b1..d1c47ee 100644 --- a/crates/scripts/src/interpreter.py +++ b/crates/scripts/src/interpreter.py @@ -51,7 +51,7 @@ def cpython_abi_info(sys_config_vars): # extension modules is the best option. # https://github.com/pypa/pip/issues/3383#issuecomment-173267692 has_ext = "_d.pyd" in EXTENSION_SUFFIXES - if with_debug or (with_debug is None and (has_refcount or has_ext)): + if (with_debug is None and (has_refcount or has_ext)) or bool(with_debug): debug = True if sys.version_info >= (3, 13): @@ -59,7 +59,7 @@ def cpython_abi_info(sys_config_vars): if sys.version_info < (3, 8): with_pymalloc = sys_config_vars.get("WITH_PYMALLOC") - pymalloc = with_pymalloc or with_pymalloc is None + pymalloc = with_pymalloc is None or bool(with_pymalloc) if sys.version_info < (3, 3): unicode_size = sys_config_vars.get("Py_UNICODE_SIZE") diff --git a/src/commands/python/list.rs b/src/commands/python/list.rs index 141aa26..426da69 100644 --- a/src/commands/python/list.rs +++ b/src/commands/python/list.rs @@ -1,8 +1,6 @@ // Copyright 2026 Pex project contributors. // SPDX-License-Identifier: Apache-2.0 -use std::path::PathBuf; - use clap::{ArgAction, Args}; use cli::{Json, Output}; use interpreter::{ @@ -12,7 +10,9 @@ use interpreter::{ SearchPath, SelectionStrategy, }; +use log::debug; use owo_colors::OwoColorize; +use rayon::iter::{IntoParallelIterator, ParallelIterator}; use scripts::{IdentifyInterpreter, Scripts}; #[derive(Args)] @@ -28,9 +28,9 @@ pub struct List { #[command(flatten)] output: Output, - /// Constrain the interpreters to those meeting any of the given constraints. + /// Limit the interpreters to those meeting any of the given constraints. /// - /// Interpreter constraints are composed of implementation names and a version specifiers in + /// Interpreter constraints are composed of implementation names and version specifiers in /// one of the following forms: /// + `` /// + `` @@ -46,8 +46,8 @@ pub struct List { /// /// Some examples: /// + `PyPy`: any PyPy interpreter - /// + `==3.14.*`: any Python interpreter version 3.14 - /// + `CPython>=3.12`: any CPython interpreter version 3.12 or greater + /// + `>=3.12`: any Python interpreter version 3.12 or greater + /// + `CPython[free-threaded]==3.14.*`: any free-threaded CPython 3.14 interpreter /// /// [^1]: https://peps.python.org/pep-0440/ /// [^2]: https://packaging.python.org/specifications/version-specifiers/#version-specifiers @@ -67,28 +67,30 @@ pub struct List { impl List { pub fn execute(self) -> anyhow::Result<()> { let ics = InterpreterConstraints::from(self.constraints); - let pythons = ics.iter_possibly_compatible_python_exes( - SelectionStrategy::Newest, - SearchPath::from_env()?, - false, - )?; - let pythons: Vec = if !ics.is_empty() { + let mut pythons = ics + .iter_possibly_compatible_python_exes( + SelectionStrategy::Newest, + SearchPath::from_env()?, + false, + )? + .collect::>(); + + if !ics.is_empty() { let identification_script = IdentifyInterpreter::read(&mut Scripts::Embedded)?; - pythons - .filter_map(|python| { - if let Some(interpreter) = - Interpreter::load(&python, &identification_script).ok() - && ics.contains(&interpreter) - { - Some(python) - } else { - None - } - }) + pythons = pythons + .into_par_iter() + .filter_map( + |python| match Interpreter::load(&python, &identification_script) { + Ok(interpreter) if ics.contains(&interpreter) => Some(python), + Err(err) => { + debug!("Failed to load {python}: {err}", python = python.display()); + None + } + _ => None, + }, + ) .collect() - } else { - pythons.collect() - }; + } let mut out = self.output.writer()?; if self.json {